From 803b9088c2f94c288b135a5e0c5887109aa8750b Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 5 Feb 2025 13:15:41 -0800 Subject: [PATCH 01/87] updated metadata.json with all documentation links --- scenarios/metadata.json | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/scenarios/metadata.json b/scenarios/metadata.json index bb8d8aeb3..a111e0012 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -261,7 +261,7 @@ "description": "This tutorial shows how to deploy Inspektor Gadget in an AKS cluster", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/DeployIGonAKS/deploy-ig-on-aks.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/azure-kubernetes/logs/capture-system-insights-from-aks", "nextSteps": [ { "title": "Real-world scenarios where Inspektor Gadget can help you", @@ -392,7 +392,7 @@ "description": "Learn how to obtainer Performance metrics from a Linux system.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/ObtainPerformanceMetricsLinuxSustem/obtain-performance-metrics-linux-system.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/collect-performance-metrics-from-a-linux-system", "configurations": { "permissions": [], "configurableParams": [ @@ -418,7 +418,7 @@ "description": "Create the infrastructure needed to deploy a highly available PostgreSQL database on AKS using the CloudNativePG operator.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/create-postgresql-ha.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/create-postgresql-ha?tabs=helm", "nextSteps": [ { "title": "Deploy a highly available PostgreSQL database on AKS with Azure CLI", @@ -436,18 +436,7 @@ "description": "In this article, you deploy a highly available PostgreSQL database on AKS using the CloudNativePG operator.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/deploy-postgresql-ha.md", - "documentationUrl": "", - "configurations": { - } - }, - { - "status": "inactive", - "key": "azure-aks-docs/articles/aks/postgresql-ha-overview.md", - "title": "Overview of deploying a highly available PostgreSQL database on AKS with Azure CLI", - "description": "Learn how to deploy a highly available PostgreSQL database on AKS using the CloudNativePG operator.", - "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/postgresql-ha-overview.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/deploy-postgresql-ha", "configurations": { } }, @@ -458,7 +447,7 @@ "description": "This tutorial shows how to create a Container App leveraging Blob Store, SQL, and Computer Vision", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/CreateContainerAppDeploymentFromSource/create-container-app-deployment-from-source.md", - "documentationUrl": "", + "documentationUrl": "https://github.com/Azure/computer-vision-nextjs-webapp", "nextSteps": [ { "title": "Azure Container Apps documentation", @@ -496,7 +485,7 @@ } }, { - "status": "active", + "status": "inactive", "key": "AIChatApp/ai-chat-app.md", "title": "Create an Azure OpenAI, LangChain, ChromaDB, and Chainlit Chat App in Container Apps", "description": "", @@ -541,7 +530,7 @@ "description": "In this article, you create the infrastructure needed to deploy Apache Airflow on Azure Kubernetes Service (AKS) using Helm.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/airflow-create-infrastructure.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/airflow-create-infrastructure", "nextSteps": [ { "title": "Deploy Apache Airflow on AKS", @@ -559,7 +548,7 @@ "description": "In this article, you create the infrastructure needed to deploy Apache Airflow on Azure Kubernetes Service (AKS) using Helm.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/airflow-deploy.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/airflow-deploy", "nextSteps": [ { "title": "Deploy a MongoDB cluster on Azure Kubernetes Service (AKS)", @@ -639,7 +628,7 @@ "description": "Learn how to use the Azure CLI to create an Azure OpenAI resource and manage deployments with the Azure OpenAI Service.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/CreateAOAIDeployment/create-aoai-deployment.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=cli", "nextSteps": [], "configurations": { "permissions": [] @@ -652,7 +641,7 @@ "description": "Learn how to enable the AI toolchain operator add-on on Azure Kubernetes Service (AKS) to simplify OSS AI model management and deployment", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/AKSKaito/aks-kaito.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/ai-toolchain-operator", "nextSteps": [ { "title": "Check out the KAITO GitHub repository", @@ -738,7 +727,7 @@ "description": "Learn how to create an Azure Kubernetes Service (AKS) cluster with enclave confidential containers a Hello World app by using the Azure CLI.", "stackDetails": [], "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/confidential-computing/confidential-enclave-nodes-aks-get-started.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/confidential-computing/confidential-enclave-nodes-aks-get-started", "nextSteps": [ { "title": "Samples to run Python, Node, and other applications through confidential containers", @@ -760,7 +749,7 @@ "description": "Learn how to quickly create an Azure Linux Container Host for AKS cluster using the Azure CLI.", "stackDetails": [], "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-management-docs/articles/azure-linux/quickstart-azure-cli.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/azure-linux/quickstart-azure-cli", "nextSteps": [ { "title": "Azure Linux Container Host tutorial", @@ -779,7 +768,7 @@ "description": "Learn how to use the Azure CLI to create a custom VM image that you can use to deploy a Virtual Machine Scale Set", "stackDetails": [], "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-use-custom-image-cli", "nextSteps": [ { "title": "Deploy applications to your scale sets", @@ -840,7 +829,7 @@ "description": "In this Azure Kubernetes Service (AKS) article, you learn how to configure your Azure Kubernetes Service pod to authenticate with workload identity.", "stackDetails": [], "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/workload-identity-migrate-from-pod-identity.md", - "documentationUrl": "", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/workload-identity-migrate-from-pod-identity", "nextSteps": [ { "title": "Use Microsoft Entra Workload ID with Azure Kubernetes Service (AKS)", From 19fd6992e25986bccb6bd7f649eb3d0661b3858d Mon Sep 17 00:00:00 2001 From: naman-msft Date: Tue, 25 Feb 2025 18:11:45 -0800 Subject: [PATCH 02/87] added 2 new support docs --- .../fix-fstab-issues-repair-vm.md | 88 +++++++++++++++++++ .../kernel-related-boot-issues-repairvm.md | 84 ++++++++++++++++++ scenarios/metadata.json | 52 +++++++++++ 3 files changed, 224 insertions(+) create mode 100644 scenarios/FixFstabIssuesRepairVM/fix-fstab-issues-repair-vm.md create mode 100644 scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md diff --git a/scenarios/FixFstabIssuesRepairVM/fix-fstab-issues-repair-vm.md b/scenarios/FixFstabIssuesRepairVM/fix-fstab-issues-repair-vm.md new file mode 100644 index 000000000..81e5392f1 --- /dev/null +++ b/scenarios/FixFstabIssuesRepairVM/fix-fstab-issues-repair-vm.md @@ -0,0 +1,88 @@ +--- +title: Troubleshoot Linux VM boot issues due to fstab errors | Microsoft Learn +description: Explains why Linux VM cannot start and how to solve the problem. +services: virtual-machines +documentationcenter: '' +author: divargas-msft +ms.author: divargas +manager: dcscontentpm +tags: '' +ms.custom: sap:My VM is not booting, linux-related-content, devx-track-azurecli, mode-api, innovation-engine +ms.service: azure-virtual-machines +ms.collection: linux +ms.topic: troubleshooting +ms.workload: infrastructure-services +ms.tgt_pltfrm: vm-linux +ms.devlang: azurecli +ms.date: 02/25/2025 +--- + + +# Troubleshoot Linux VM boot issues due to fstab errors + +**Applies to:** :heavy_check_mark: Linux VMs + + + +The Linux filesystem table, fstab is a configuration table which is designed to configure rules where specific file systems are detected and mounted in an orderly manner during the system boot process. +This article discusses multiple conditions where a wrong fstab configuration can lead to boot issue and provides troubleshooting guidance. + +Few common reasons that can lead to Virtual Machine Boot issues due to fstab misconfiguration are listed below: + +* Traditional filesystem name is used instead of the Universally Unique Identifier (UUID) of the filesystem. +* An incorrect UUID is used. +* An entry exists for an unattached device without `nofail` option within fstab configuration. +* Incorrect entry within fstab configuration. + +## Identify fstab issues + +Check the current boot state of the VM in the serial log within the [Boot diagnostics] (/azure/virtual-machines/boot-diagnostics#boot-diagnostics-view) blade in the Azure portal. The VM will be in an Emergency Mode. You see log entries that resemble the following example leading to the Emergency Mode state: + +```output +[K[[1;31m TIME [0m] Timed out waiting for device dev-incorrect.device. +[[1;33mDEPEND[0m] Dependency failed for /data. +[[1;33mDEPEND[0m] Dependency failed for Local File Systems. +... +Welcome to emergency mode! After logging in, type "journalctl -xb" to viewsystem logs, "systemctl reboot" to reboot, "systemctl default" to try again to boot into default mode. +Give root password for maintenance +(or type Control-D to continue) +``` + + >[!Note] + > "/data" is an example of mount point used. Dependency failure for filesystem mount point will differ based on the names used. + +## Resolution + +There are 2 ways to resolve the issue: + +* Repair the VM online + * [Use the Serial Console](#use-the-serial-console) +* Repair the vm offline + * [Use Azure Linux Auto Repair (ALAR)](#use-azure-linux-auto-repair-alar) + * [Use Manual Method](#use-manual-method) + +#### Use Azure Linux Auto Repair (ALAR) + +Azure Linux Auto Repair (ALAR) scripts is a part of VM repair extension described in [Repair a Linux VM by using the Azure Virtual Machine repair commands](./repair-linux-vm-using-azure-virtual-machine-repair-commands.md). ALAR covers automation of multiple repair scenarios including `/etc/fstab` issues. + +The ALAR scripts use the repair extension `run` command and its `--run-id` option. The script-id for the automated recovery is: **linux-alar2**. Implement the following steps to automate fstab errors via offline ALAR approach: + +```azurecli-interactive +output=$(az extension add -n vm-repair; az extension update -n vm-repair; az vm repair repair-button --button-command 'fstab' --verbose --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME) +value=$(echo "$output" | jq -r '.message') +extracted=$(echo $value) +echo "$extracted" +``` + +> [!NOTE] +> The fstab repair script will take a backup of the original file and strip off any lines in the /etc/fstab file which are not needed to boot a system. After successful start of the OS, edit the fstab again and correct any errors which didn't allow a reboot of the system before. + +[!INCLUDE [Azure Help Support](../../../includes/azure-help-support.md)] \ No newline at end of file diff --git a/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md b/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md new file mode 100644 index 000000000..354e5e38e --- /dev/null +++ b/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md @@ -0,0 +1,84 @@ +--- +title: Recover Azure Linux VM from kernel panic due to missing initramfs +description: Provides solutions to an issue in which a Linux virtual machine (VM) can't boot after applying kernel changes. +author: divargas-msft +ms.author: divargas +ms.date: 02/25/2025 +ms.reviewer: jofrance +ms.service: azure-virtual-machines +ms.custom: sap:Cannot start or stop my VM, devx-track-azurecli, mode-api, innovation-engine, linux-related-content +ms.workload: infrastructure-services +ms.tgt_pltfrm: vm-linux +ms.collection: linux +ms.topic: troubleshooting +--- + +# Azure Linux virtual machine fails to boot after applying kernel changes + +**Applies to:** :heavy_check_mark: Linux VMs + + + + +## Prerequisites + +Make sure the [serial console](serial-console-linux.md) is enabled and functional in the Linux VM. + +## Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0) + +This error occurs because of a recent system update (kernel). It's most commonly seen in RHEL-based distributions. +You can [identify this issue from the Azure serial console](#identify-kernel-boot-issue). You'll see any of the following error messages: + +1. "Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0)" + + ```output + [ 301.026129] Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0) + [ 301.027122] CPU: 0 PID: 1 Comm: swapper/0 Tainted: G ------------ T 3.10.0-1160.36.2.el7.x86_64 #1 + [ 301.027122] Hardware name: Microsoft Corporation Virtual Machine/Virtual Machine, BIOS 090008 12/07/2018 + [ 301.027122] Call Trace: + [ 301.027122] [] dump_stack+0x19/0x1b + [ 301.027122] [] panic+0xe8/0x21f + [ 301.027122] [] mount_block_root+0x291/0x2a0 + [ 301.027122] [] mount_root+0x53/0x56 + [ 301.027122] [] prepare_namespace+0x13c/0x174 + [ 301.027122] [] kernel_init_freeable+0x222/0x249 + [ 301.027122] [] ? initcall_blcklist+0xb0/0xb0 + [ 301.027122] [] ? rest_init+0x80/0x80 + [ 301.027122] [] kernel_init+0xe/0x100 + [ 301.027122] [] ret_from_fork_nospec_begin+0x21/0x21 + [ 301.027122] [] ? rest_init+0x80/0x80 + [ 301.027122] Kernel Offset: 0xc00000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff) + ``` + +2. "error: file '/initramfs-*.img' not found" + + > error: file '/initramfs-3.10.0-1160.36.2.el7.x86_64.img' not found. + +This kind of error indicates that the initramfs file isn't generated, the GRUB configuration file has the initrd entry missing after a patching process, or a GRUB manual misconfiguration. + +### Regenerate missing initramfs by using Azure Repair VM ALAR scripts + +1. Create a repair VM by running the following Bash command line with [Azure Cloud Shell](/azure/cloud-shell/overview). For more information, see [Use Azure Linux Auto Repair (ALAR) to fix a Linux VM - initrd option](repair-linux-vm-using-ALAR.md#initrd). This command will regenerate the initrd/initramfs image, regenerate the GRUB configuration file if it has the initrd entry missing, and swap the OS disk + +```azurecli-interactive +output=$(az extension add -n vm-repair; az extension update -n vm-repair; az vm repair repair-button --button-command 'initrd' --verbose --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME) +value=$(echo "$output" | jq -r '.message') +extracted=$(echo $value) +echo "$extracted" +``` + +2. Once the repair VM command has been executed, restart the original VM and validate that it's able to boot up. + +## Next steps + +If the specific boot error isn't a kernel related boot issue, see [Troubleshoot Azure Linux Virtual Machines boot errors](./boot-error-troubleshoot-linux.md) for further troubleshooting options. + +[!INCLUDE [Azure Help Support](../../../includes/azure-help-support.md)] \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index eba67fd3a..db5430982 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -919,5 +919,57 @@ "configurations": { "permissions": [] } + }, + { + "status": "active", + "key": "FixFstabIssuesRepairVM/fix-fstab-issues-repair-vm.md", + "title": "Troubleshoot Linux VM boot issues due to fstab errors", + "description": "Explains why Linux VM cannot start and how to solve the problem.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/FixFstabIssuesRepairVM/fix-fstab-issues-repair-vm.md", + "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/linux-virtual-machine-cannot-start-fstab-errors#use-azure-linux-auto-repair-alar", + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "MY_RESOURCE_GROUP_NAME", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "MY_VM_NAME", + "title": "VM Name", + "defaultValue": "" + } + ] + } + }, + { + "status": "active", + "key": "KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md", + "title": "Troubleshoot Linux VM boot issues due to fstab errors", + "description": "Explains why Linux VM cannot start and how to solve the problem.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md", + "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/kernel-related-boot-issues#missing-initramfs-alar", + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "MY_RESOURCE_GROUP_NAME", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "MY_VM_NAME", + "title": "VM Name", + "defaultValue": "" + } + ] + } } ] From 31829ace3fd9a210bab4527323f402044dca4a60 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Fri, 28 Feb 2025 01:14:10 -0800 Subject: [PATCH 03/87] updated ai tool --- tools/ada.py | 85 +++++++--- tools/converted_doc.md | 308 ++++++++++++++++++++++++++++++++++ tools/doc.md | 211 +++++++++++++++++++++++ tools/execution_log.csv | 70 ++++++++ tools/generated_exec_doccc.md | 272 ++++++++++++++++++++++++++++++ tools/stdout.txt | 20 --- 6 files changed, 923 insertions(+), 43 deletions(-) create mode 100644 tools/converted_doc.md create mode 100644 tools/doc.md create mode 100644 tools/generated_exec_doccc.md delete mode 100644 tools/stdout.txt diff --git a/tools/ada.py b/tools/ada.py index d97c2b9d2..9c87b2e9e 100644 --- a/tools/ada.py +++ b/tools/ada.py @@ -4,7 +4,7 @@ import sys import subprocess import shutil -import pkg_resources +from importlib.metadata import version, PackageNotFoundError import csv import time from datetime import datetime @@ -13,11 +13,11 @@ client = AzureOpenAI( api_key=os.getenv("AZURE_OPENAI_API_KEY"), - api_version="2024-02-01", + api_version="2024-12-01-preview", azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT") ) -deployment_name = 'gpt-4o' +deployment_name = 'o3-mini' REQUIRED_PACKAGES = [ 'openai', @@ -27,12 +27,17 @@ for package in REQUIRED_PACKAGES: try: - pkg_resources.get_distribution(package) - except pkg_resources.DistributionNotFound: + # Attempt to get the package version + version(package) + except PackageNotFoundError: subprocess.check_call([sys.executable, "-m", "pip", "install", package]) system_prompt = """Exec Docs is a vehicle that transforms standard markdown into interactive, executable learning content, allowing code commands within the document to be run step-by-step or “one-click”. This is powered by the Innovation Engine, an open-source CLI tool that powers the execution and testing of these markdown scripts and can integrate with automated CI/CD pipelines. You are an Exec Doc writing expert. You will either write a new exec doc from scratch if no doc is attached or update an existing one if it is attached. You must adhere to the following rules while presenting your output: +## IF YOU ARE UPDATING AN EXISTING DOC + +Ensure that every piece of information outside of code blocks – such as metadata, descriptions, comments, instructions, and any other narrative content – is preserved. The final output should be a comprehensive document that retains all correct code blocks as well as the rich contextual and descriptive details from the source doc, creating the best of both worlds. + ### Prerequisites Check if all prerequisites below are met before writing the Exec Doc. ***If any of the below prerequisites are not met, then either add them to the Exec Doc in progress or find another valid doc that can fulfill them. Do not move to the next step until then*** @@ -322,8 +327,9 @@ def main(): if os.path.isfile(user_input) and user_input.endswith('.md'): input_type = 'file' - with open(user_input, "r") as f: + with open(user_input, "r", encoding='latin-1') as f: input_content = f.read() + input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" else: input_type = 'workload_description' input_content = user_input @@ -335,7 +341,7 @@ def main(): if input_type == 'file': output_file = f"converted_{os.path.splitext(os.path.basename(user_input))[0]}.md" else: - output_file = "generated_exec_doc.md" + output_file = "generated_exec_doccc.md" start_time = time.time() errors_encountered = [] @@ -361,7 +367,7 @@ def main(): {"role": "system", "content": system_prompt}, {"role": "user", "content": input_content}, {"role": "assistant", "content": output_file_content}, - {"role": "user", "content": f"The following error(s) have occurred during testing:\n{errors_text}\nPlease carefully analyze these errors and make necessary corrections to the document to prevent them from happening again. Try to find different solutions if the same errors keep occurring. \nGiven that context, please think hard and don't hurry. I want you to correct the converted document in ALL instances where this error has been or can be found. Then, correct ALL other errors apart from this that you see in the doc. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"} + {"role": "user", "content": f"The following error(s) have occurred during testing:\n{errors_text}\n{additional_instruction}\n\nPlease carefully analyze these errors and make necessary corrections to the document to prevent them from happening again. Try to find different solutions if the same errors keep occurring. \nGiven that context, please think hard and don't hurry. I want you to correct the converted document in ALL instances where this error has been or can be found. Then, correct ALL other errors apart from this that you see in the doc. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"} ] ) output_file_content = response.choices[0].message.content @@ -386,7 +392,10 @@ def main(): response = client.chat.completions.create( model=deployment_name, messages=[ - f"The following errors have occurred during testing:\n{errors_text}\n{additional_instruction}\nPlease carefully analyze these errors and make necessary corrections to the document to prevent them from happening again. ONLY GIVE THE UPDATED DOC, NOTHING ELSE" + {"role": "system", "content": system_prompt}, + {"role": "user", "content": input_content}, + {"role": "assistant", "content": output_file_content}, + {"role": "user", "content": f"Take the working converted Exec Doc and merge it with the original source document provided for conversion as needed. Ensure that every piece of information outside of code blocks – such as metadata, descriptions, comments, instructions, and any other narrative content – is preserved. The final output should be a comprehensive document that retains all correct code blocks as well as the rich contextual and descriptive details from the source doc, creating the best of both worlds. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"} ] ) output_file_content = response.choices[0].message.content @@ -399,24 +408,54 @@ def main(): error_log = get_last_error_log() errors_encountered.append(error_log.strip()) errors_text = "\n\n ".join(errors_encountered) - # Process and count error messages + + # Process and categorize error messages error_counts = defaultdict(int) - for error in errors_encountered: - lines = error.strip().split('\n') - for line in lines: - if 'Error' in line or 'Exception' in line: - error_counts[line] += 1 - - # Identify repeating errors - repeating_errors = {msg: count for msg, count in error_counts.items() if count > 1} - - # Prepare additional instruction if there are repeating errors - if repeating_errors: - repeating_errors_text = "\n".join([f"Error '{msg}' has occurred {count} times." for msg, count in repeating_errors.items()]) - additional_instruction = f"The following errors have occurred multiple times:\n{repeating_errors_text}\nPlease consider trying a different approach to fix these errors." + # Extract the core error message - focus on the actual error type + error_key = "" + for line in error_log.strip().split('\n'): + if 'Error:' in line: + error_key = line.strip() + break + + if not error_key and error_log.strip(): + error_key = error_log.strip().split('\n')[0] # Use first line if no clear error + + # Store this specific error type and count occurrences + if error_key: + error_counts[error_key] += 1 + for prev_error in errors_encountered[:-1]: # Check previous errors + if error_key in prev_error: + error_counts[error_key] += 1 + + # Progressive strategies based on error repetition + strategies = [ + "Look carefully at the exact error message and fix that specific issue.", + "Simplify the code block causing the error. Break it into smaller, simpler steps.", + "Remove the result block from the code block causing the error.", + "Try a completely different command or approach that achieves the same result.", + "Fundamentally reconsider this section. Replace it with the most basic, reliable approach possible.", + "Remove the problematic section entirely and rebuild it from scratch with a minimalist approach." + ] + + # Determine which strategy to use based on error count + if error_key in error_counts: + strategy_index = min(error_counts[error_key] - 1, len(strategies) - 1) + current_strategy = strategies[strategy_index] + + additional_instruction = f""" + Error '{error_key}' has occurred {error_counts[error_key]} times. + + NEW STRATEGY: {current_strategy} + + Previous approaches aren't working. Make a significant change following this strategy. + Focus on reliability over complexity. Remember to provide valid JSON output where needed. + """ else: additional_instruction = "" + print(f"\nError: {error_log.strip()}") + print(f"\nStrategy: {additional_instruction}") attempt += 1 success = False diff --git a/tools/converted_doc.md b/tools/converted_doc.md new file mode 100644 index 000000000..5aa6c8e47 --- /dev/null +++ b/tools/converted_doc.md @@ -0,0 +1,308 @@ +--- +title: 'Quickstart: Use the Azure CLI to create a Batch account and run a job' +description: Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool. +ms.topic: quickstart +ms.date: 04/12/2023 +author: azurecli +ms.author: azurecli +ms.custom: mvc, devx-track-azurecli, mode-api, linux-related-content, innovation-engine +--- + +# Quickstart: Use the Azure CLI to create a Batch account and run a job + +This quickstart shows you how to get started with Azure Batch by using Azure CLI commands and scripts to create and manage Batch resources. You create a Batch account that has a pool of virtual machines (compute nodes). You then create and run a job with tasks that run on the pool nodes. + +After you complete this quickstart, you will understand the [key concepts of the Batch service](batch-service-workflow-features.md) and be ready to use Batch with more realistic, larger scale workloads. + +## Prerequisites + +- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] + +- Azure Cloud Shell or Azure CLI. + + You can run the Azure CLI commands in this quickstart interactively in Azure Cloud Shell. To run the commands in Cloud Shell, select **Open Cloudshell** at the upper-right corner of a code block. Select **Copy** to copy the code, and paste it into Cloud Shell to run it. You can also [run Cloud Shell from within the Azure portal](https://shell.azure.com). Cloud Shell always uses the latest version of the Azure CLI. + + Alternatively, you can [install Azure CLI locally](/cli/azure/install-azure-cli) to run the commands. The steps in this article require Azure CLI version 2.0.20 or later. Run [az version](/cli/azure/reference-index?#az-version) to see your installed version and dependent libraries, and run [az upgrade](/cli/azure/reference-index?#az-upgrade) to upgrade. If you use a local installation, ensure you are already signed in to Azure. + +>[!NOTE] +>For some regions and subscription types, quota restrictions might cause Batch account or node creation to fail or not complete. In this situation, you can request a quota increase at no charge. For more information, see [Batch service quotas and limits](batch-quota-limit.md). + +## Create a resource group + +In this section, we create an Azure resource group that will serve as a logical container for all the resources used in this quickstart. To ensure uniqueness, a random suffix is appended to the resource group name. We use the location "centralindia" consistently across all resources. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP="qsBatch$RANDOM_SUFFIX" +export LOCATION="centralindia" +az group create --name $RESOURCE_GROUP --location $LOCATION +``` + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/qsBatchxxx", + "location": "centralindia", + "managedBy": null, + "name": "qsBatchxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create a storage account + +Next, create an Azure Storage account to be linked with your Batch account. Although this quickstart doesn't directly use the storage account, real-world Batch workloads typically link a storage account to deploy applications and manage data. + +```bash +export STORAGE_ACCOUNT="mybatchstorage$RANDOM_SUFFIX" +az storage account create \ + --resource-group $RESOURCE_GROUP \ + --name $STORAGE_ACCOUNT \ + --location $LOCATION \ + --sku Standard_LRS +``` + + +```JSON +{ + "sku": { + "name": "Standard_LRS" + }, + "kind": "Storage", + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/qsBatchxxx/providers/Microsoft.Storage/storageAccounts/mybatchstoragexxx", + "location": "centralindia", + "name": "mybatchstoragexxx", + "type": "Microsoft.Storage/storageAccounts", + "statusOfPrimary": "available" +} +``` + +## Create a Batch account + +Create a Batch account in your resource group and link it with the storage account created earlier. Note that we are using the "centralindia" location to ensure consistency across resources. + +```bash +export BATCH_ACCOUNT="mybatchaccount$RANDOM_SUFFIX" +az batch account create \ + --name $BATCH_ACCOUNT \ + --storage-account $STORAGE_ACCOUNT \ + --resource-group $RESOURCE_GROUP \ + --location $LOCATION +``` + + +```JSON +{ + "accountEndpoint": "mybatchaccountxxx.centralindia.batch.azure.com", + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/qsBatchxxx/providers/Microsoft.Batch/batchAccounts/mybatchaccountxxx", + "location": "centralindia", + "name": "mybatchaccountxxx", + "resourceGroup": "qsBatchxxx", + "type": "Microsoft.Batch/batchAccounts" +} +``` + +Before proceeding with further Batch operations, sign in to your Batch account so that subsequent commands use the correct account context. A brief delay is introduced to ensure the account has propagated fully. + +```bash +az batch account login \ + --name $BATCH_ACCOUNT \ + --resource-group $RESOURCE_GROUP \ + --shared-key-auth +sleep 30 +``` + + +```JSON +{ + "message": "Login to Batch account mybatchaccountxxx in resource group qsBatchxxx was successful." +} +``` + +## Create a pool of compute nodes + +Now, create a pool of Linux compute nodes within your Batch account. In this example, we create a pool with two Standard_A1_v2 VMs running Ubuntu 20.04 LTS. This configuration provides a balance between performance and cost for this quickstart. + +```bash +export POOL_ID="myPool$RANDOM_SUFFIX" +az batch pool create \ + --id $POOL_ID \ + --image canonical:0001-com-ubuntu-server-focal:20_04-lts \ + --node-agent-sku-id "batch.node.ubuntu 20.04" \ + --target-dedicated-nodes 2 \ + --vm-size Standard_A1_v2 +``` + + +```JSON +{ + "id": "myPoolxxx", + "allocationState": "resizing", + "vmSize": "Standard_A1_v2", + "targetDedicatedNodes": 2, + "provisioningState": "InProgress" +} +``` + +Batch immediately begins creating the pool, although it may take a few minutes to allocate and start the compute nodes. To check the pool allocation state reliably and avoid JSON parsing errors, query only the allocationState property: + +```bash +az batch pool show --pool-id $POOL_ID --query "allocationState" --output json +``` + + +```JSON +"resizing" +``` + +## Create a job + +Create a Batch job that will run on the pool. A job logically groups one or more tasks and specifies common settings such as the target pool. + +```bash +export JOB_ID="myJob$RANDOM_SUFFIX" +az batch job create \ + --id $JOB_ID \ + --pool-id $POOL_ID +``` + + +```JSON +{ + "id": "myJobxxx", + "poolInfo": { + "poolId": "myPoolxxx" + }, + "priority": 0, + "onAllTasksComplete": "noAction" +} +``` + +## Create job tasks + +Batch provides several methods to deploy applications and scripts to compute nodes. In the following loop, four parallel tasks (named myTask1 through myTask4) are created. Each task runs a command that prints Batch environment variables on the compute node and then waits for 90 seconds. + +```bash +for i in {1..4} +do + az batch task create \ + --task-id myTask$i \ + --job-id $JOB_ID \ + --command-line "/bin/bash -c 'printenv | grep AZ_BATCH; sleep 90s'" +done +``` + +Each task's output will display the environment settings specific to the node where it is executed. + +## View task status + +After creating the tasks, they are queued for execution. When a compute node becomes available, the task will run. Use the following command to view the status of a specific task (for example, myTask1): + +```bash +az batch task show \ + --job-id $JOB_ID \ + --task-id myTask1 +``` + + +```JSON +{ + "id": "myTask1", + "state": "active", + "executionInfo": { + "startTime": "2023-xx-xxTxx:xx:xxZ", + "endTime": null, + "retryCount": 0, + "exitCode": null + }, + "nodeInfo": { + "nodeId": "tvm-xxxxxxxx" + } +} +``` + +An exitCode of 0 in the output indicates that the task completed successfully. The nodeId property indicates the compute node where the task ran. + +## View task output + +To display the file output generated by a task on a compute node, list the files produced by the task. In the following example, the files generated by myTask1 are listed: + +```bash +az batch task file list \ + --job-id $JOB_ID \ + --task-id myTask1 \ + --output table +``` + + +```JSON +[ + { + "Name": "stdout.txt", + "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/stdout.txt", + "Is Directory": false, + "Content Length": 695 + }, + { + "Name": "certs", + "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/certs", + "Is Directory": true + }, + { + "Name": "wd", + "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/wd", + "Is Directory": true + }, + { + "Name": "stderr.txt", + "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/stderr.txt", + "Is Directory": false, + "Content Length": 0 + } +] +``` + +To download the standard output file (stdout.txt) to your local directory, run the following command: + +```bash +az batch task file download \ + --job-id $JOB_ID \ + --task-id myTask1 \ + --file-path stdout.txt \ + --destination ./stdout.txt +``` + +You can then open the downloaded stdout.txt in a text editor. Typically, the file contains the Batch environment variables set on the compute node, such as: + +```text +AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1 +AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup +AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/certs +AZ_BATCH_ACCOUNT_URL=https://mybatchaccountxxx.centralindia.batch.azure.com/ +AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/wd +AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared +AZ_BATCH_TASK_USER=_azbatch +AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks +AZ_BATCH_JOB_ID=myJobxxx +AZ_BATCH_NODE_IS_DEDICATED=true +AZ_BATCH_NODE_ID=tvm-xxxxxxxx_2-20180703t215033z +AZ_BATCH_POOL_ID=myPoolxxx +AZ_BATCH_TASK_ID=myTask1 +AZ_BATCH_ACCOUNT_NAME=mybatchaccountxxx +AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin +``` + +## Clean up resources + +If you want to continue with Batch tutorials and samples, you can keep the Batch account and linked storage account that you created in this quickstart. There is no charge for the Batch account itself. Pools and nodes do incur charges while running, even if no jobs are active. To avoid accidental deletions during automated execution, deletion commands have been removed from this document. When you no longer need these resources, please delete the resource group and its related resources manually. + +## Next steps + +In this quickstart, you created a Batch account and a compute pool, created and ran a Batch job with tasks, and viewed task outputs generated on the compute nodes. Now that you understand the key concepts of the Batch service, you're ready to use Batch for more realistic, larger scale workloads. To dive deeper into Azure Batch, continue with the Batch tutorials. + +> [!div class="nextstepaction"] +> [Tutorial: Run a parallel workload with Azure Batch](./tutorial-parallel-python.md) \ No newline at end of file diff --git a/tools/doc.md b/tools/doc.md new file mode 100644 index 000000000..dcc7b4b61 --- /dev/null +++ b/tools/doc.md @@ -0,0 +1,211 @@ +--- +title: 'Quickstart: Use the Azure CLI to create a Batch account and run a job' +description: Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool. +ms.topic: quickstart +ms.date: 04/12/2023 +ms.custom: mvc, devx-track-azurecli, mode-api, linux-related-content +--- + +# Quickstart: Use the Azure CLI to create a Batch account and run a job + +This quickstart shows you how to get started with Azure Batch by using Azure CLI commands and scripts to create and manage Batch resources. You create a Batch account that has a pool of virtual machines, or compute nodes. You then create and run a job with tasks that run on the pool nodes. + +After you complete this quickstart, you understand the [key concepts of the Batch service](batch-service-workflow-features.md) and are ready to use Batch with more realistic, larger scale workloads. + +## Prerequisites + +- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] + +- Azure Cloud Shell or Azure CLI. + + You can run the Azure CLI commands in this quickstart interactively in Azure Cloud Shell. To run the commands in the Cloud Shell, select **Open Cloudshell** at the upper-right corner of a code block. Select **Copy** to copy the code, and paste it into Cloud Shell to run it. You can also [run Cloud Shell from within the Azure portal](https://shell.azure.com). Cloud Shell always uses the latest version of the Azure CLI. + + Alternatively, you can [install Azure CLI locally](/cli/azure/install-azure-cli) to run the commands. The steps in this article require Azure CLI version 2.0.20 or later. Run [az version](/cli/azure/reference-index?#az-version) to see your installed version and dependent libraries, and run [az upgrade](/cli/azure/reference-index?#az-upgrade) to upgrade. If you use a local installation, sign in to Azure by using the [az login](/cli/azure/reference-index#az-login) command. + +>[!NOTE] +>For some regions and subscription types, quota restrictions might cause Batch account or node creation to fail or not complete. In this situation, you can request a quota increase at no charge. For more information, see [Batch service quotas and limits](batch-quota-limit.md). + +## Create a resource group + +Run the following [az group create](/cli/azure/group#az-group-create) command to create an Azure resource group named `qsBatch` in the `eastus2` Azure region. The resource group is a logical container that holds the Azure resources for this quickstart. + +```azurecli-interactive +az group create \ + --name qsBatch \ + --location eastus2 +``` + +## Create a storage account + +Use the [az storage account create](/cli/azure/storage/account#az-storage-account-create) command to create an Azure Storage account to link to your Batch account. Although this quickstart doesn't use the storage account, most real-world Batch workloads use a linked storage account to deploy applications and store input and output data. + +Run the following command to create a Standard_LRS SKU storage account named `mybatchstorage` in your resource group: + +```azurecli-interactive +az storage account create \ + --resource-group qsBatch \ + --name mybatchstorage \ + --location eastus2 \ + --sku Standard_LRS +``` + +## Create a Batch account + +Run the following [az batch account create](/cli/azure/batch/account#az-batch-account-create) command to create a Batch account named `mybatchaccount` in your resource group and link it with the `mybatchstorage` storage account. + +```azurecli-interactive +az batch account create \ + --name mybatchaccount \ + --storage-account mybatchstorage \ + --resource-group qsBatch \ + --location eastus2 +``` + +Sign in to the new Batch account by running the [az batch account login](/cli/azure/batch/account#az-batch-account-login) command. Once you authenticate your account with Batch, subsequent `az batch` commands in this session use this account context. + +```azurecli-interactive +az batch account login \ + --name mybatchaccount \ + --resource-group qsBatch \ + --shared-key-auth +``` + +## Create a pool of compute nodes + +Run the [az batch pool create](/cli/azure/batch/pool#az-batch-pool-create) command to create a pool of Linux compute nodes in your Batch account. The following example creates a pool named `myPool` that consists of two Standard_A1_v2 size VMs running Ubuntu 20.04 LTS OS. This node size offers a good balance of performance versus cost for this quickstart example. + +```azurecli-interactive +az batch pool create \ + --id myPool \ + --image canonical:0001-com-ubuntu-server-focal:20_04-lts \ + --node-agent-sku-id "batch.node.ubuntu 20.04" \ + --target-dedicated-nodes 2 \ + --vm-size Standard_A1_v2 +``` + +Batch creates the pool immediately, but takes a few minutes to allocate and start the compute nodes. To see the pool status, use the [az batch pool show](/cli/azure/batch/pool#az-batch-pool-show) command. This command shows all the properties of the pool, and you can query for specific properties. The following command queries for the pool allocation state: + +```azurecli-interactive +az batch pool show --pool-id myPool \ + --query "allocationState" +``` + +While Batch allocates and starts the nodes, the pool is in the `resizing` state. You can create a job and tasks while the pool state is still `resizing`. The pool is ready to run tasks when the allocation state is `steady` and all the nodes are running. + +## Create a job + +Use the [az batch job create](/cli/azure/batch/job#az-batch-job-create) command to create a Batch job to run on your pool. A Batch job is a logical group of one or more tasks. The job includes settings common to the tasks, such as the pool to run on. The following example creates a job called `myJob` on `myPool` that initially has no tasks. + +```azurecli-interactive +az batch job create \ + --id myJob \ + --pool-id myPool +``` + +## Create job tasks + +Batch provides several ways to deploy apps and scripts to compute nodes. Use the [az batch task create](/cli/azure/batch/task#az-batch-task-create) command to create tasks to run in the job. Each task has a command line that specifies an app or script. + +The following Bash script creates four identical, parallel tasks called `myTask1` through `myTask4`. The task command line displays the Batch environment variables on the compute node, and then waits 90 seconds. + +```azurecli-interactive +for i in {1..4} +do + az batch task create \ + --task-id myTask$i \ + --job-id myJob \ + --command-line "/bin/bash -c 'printenv | grep AZ_BATCH; sleep 90s'" +done +``` + +The command output shows the settings for each task. Batch distributes the tasks to the compute nodes. + +## View task status + +After you create the task, Batch queues the task to run on the pool. Once a node is available, the task runs on the node. + +Use the [az batch task show](/cli/azure/batch/task#az-batch-task-show) command to view the status of Batch tasks. The following example shows details about the status of `myTask1`: + +```azurecli-interactive +az batch task show \ + --job-id myJob \ + --task-id myTask1 +``` + +The command output includes many details. For example, an `exitCode` of `0` indicates that the task command completed successfully. The `nodeId` shows the name of the pool node that ran the task. + +## View task output + +Use the [az batch task file list](/cli/azure/batch/task#az-batch-task-file-show) command to list the files a task created on a node. The following command lists the files that `myTask1` created: + +```azurecli-interactive +az batch task file list \ + --job-id myJob \ + --task-id myTask1 \ + --output table +``` + +Results are similar to the following output: + +```output +Name URL Is Directory Content Length +---------- ---------------------------------------------------------------------------------------- -------------- ---------------- +stdout.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stdout.txt False 695 +certs https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/certs True +wd https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/wd True +stderr.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stderr.txt False 0 + +``` + +The [az batch task file download](/cli/azure/batch/task#az-batch-task-file-download) command downloads output files to a local directory. Run the following example to download the *stdout.txt* file: + +```azurecli-interactive +az batch task file download \ + --job-id myJob \ + --task-id myTask1 \ + --file-path stdout.txt \ + --destination ./stdout.txt +``` + +You can view the contents of the standard output file in a text editor. The following example shows a typical *stdout.txt* file. The standard output from this task shows the Azure Batch environment variables that are set on the node. You can refer to these environment variables in your Batch job task command lines, and in the apps and scripts the command lines run. + +```text +AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1 +AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup +AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/certs +AZ_BATCH_ACCOUNT_URL=https://mybatchaccount.eastus2.batch.azure.com/ +AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/wd +AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared +AZ_BATCH_TASK_USER=_azbatch +AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks +AZ_BATCH_JOB_ID=myJob +AZ_BATCH_NODE_IS_DEDICATED=true +AZ_BATCH_NODE_ID=tvm-257509324_2-20180703t215033z +AZ_BATCH_POOL_ID=myPool +AZ_BATCH_TASK_ID=myTask1 +AZ_BATCH_ACCOUNT_NAME=mybatchaccount +AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin +``` + +## Clean up resources + +If you want to continue with Batch tutorials and samples, you can use the Batch account and linked storage account that you created in this quickstart. There's no charge for the Batch account itself. + +Pools and nodes incur charges while the nodes are running, even if they aren't running jobs. When you no longer need a pool, use the [az batch pool delete](/cli/azure/batch/pool#az-batch-pool-delete) command to delete it. Deleting a pool deletes all task output on the nodes, and the nodes themselves. + +```azurecli-interactive +az batch pool delete --pool-id myPool +``` + +When you no longer need any of the resources you created for this quickstart, you can use the [az group delete](/cli/azure/group#az-group-delete) command to delete the resource group and all its resources. To delete the resource group and the storage account, Batch account, node pools, and all related resources, run the following command: + +```azurecli-interactive +az group delete --name qsBatch +``` + +## Next steps + +In this quickstart, you created a Batch account and pool, created and ran a Batch job and tasks, and viewed task output from the nodes. Now that you understand the key concepts of the Batch service, you're ready to use Batch with more realistic, larger scale workloads. To learn more about Azure Batch, continue to the Azure Batch tutorials. + +> [!div class="nextstepaction"] +> [Tutorial: Run a parallel workload with Azure Batch](./tutorial-parallel-python.md) \ No newline at end of file diff --git a/tools/execution_log.csv b/tools/execution_log.csv index e78532cad..c4bb91b1a 100644 --- a/tools/execution_log.csv +++ b/tools/execution_log.csv @@ -115,3 +115,73 @@ StdErr: Error: invalid character 'K' looking for beginning of value StdErr:",216.4925456047058,Failure 2025-01-25 18:47:18,workload_description,new.py,generated_exec_doc.md,0,,1.9009339809417725,Success +2025-02-27 18:23:33,workload_description,create a linux vm and ssh into it,generated_exec_doccc.md,3,"time=2025-02-27T18:07:32-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. +Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. +' +StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. + + The 'ie test' command timed out after 11 minutes. + + time=2025-02-27T18:21:11-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. +Error: command exited with 'exit status 255' and the message 'Pseudo-terminal will not be allocated because stdin is not a terminal. +Host key verification failed. +' +StdErr: Pseudo-terminal will not be allocated because stdin is not a terminal. +Host key verification failed.",995.1571435928345,Success +2025-02-27 18:53:06,workload_description,"a Highly Available Kubernetes Cluster with Azure Kubernetes Service (AKS) integrated with Azure Application Gateway for Ingress, Azure Monitor for observability, and Azure Key Vault for managing secrets",generated_exec_doccc.md,11,"time=2025-02-27T18:38:39-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_CZTwyPesQwkinO2v5C6Qixm2sUMloYXQ' is not valid according to the validation procedure. The tracking id is '349cfbaa-ffeb-4e48-b08e-be4f80fca1f4'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_CZTwyPesQwkinO2v5C6Qixm2sUMloYXQ' is not valid according to the validation procedure. The tracking id is '349cfbaa-ffeb-4e48-b08e-be4f80fca1f4'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:40:04-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_nr8WC0bXZgyDtWQMfRU7c0F5UmwwwyLz' is not valid according to the validation procedure. The tracking id is '99a4798e-fab8-4318-8615-8a97885df765'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_nr8WC0bXZgyDtWQMfRU7c0F5UmwwwyLz' is not valid according to the validation procedure. The tracking id is '99a4798e-fab8-4318-8615-8a97885df765'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:41:42-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_MiLFU4nLcNB8R1oPIJYpWo8pNEbgMrKD' is not valid according to the validation procedure. The tracking id is '1e690576-da85-4ff8-b236-79f5140a5813'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_MiLFU4nLcNB8R1oPIJYpWo8pNEbgMrKD' is not valid according to the validation procedure. The tracking id is '1e690576-da85-4ff8-b236-79f5140a5813'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:43:12-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_LdycjlTkciJd3QVKUV5QEs52g5wjnbNJ' is not valid according to the validation procedure. The tracking id is '2dffea2d-e53e-4124-9389-df04d4d0edb6'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_LdycjlTkciJd3QVKUV5QEs52g5wjnbNJ' is not valid according to the validation procedure. The tracking id is '2dffea2d-e53e-4124-9389-df04d4d0edb6'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:44:01-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_PRCvjzLZPvoXoogOATzdpG9TydLmHtUj' is not valid according to the validation procedure. The tracking id is 'e8a5569f-1f06-4edc-a95c-723fcd90237f'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_PRCvjzLZPvoXoogOATzdpG9TydLmHtUj' is not valid according to the validation procedure. The tracking id is 'e8a5569f-1f06-4edc-a95c-723fcd90237f'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:45:32-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_DxB7osxySJemBqPH0d6CAdK1joj5iBok' is not valid according to the validation procedure. The tracking id is 'd8585e3f-d93e-4c33-b9e6-5618df905395'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_DxB7osxySJemBqPH0d6CAdK1joj5iBok' is not valid according to the validation procedure. The tracking id is 'd8585e3f-d93e-4c33-b9e6-5618df905395'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:47:00-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_ULJARbnDhr3T6VwVPVp8SpT0xE1rrh4p' is not valid according to the validation procedure. The tracking id is '6dbe9a92-40e0-4b80-8707-caa255428cae'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_ULJARbnDhr3T6VwVPVp8SpT0xE1rrh4p' is not valid according to the validation procedure. The tracking id is '6dbe9a92-40e0-4b80-8707-caa255428cae'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:48:00-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_sy5T3Agi8rRCUB5nF3IenXgZMaW6Tnya' is not valid according to the validation procedure. The tracking id is '72183f13-9a40-4d3f-9528-6135b13db9d3'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_sy5T3Agi8rRCUB5nF3IenXgZMaW6Tnya' is not valid according to the validation procedure. The tracking id is '72183f13-9a40-4d3f-9528-6135b13db9d3'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:49:50-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_QIklTpsoXUWqVVuBKKQN1qQjhhl7U3ee' is not valid according to the validation procedure. The tracking id is '1d7f57f3-53a7-4ec9-8157-90cf4bb96df8'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_QIklTpsoXUWqVVuBKKQN1qQjhhl7U3ee' is not valid according to the validation procedure. The tracking id is '1d7f57f3-53a7-4ec9-8157-90cf4bb96df8'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:51:58-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_FfxhYnBWZqJR4Bus0lsvDlUrpWdj1NLQ' is not valid according to the validation procedure. The tracking id is '49e842f4-d245-454a-814a-183f68615efe'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_FfxhYnBWZqJR4Bus0lsvDlUrpWdj1NLQ' is not valid according to the validation procedure. The tracking id is '49e842f4-d245-454a-814a-183f68615efe'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} + + time=2025-02-27T18:53:06-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_OiVkDXNnyLGZHtL9M3XLZKPNnaMATBx6' is not valid according to the validation procedure. The tracking id is '26b03af6-8be7-4272-a5a5-6c59aad9b563'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_OiVkDXNnyLGZHtL9M3XLZKPNnaMATBx6' is not valid according to the validation procedure. The tracking id is '26b03af6-8be7-4272-a5a5-6c59aad9b563'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}}",957.5963819026947,Failure +2025-02-28 00:31:26,file,doc1.md,converted_doc1.md,0,,91.56127834320068,Success diff --git a/tools/generated_exec_doccc.md b/tools/generated_exec_doccc.md new file mode 100644 index 000000000..0c8a95dc5 --- /dev/null +++ b/tools/generated_exec_doccc.md @@ -0,0 +1,272 @@ +--- +title: Highly Available Kubernetes Cluster with AKS, Application Gateway, Monitor, and Key Vault +description: This Exec Doc demonstrates how to deploy a highly available Azure Kubernetes Service (AKS) cluster integrated with Azure Application Gateway for Ingress, Azure Monitor for observability, and Azure Key Vault for managing secrets. +ms.topic: quickstart +ms.date: 10/11/2023 +author: azureuser +ms.author: azurealias +ms.custom: innovation-engine, azurecli, kubernetes, monitoring +--- + +# Highly Available Kubernetes Cluster with AKS, Application Gateway, Monitor, and Key Vault + +This Exec Doc walks you through the deployment of a highly available AKS cluster integrated with an Azure Application Gateway used for Ingress, Azure Monitor for observability, and Azure Key Vault for securely managing secrets. Each section includes code blocks with environment variable declarations and inline explanations that automate the cloud infrastructure deployment and help you learn as you go. + +## Overview of the Deployment + +In this workflow, we perform the following steps: + +1. Create a resource group. +2. Create a dedicated virtual network and subnet for the Application Gateway. +3. Deploy an Azure Application Gateway. +4. Update the Application Gateway routing rule to assign an explicit priority. +5. Create an Azure Key Vault to manage secrets. +6. Retrieve the Application Gateway resource ID for integration. +7. Deploy an AKS cluster with: + - Ingress add-on integration with the Application Gateway. + - Monitoring add-on for Azure Monitor. +8. Enable the Azure Key Vault secrets provider add-on on the AKS cluster. + +For all resources that require unique names, a randomly generated suffix is appended. Code blocks are of type "bash" ensuring that they are executable via Innovation Engine. + +## Step 1: Create a Resource Group + +We start by defining our environment variables and creating a resource group to contain all the resources used in this deployment. + +```bash +export REGION="WestUS2" +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RG_NAME="MyAKSResourceGroup$RANDOM_SUFFIX" +az group create --name $RG_NAME --location $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx", + "location": "westus2", + "managedBy": null, + "name": "MyAKSResourceGroupxxxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Step 2: Create a Virtual Network for the Application Gateway + +Next, we create a virtual network and a dedicated subnet for our Application Gateway. This isolation ensures that the Application Gateway is deployed within its own network segment. + +```bash +export VNET_NAME="MyVnet$RANDOM_SUFFIX" +export SUBNET_NAME="AppGwSubnet" +az network vnet create --resource-group $RG_NAME --name $VNET_NAME --address-prefix 10.0.0.0/16 --subnet-name $SUBNET_NAME --subnet-prefix 10.0.1.0/24 +``` + +Results: + + +```JSON +{ + "newVNet": true, + "subnets": [ + { + "addressPrefix": "10.0.1.0/24", + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/virtualNetworks/MyVnetxxxxxx/subnets/AppGwSubnet", + "name": "AppGwSubnet" + } + ] +} +``` + +## Step 3: Deploy the Azure Application Gateway + +We deploy the Application Gateway using the Standard_V2 SKU for high availability and scalability. The default request routing rule "rule1" is automatically created but without a priority, which must be rectified for newer API versions. + +```bash +export AAGW_NAME="MyAppGateway$RANDOM_SUFFIX" +az network application-gateway create --name $AAGW_NAME --resource-group $RG_NAME --location $REGION --sku Standard_V2 --capacity 2 --vnet-name $VNET_NAME --subnet $SUBNET_NAME --http-settings-port 80 +``` + +Results: + + +```JSON +{ + "applicationGateway": { + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx", + "location": "westus2", + "name": "MyAppGatewayxxxxxx", + "provisioningState": "Succeeded", + "sku": { + "capacity": 2, + "name": "Standard_V2" + }, + "type": "Microsoft.Network/applicationGateways" + } +} +``` + +## Step 4: Update the Application Gateway Routing Rule Priority + +Instead of deleting and recreating the default rule, we update the existing request routing rule "rule1" to assign it an explicit priority. This addresses the error regarding an empty priority field required by API versions starting from 2021-08-01. + +```bash +# Wait until the Application Gateway is fully provisioned. +az network application-gateway wait --name $AAGW_NAME --resource-group $RG_NAME --created + +# Update the default request routing rule (rule1) with an explicit priority. +az network application-gateway rule update --resource-group $RG_NAME --gateway-name $AAGW_NAME --name rule1 --priority 1 +``` + +Results: + + +```JSON +{ + "name": "rule1", + "priority": 1, + "ruleType": "Basic", + "httpListener": { + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx/httpListeners/appGatewayHttpListener" + }, + "backendAddressPool": { + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx/backendAddressPools/BackendAddressPool_1" + }, + "backendHttpSettings": { + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx/backendHttpSettingsCollection/appGatewayBackendHttpSettings" + } +} +``` + +## Step 5: Create an Azure Key Vault + +Create an Azure Key Vault to securely store and manage application secrets and certificates. The Key Vault integration with AKS allows your cluster to securely retrieve secrets when needed. + +```bash +export KEYVAULT_NAME="myKeyVault$RANDOM_SUFFIX" +az keyvault create --name $KEYVAULT_NAME --resource-group $RG_NAME --location $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceVaults/myKeyVaultxxxxxx", + "location": "westus2", + "name": "myKeyVaultxxxxxx", + "properties": { + "sku": { + "family": "A", + "name": "standard" + }, + "tenantId": "xxxxx-xxxxx-xxxxx-xxxxx", + "accessPolicies": [] + }, + "type": "Microsoft.KeyVault/vaults" +} +``` + +## Step 6: Retrieve Application Gateway Resource ID + +Before deploying the AKS cluster, retrieve the Application Gateway resource ID. This ID is required for integrating the Application Gateway Ingress add-on with AKS. + +```bash +export AAGW_ID=$(az network application-gateway show --name $AAGW_NAME --resource-group $RG_NAME --query id -o tsv) +echo $AAGW_ID +``` + +Results: + + +```text +/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx +``` + +## Step 7: Deploy the AKS Cluster with Ingress and Monitoring Add-ons + +Deploy the AKS cluster using three nodes. The cluster is integrated with the Application Gateway Ingress add-on using the Application Gateway resource ID obtained in the previous step. Additionally, the monitoring add-on is enabled for integration with Azure Monitor. + +```bash +export AKS_CLUSTER_NAME="MyAKSCluster$RANDOM_SUFFIX" +az aks create --resource-group $RG_NAME --name $AKS_CLUSTER_NAME --node-count 3 --enable-addons ingress-appgw,monitoring --appgw-id $AAGW_ID --network-plugin azure --location $REGION --generate-ssh-keys +``` + +Results: + + +```JSON +{ + "aadProfile": null, + "addonProfiles": { + "ingressApplicationGateway": { + "config": { + "appgwId": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx" + }, + "enabled": true, + "identity": {} + }, + "omsagent": { + "config": { + "logAnalyticsWorkspaceResourceID": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.OperationalInsights/workspaces/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus" + }, + "enabled": true + } + }, + "dnsPrefix": "myaksclustercxxxxxx", + "enableRBAC": true, + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.ContainerService/managedClusters/MyAKSClusterxxxxxx", + "location": "westus2", + "name": "MyAKSClusterxxxxxx", + "provisioningState": "Succeeded", + "resourceGroup": "MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus", + "type": "Microsoft.ContainerService/managedClusters" +} +``` + +## Step 8: Enable Azure Key Vault Secrets Provider Add-on on AKS + +Integrate the AKS cluster with Azure Key Vault by enabling the Azure Key Vault secrets provider add-on. This add-on securely mounts secrets stored in Azure Key Vault as volumes within your pods. + +```bash +az aks enable-addons --addons azure-keyvault-secrets-provider --name $AKS_CLUSTER_NAME --resource-group $RG_NAME +``` + +Results: + + +```JSON +{ + "addonProfiles": { + "azureKeyvaultSecretsProvider": { + "config": {}, + "enabled": true + }, + "ingressApplicationGateway": { + "config": { + "appgwId": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx" + }, + "enabled": true + }, + "omsagent": { + "config": { + "logAnalyticsWorkspaceResourceID": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.OperationalInsights/workspaces/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus" + }, + "enabled": true + } + }, + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.ContainerService/managedClusters/MyAKSClusterxxxxxx", + "name": "MyAKSClusterxxxxxx" +} +``` + +## Summary + +In this Exec Doc, you deployed a highly available AKS cluster integrated with an Application Gateway used for Ingress, Azure Monitor for observability, and Azure Key Vault for secure secret management. A dedicated virtual network was created for the Application Gateway, and after the gateway was provisioned, the default Application Gateway routing rule was updated to include a defined priority—thereby addressing the API validation requirement. With clearly defined environment variables and inline explanations, you can now deploy this production-grade infrastructure using the Innovation Engine without encountering deployment errors. + +Feel free to execute these commands step-by-step in your preferred Azure CLI environment. \ No newline at end of file diff --git a/tools/stdout.txt b/tools/stdout.txt deleted file mode 100644 index 01537152b..000000000 --- a/tools/stdout.txt +++ /dev/null @@ -1,20 +0,0 @@ -AZ_BATCH_NODE_MOUNTS_DIR=/mnt/batch/tasks/fsmounts -AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/wd -AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1 -AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared -AZ_BATCH_TASK_USER=_azbatch -AZ_BATCH_NODE_IS_DEDICATED=true -AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup -AZ_BATCH_JOB_ID=myJob -AZ_BATCH_NODE_STARTUP_WORKING_DIR=/mnt/batch/tasks/startup/wd -AZ_BATCH_TASK_ID=myTask1 -AZ_BATCH_ACCOUNT_NAME=batchaccountd980a9 -AZ_BATCH_RESERVED_EPHEMERAL_DISK_SPACE_BYTES=1000000000 -AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks -AZ_BATCH_POOL_ID=myPool -AZ_BATCH_RESERVED_DISK_SPACE_BYTES=1000000000 -AZ_BATCH_ACCOUNT_URL=https://batchaccountd980a9.eastus2.batch.azure.com/ -AZ_BATCH_NODE_ID=tvmps_38766d42b76cb3aeb30719a252fa0782d11ba04294b3f4c339ccb3f08dbdb2a4_d -AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin -AZ_BATCH_OS_RESERVED_EPHEMERAL_DISK_SPACE_BYTES=1000000000 -AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/certs From 8f325c24f08440607503d2c2af6ac2e1437c43b7 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Sun, 2 Mar 2025 18:10:17 -0800 Subject: [PATCH 04/87] added 3rd support doc --- .../troubleshoot-vm-grub-error-repairvm.md | 104 ++++++++ ...-vm-grub-error-repairvm.md:Zone.Identifier | 4 + scenarios/metadata.json | 26 ++ tools/ada.py | 2 +- tools/converted_doc.md | 252 +++++++----------- 5 files changed, 226 insertions(+), 162 deletions(-) create mode 100644 scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md create mode 100644 scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier diff --git a/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md b/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md new file mode 100644 index 000000000..48750ef8e --- /dev/null +++ b/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md @@ -0,0 +1,104 @@ +--- +title: Linux VM boots to GRUB rescue +description: Provides troubleshooting guidance for GRUB rescue issues with Linux virtual machines. +services: virtual-machines +documentationcenter: '' +author: divargas +ms.service: azure-virtual-machines +ms.collection: linux +ms.workload: infrastructure-services +ms.tgt_pltfrm: vm-linux +ms.custom: sap:My VM is not booting, linux-related-content +ms.topic: troubleshooting +ms.date: 02/25/2025 +ms.author: divargas +ms.reviewer: ekpathak, v-leedennis, v-weizhu +--- + +# Linux virtual machine boots to GRUB rescue + +**Applies to:** :heavy_check_mark: Linux VMs + + + + + +This article discusses multiple conditions that cause GRUB rescue issues and provides troubleshooting guidance. + +During the boot process, the boot loader tries to locate the Linux kernel and hand off the boot control. If this handoff can't be performed, the virtual machine (VM) enters a GRUB rescue console. The GRUB rescue console prompt isn't shown in the Azure serial console log, but it can be shown in the [Azure boot diagnostics screenshot](/azure/virtual-machines/boot-diagnostics#boot-diagnostics-view). + +## Identify GRUB rescue issue + +[View a boot diagnostics screenshot](/azure/virtual-machines/boot-diagnostics#boot-diagnostics-view) in the VM **Boot diagnostics** page of the Azure portal. This screenshot helps diagnose the GRUB rescue issue and determine if a boot error causes the issue. + +The following text is an example of a GRUB rescue issue: + +```output +error: file '/boot/grub2/i386-pc/normal.mod' not found. +Entering rescue mode... +grub rescue> +``` + +## Troubleshoot GRUB rescue issue offline + +1. To troubleshoot a GRUB rescue issue, a rescue/repair VM is required. Use [vm repair commands](repair-linux-vm-using-azure-virtual-machine-repair-commands.md) to create a repair VM that has a copy of the affected VM's OS disk attached. Mount the copy of the OS file systems in the repair VM by using [chroot](chroot-environment-linux.md). + + > [!NOTE] + > Alternatively, you can create a rescue VM manually by using the Azure portal. For more information, see [Troubleshoot a Linux VM by attaching the OS disk to a recovery VM using the Azure portal](troubleshoot-recovery-disks-portal-linux.md). + +2. [Identify GRUB rescue issue](#identify-grub-rescue-issue). When you encounter one of the following GRUB rescue issues, go to the corresponding section to resolve it: + + * [Error: unknown filesystem](#unknown-filesystem) + * [Error 15: File not found](#error15) + * [Error: file '/boot/grub2/i386-pc/normal.mod' not found](#normal-mod-file-not-found) + * [Error: no such partition](#no-such-partition) + * [Error: symbol 'grub_efi_get_secure_boot' not found](#grub_efi_get_secure_boot) + * [Other GRUB rescue errors](#other-grub-rescue-errors) + +3. After the GRUB rescue issue is resolved, perform the following actions: + + 1. Unmount the copy of the file systems from the rescue/repair VM. + + 2. Run the `az vm repair restore` command to swap the repaired OS disk with the original OS disk of the VM. For more information, see Step 5 in [Repair a Linux VM by using the Azure Virtual Machine repair commands](repair-linux-vm-using-azure-virtual-machine-repair-commands.md). + + 3. Check whether the VM can start by taking a look at the Azure serial console or by trying to connect to the VM. + +4. If the entire /boot partition or other important contents are missing and can't be recovered, we recommend restoring the VM from a backup. For more information, see [How to restore Azure VM data in Azure portal](/azure/backup/backup-azure-arm-restore-vms). + +See the following sections for detailed errors, possible causes, and solutions. + +> [!NOTE] +> In the commands mentioned in the following sections, replace `/dev/sdX` with the corresponding Operating System (OS) disk device. + +### Reinstall GRUB and regenerate GRUB configuration file using Auto Repair (ALAR) + +Azure Linux Auto Repair (ALAR) scripts are part of the VM repair extension described in [Use Azure Linux Auto Repair (ALAR) to fix a Linux VM](./repair-linux-vm-using-alar.md). ALAR covers the automation of multiple repair scenarios, including GRUB rescue issues. + +The ALAR scripts use the repair extension `repair-button` to fix GRUB issues by specifying `--button-command grubfix` for Generation 1 VMs, or `--button-command efifix` for Generation 2 VMs. This parameter triggers the automated recovery. Implement the following step to automate the fix of common GRUB errors that could be fixed by reinstalling GRUB and regenerating the corresponding configuration file: + +```azurecli-interactive +GEN=$(az vm get-instance-view --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --query "instanceView.hyperVGeneration" --output tsv) +if [[ "$GEN" =~ "[Vv]?2" ]]; then ALAR="efifix"; else ALAR="grubfix"; fi +output=$(az extension add -n vm-repair; az extension update -n vm-repair; az vm repair repair-button --button-command $ALAR --verbose --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME) +value=$(echo "$output" | jq -r '.message') +extracted=$(echo $value) +echo "$extracted" +``` + +The repair VM script, in conjunction with the ALAR script, temporarily creates a resource group, a repair VM, and a copy of the affected VM's OS disk. It reinstalls GRUB and regenerates the corresponding GRUB configuration file and then it swaps the OS disk of the broken VM with the copied fixed disk. Finally, the `repair-button` script will automatically delete the resource group containing the temporary repair VM. + +## Next steps + +If the specific boot error isn't a GRUB rescue issue, refer to [Troubleshoot Azure Linux Virtual Machines boot errors](boot-error-troubleshoot-linux.md) for further troubleshooting options. + +[!INCLUDE [Third-party disclaimer](../../../includes/third-party-disclaimer.md)] + +[!INCLUDE [Third-party contact disclaimer](../../../includes/third-party-contact-disclaimer.md)] \ No newline at end of file diff --git a/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier b/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier new file mode 100644 index 000000000..76deaeb95 --- /dev/null +++ b/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier @@ -0,0 +1,4 @@ +[ZoneTransfer] +ZoneId=3 +ReferrerUrl=https://teams.microsoft.com/ +HostUrl=https://microsoft-my.sharepoint.com/personal/divargas_microsoft_com/_layouts/15/download.aspx?UniqueId=565b75b9-0a61-4cdc-9c14-a3416f9b3301&Translate=false&ApiVersion=2.0 diff --git a/scenarios/metadata.json b/scenarios/metadata.json index db5430982..efc983f17 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -971,5 +971,31 @@ } ] } + }, + { + "status": "active", + "key": "TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md", + "title": "Linux VM boots to GRUB rescue", + "description": "Provides troubleshooting guidance for GRUB rescue issues with Linux virtual machines.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md", + "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/troubleshoot-vm-boot-error", + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "MY_RESOURCE_GROUP_NAME", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "MY_VM_NAME", + "title": "VM Name", + "defaultValue": "" + } + ] + } } ] diff --git a/tools/ada.py b/tools/ada.py index 9c87b2e9e..03f2477a4 100644 --- a/tools/ada.py +++ b/tools/ada.py @@ -455,7 +455,7 @@ def main(): additional_instruction = "" print(f"\nError: {error_log.strip()}") - print(f"\nStrategy: {additional_instruction}") + print(f"\n{'!'*40}\nApplying an error troubleshooting strategy...\n{'!'*40}") attempt += 1 success = False diff --git a/tools/converted_doc.md b/tools/converted_doc.md index 5aa6c8e47..d4aaeaaa9 100644 --- a/tools/converted_doc.md +++ b/tools/converted_doc.md @@ -3,16 +3,16 @@ title: 'Quickstart: Use the Azure CLI to create a Batch account and run a job' description: Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool. ms.topic: quickstart ms.date: 04/12/2023 -author: azurecli -ms.author: azurecli ms.custom: mvc, devx-track-azurecli, mode-api, linux-related-content, innovation-engine +author: (preserved) +ms.author: (preserved) --- # Quickstart: Use the Azure CLI to create a Batch account and run a job -This quickstart shows you how to get started with Azure Batch by using Azure CLI commands and scripts to create and manage Batch resources. You create a Batch account that has a pool of virtual machines (compute nodes). You then create and run a job with tasks that run on the pool nodes. +This quickstart shows you how to get started with Azure Batch by using Azure CLI commands and scripts to create and manage Batch resources. You create a Batch account that has a pool of virtual machines, or compute nodes. You then create and run a job with tasks that run on the pool nodes. -After you complete this quickstart, you will understand the [key concepts of the Batch service](batch-service-workflow-features.md) and be ready to use Batch with more realistic, larger scale workloads. +After you complete this quickstart, you understand the [key concepts of the Batch service](batch-service-workflow-features.md) and are ready to use Batch with more realistic, larger scale workloads. ## Prerequisites @@ -20,115 +20,95 @@ After you complete this quickstart, you will understand the [key concepts of the - Azure Cloud Shell or Azure CLI. - You can run the Azure CLI commands in this quickstart interactively in Azure Cloud Shell. To run the commands in Cloud Shell, select **Open Cloudshell** at the upper-right corner of a code block. Select **Copy** to copy the code, and paste it into Cloud Shell to run it. You can also [run Cloud Shell from within the Azure portal](https://shell.azure.com). Cloud Shell always uses the latest version of the Azure CLI. + You can run the Azure CLI commands in this quickstart interactively in Azure Cloud Shell. To run the commands in the Cloud Shell, select **Open Cloudshell** at the upper-right corner of a code block. Select **Copy** to copy the code, and paste it into Cloud Shell to run it. You can also [run Cloud Shell from within the Azure portal](https://shell.azure.com). Cloud Shell always uses the latest version of the Azure CLI. - Alternatively, you can [install Azure CLI locally](/cli/azure/install-azure-cli) to run the commands. The steps in this article require Azure CLI version 2.0.20 or later. Run [az version](/cli/azure/reference-index?#az-version) to see your installed version and dependent libraries, and run [az upgrade](/cli/azure/reference-index?#az-upgrade) to upgrade. If you use a local installation, ensure you are already signed in to Azure. + Alternatively, you can [install Azure CLI locally](/cli/azure/install-azure-cli) to run the commands. The steps in this article require Azure CLI version 2.0.20 or later. Run [az version](/cli/azure/reference-index?#az-version) to see your installed version and dependent libraries, and run [az upgrade](/cli/azure/reference-index?#az-upgrade) to upgrade. If you use a local installation, sign in to Azure by using the appropriate command. >[!NOTE] >For some regions and subscription types, quota restrictions might cause Batch account or node creation to fail or not complete. In this situation, you can request a quota increase at no charge. For more information, see [Batch service quotas and limits](batch-quota-limit.md). -## Create a resource group +## Setup Environment Variables -In this section, we create an Azure resource group that will serve as a logical container for all the resources used in this quickstart. To ensure uniqueness, a random suffix is appended to the resource group name. We use the location "centralindia" consistently across all resources. +Below, we declare environment variables that will be used throughout this Exec Doc. We include a random suffix to uniquely name resources and avoid collisions on repeated executions. ```bash export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="eastus2" export RESOURCE_GROUP="qsBatch$RANDOM_SUFFIX" -export LOCATION="centralindia" -az group create --name $RESOURCE_GROUP --location $LOCATION +export STORAGE_ACCOUNT="mybatchstorage$RANDOM_SUFFIX" +export BATCH_ACCOUNT="mybatchaccount$RANDOM_SUFFIX" +export POOL_ID="myPool$RANDOM_SUFFIX" +export JOB_ID="myJob$RANDOM_SUFFIX" +``` + +## Create a resource group + +Run the following [az group create](/cli/azure/group#az-group-create) command to create an Azure resource group. The resource group is a logical container that holds the Azure resources for this quickstart. + +```azurecli-interactive +az group create \ + --name $RESOURCE_GROUP \ + --location $REGION ``` +Results: + + ```JSON { - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/qsBatchxxx", - "location": "centralindia", - "managedBy": null, - "name": "qsBatchxxx", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" + "id": "/subscriptions/xxxxx/resourceGroups/qsBatchxxx", + "location": "eastus2", + "managedBy": null, + "name": "qsBatchxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" } ``` ## Create a storage account -Next, create an Azure Storage account to be linked with your Batch account. Although this quickstart doesn't directly use the storage account, real-world Batch workloads typically link a storage account to deploy applications and manage data. +Use the [az storage account create](/cli/azure/storage/account#az-storage-account-create) command to create an Azure Storage account to link to your Batch account. Although this quickstart doesn't use the storage account, most real-world Batch workloads use a linked storage account to deploy applications and store input and output data. -```bash -export STORAGE_ACCOUNT="mybatchstorage$RANDOM_SUFFIX" +Run the following command to create a Standard_LRS SKU storage account in your resource group: + +```azurecli-interactive az storage account create \ --resource-group $RESOURCE_GROUP \ --name $STORAGE_ACCOUNT \ - --location $LOCATION \ + --location $REGION \ --sku Standard_LRS ``` - -```JSON -{ - "sku": { - "name": "Standard_LRS" - }, - "kind": "Storage", - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/qsBatchxxx/providers/Microsoft.Storage/storageAccounts/mybatchstoragexxx", - "location": "centralindia", - "name": "mybatchstoragexxx", - "type": "Microsoft.Storage/storageAccounts", - "statusOfPrimary": "available" -} -``` - ## Create a Batch account -Create a Batch account in your resource group and link it with the storage account created earlier. Note that we are using the "centralindia" location to ensure consistency across resources. +Run the following [az batch account create](/cli/azure/batch/account#az-batch-account-create) command to create a Batch account in your resource group and link it with the storage account. -```bash -export BATCH_ACCOUNT="mybatchaccount$RANDOM_SUFFIX" +```azurecli-interactive az batch account create \ --name $BATCH_ACCOUNT \ --storage-account $STORAGE_ACCOUNT \ --resource-group $RESOURCE_GROUP \ - --location $LOCATION -``` - - -```JSON -{ - "accountEndpoint": "mybatchaccountxxx.centralindia.batch.azure.com", - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/qsBatchxxx/providers/Microsoft.Batch/batchAccounts/mybatchaccountxxx", - "location": "centralindia", - "name": "mybatchaccountxxx", - "resourceGroup": "qsBatchxxx", - "type": "Microsoft.Batch/batchAccounts" -} + --location $REGION ``` -Before proceeding with further Batch operations, sign in to your Batch account so that subsequent commands use the correct account context. A brief delay is introduced to ensure the account has propagated fully. +Sign in to the new Batch account by running the [az batch account login](/cli/azure/batch/account#az-batch-account-login) command. Once you authenticate your account with Batch, subsequent `az batch` commands in this session use this account context. -```bash +```azurecli-interactive az batch account login \ --name $BATCH_ACCOUNT \ --resource-group $RESOURCE_GROUP \ --shared-key-auth -sleep 30 -``` - - -```JSON -{ - "message": "Login to Batch account mybatchaccountxxx in resource group qsBatchxxx was successful." -} ``` ## Create a pool of compute nodes -Now, create a pool of Linux compute nodes within your Batch account. In this example, we create a pool with two Standard_A1_v2 VMs running Ubuntu 20.04 LTS. This configuration provides a balance between performance and cost for this quickstart. +Run the [az batch pool create](/cli/azure/batch/pool#az-batch-pool-create) command to create a pool of Linux compute nodes in your Batch account. The following example creates a pool that consists of two Standard_A1_v2 size VMs running Ubuntu 20.04 LTS OS. This node size offers a good balance of performance versus cost for this quickstart example. -```bash -export POOL_ID="myPool$RANDOM_SUFFIX" +```azurecli-interactive az batch pool create \ --id $POOL_ID \ --image canonical:0001-com-ubuntu-server-focal:20_04-lts \ @@ -137,56 +117,42 @@ az batch pool create \ --vm-size Standard_A1_v2 ``` - -```JSON -{ - "id": "myPoolxxx", - "allocationState": "resizing", - "vmSize": "Standard_A1_v2", - "targetDedicatedNodes": 2, - "provisioningState": "InProgress" -} -``` +Batch creates the pool immediately, but takes a few minutes to allocate and start the compute nodes. To see the pool status, use the [az batch pool show](/cli/azure/batch/pool#az-batch-pool-show) command. This command shows all the properties of the pool, and you can query for specific properties. The following command queries for the pool allocation state: -Batch immediately begins creating the pool, although it may take a few minutes to allocate and start the compute nodes. To check the pool allocation state reliably and avoid JSON parsing errors, query only the allocationState property: - -```bash -az batch pool show --pool-id $POOL_ID --query "allocationState" --output json +```azurecli-interactive +az batch pool show --pool-id $POOL_ID \ + --query "allocationState" ``` +Results: + + ```JSON -"resizing" +{ + "allocationState": "resizing" +} ``` +While Batch allocates and starts the nodes, the pool is in the `resizing` state. You can create a job and tasks while the pool state is still `resizing`. The pool is ready to run tasks when the allocation state is `steady` and all the nodes are running. + ## Create a job -Create a Batch job that will run on the pool. A job logically groups one or more tasks and specifies common settings such as the target pool. +Use the [az batch job create](/cli/azure/batch/job#az-batch-job-create) command to create a Batch job to run on your pool. A Batch job is a logical group of one or more tasks. The job includes settings common to the tasks, such as the pool to run on. The following example creates a job that initially has no tasks. -```bash -export JOB_ID="myJob$RANDOM_SUFFIX" +```azurecli-interactive az batch job create \ --id $JOB_ID \ --pool-id $POOL_ID ``` - -```JSON -{ - "id": "myJobxxx", - "poolInfo": { - "poolId": "myPoolxxx" - }, - "priority": 0, - "onAllTasksComplete": "noAction" -} -``` - ## Create job tasks -Batch provides several methods to deploy applications and scripts to compute nodes. In the following loop, four parallel tasks (named myTask1 through myTask4) are created. Each task runs a command that prints Batch environment variables on the compute node and then waits for 90 seconds. +Batch provides several ways to deploy apps and scripts to compute nodes. Use the [az batch task create](/cli/azure/batch/task#az-batch-task-create) command to create tasks to run in the job. Each task has a command line that specifies an app or script. -```bash +The following Bash script creates four identical, parallel tasks called `myTask1` through `myTask4`. The task command line displays the Batch environment variables on the compute node, and then waits 90 seconds. + +```azurecli-interactive for i in {1..4} do az batch task create \ @@ -196,79 +162,47 @@ do done ``` -Each task's output will display the environment settings specific to the node where it is executed. +Batch distributes the tasks to the compute nodes. ## View task status -After creating the tasks, they are queued for execution. When a compute node becomes available, the task will run. Use the following command to view the status of a specific task (for example, myTask1): +After you create the tasks, Batch queues them to run on the pool. Once a node is available, a task runs on the node. -```bash +Use the [az batch task show](/cli/azure/batch/task#az-batch-task-show) command to view the status of Batch tasks. The following example shows details about the status of `myTask1`: + +```azurecli-interactive az batch task show \ --job-id $JOB_ID \ --task-id myTask1 ``` - -```JSON -{ - "id": "myTask1", - "state": "active", - "executionInfo": { - "startTime": "2023-xx-xxTxx:xx:xxZ", - "endTime": null, - "retryCount": 0, - "exitCode": null - }, - "nodeInfo": { - "nodeId": "tvm-xxxxxxxx" - } -} -``` - -An exitCode of 0 in the output indicates that the task completed successfully. The nodeId property indicates the compute node where the task ran. +The command output includes many details. For example, an `exitCode` of `0` indicates that the task command completed successfully. The `nodeId` shows the name of the pool node that ran the task. ## View task output -To display the file output generated by a task on a compute node, list the files produced by the task. In the following example, the files generated by myTask1 are listed: +Use the [az batch task file list](/cli/azure/batch/task#az-batch-task-file-show) command to list the files a task created on a node. The following command lists the files that `myTask1` created: -```bash +```azurecli-interactive az batch task file list \ --job-id $JOB_ID \ --task-id myTask1 \ --output table ``` - -```JSON -[ - { - "Name": "stdout.txt", - "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/stdout.txt", - "Is Directory": false, - "Content Length": 695 - }, - { - "Name": "certs", - "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/certs", - "Is Directory": true - }, - { - "Name": "wd", - "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/wd", - "Is Directory": true - }, - { - "Name": "stderr.txt", - "URL": "https://mybatchaccountxxx.centralindia.batch.azure.com/jobs/myJobxxx/tasks/myTask1/files/stderr.txt", - "Is Directory": false, - "Content Length": 0 - } -] +Results are similar to the following output: + +```output +Name URL Is Directory Content Length +---------- ---------------------------------------------------------------------------------------- -------------- ---------------- +stdout.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stdout.txt False 695 +certs https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/certs True +wd https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/wd True +stderr.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stderr.txt False 0 ``` -To download the standard output file (stdout.txt) to your local directory, run the following command: +The [az batch task file download](/cli/azure/batch/task#az-batch-task-file-download) command downloads output files to a local directory. Run the following example to download the *stdout.txt* file: -```bash +```azurecli-interactive az batch task file download \ --job-id $JOB_ID \ --task-id myTask1 \ @@ -276,33 +210,29 @@ az batch task file download \ --destination ./stdout.txt ``` -You can then open the downloaded stdout.txt in a text editor. Typically, the file contains the Batch environment variables set on the compute node, such as: +You can view the contents of the standard output file in a text editor. The following example shows a typical *stdout.txt* file. The standard output from this task shows the Azure Batch environment variables that are set on the node. You can refer to these environment variables in your Batch job task command lines, and in the apps and scripts the command lines run. ```text AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1 AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/certs -AZ_BATCH_ACCOUNT_URL=https://mybatchaccountxxx.centralindia.batch.azure.com/ +AZ_BATCH_ACCOUNT_URL=https://mybatchaccount.eastus2.batch.azure.com/ AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/wd AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared AZ_BATCH_TASK_USER=_azbatch AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks -AZ_BATCH_JOB_ID=myJobxxx +AZ_BATCH_JOB_ID=myJob AZ_BATCH_NODE_IS_DEDICATED=true -AZ_BATCH_NODE_ID=tvm-xxxxxxxx_2-20180703t215033z -AZ_BATCH_POOL_ID=myPoolxxx +AZ_BATCH_NODE_ID=tvm-257509324_2-20180703t215033z +AZ_BATCH_POOL_ID=myPool AZ_BATCH_TASK_ID=myTask1 -AZ_BATCH_ACCOUNT_NAME=mybatchaccountxxx +AZ_BATCH_ACCOUNT_NAME=mybatchaccount AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin ``` -## Clean up resources - -If you want to continue with Batch tutorials and samples, you can keep the Batch account and linked storage account that you created in this quickstart. There is no charge for the Batch account itself. Pools and nodes do incur charges while running, even if no jobs are active. To avoid accidental deletions during automated execution, deletion commands have been removed from this document. When you no longer need these resources, please delete the resource group and its related resources manually. - ## Next steps -In this quickstart, you created a Batch account and a compute pool, created and ran a Batch job with tasks, and viewed task outputs generated on the compute nodes. Now that you understand the key concepts of the Batch service, you're ready to use Batch for more realistic, larger scale workloads. To dive deeper into Azure Batch, continue with the Batch tutorials. +In this quickstart, you created a Batch account and pool, created and ran a Batch job and tasks, and viewed task output from the nodes. Now that you understand the key concepts of the Batch service, you're ready to use Batch with more realistic, larger scale workloads. To learn more about Azure Batch, continue to the Azure Batch tutorials. > [!div class="nextstepaction"] > [Tutorial: Run a parallel workload with Azure Batch](./tutorial-parallel-python.md) \ No newline at end of file From 5c0eebf33ed7fd38ef85f056eb07b2201d6bff02 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Sun, 2 Mar 2025 18:16:49 -0800 Subject: [PATCH 05/87] added 3rd support doc --- .../troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier diff --git a/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier b/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier deleted file mode 100644 index 76deaeb95..000000000 --- a/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md:Zone.Identifier +++ /dev/null @@ -1,4 +0,0 @@ -[ZoneTransfer] -ZoneId=3 -ReferrerUrl=https://teams.microsoft.com/ -HostUrl=https://microsoft-my.sharepoint.com/personal/divargas_microsoft_com/_layouts/15/download.aspx?UniqueId=565b75b9-0a61-4cdc-9c14-a3416f9b3301&Translate=false&ApiVersion=2.0 From 6af4d227c32fd1bed0b911d767d878443ac3112a Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 3 Mar 2025 19:36:03 -0800 Subject: [PATCH 06/87] added new doc using AI agent! --- .../articles/batch/quick-create-cli.md | 247 ++++++++++++++++++ scenarios/metadata.json | 18 ++ tools/converted_doc.md | 47 ++-- tools/stdout.txt | 20 ++ 4 files changed, 313 insertions(+), 19 deletions(-) create mode 100644 scenarios/azure-docs/articles/batch/quick-create-cli.md create mode 100644 tools/stdout.txt diff --git a/scenarios/azure-docs/articles/batch/quick-create-cli.md b/scenarios/azure-docs/articles/batch/quick-create-cli.md new file mode 100644 index 000000000..ea6b37499 --- /dev/null +++ b/scenarios/azure-docs/articles/batch/quick-create-cli.md @@ -0,0 +1,247 @@ +--- +title: 'Quickstart: Use the Azure CLI to create a Batch account and run a job' +description: Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool. +ms.topic: quickstart +ms.date: 04/12/2023 +ms.custom: mvc, devx-track-azurecli, mode-api, linux-related-content, innovation-engine +author: (preserved) +ms.author: (preserved) +--- + +# Quickstart: Use the Azure CLI to create a Batch account and run a job + +This quickstart shows you how to get started with Azure Batch by using Azure CLI commands and scripts to create and manage Batch resources. You create a Batch account that has a pool of virtual machines, or compute nodes. You then create and run a job with tasks that run on the pool nodes. + +After you complete this quickstart, you understand the [key concepts of the Batch service](batch-service-workflow-features.md) and are ready to use Batch with more realistic, larger scale workloads. + +## Prerequisites + +- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] + +- Azure Cloud Shell or Azure CLI. + + You can run the Azure CLI commands in this quickstart interactively in Azure Cloud Shell. To run the commands in the Cloud Shell, select **Open Cloudshell** at the upper-right corner of a code block. Select **Copy** to copy the code, and paste it into Cloud Shell to run it. You can also [run Cloud Shell from within the Azure portal](https://shell.azure.com). Cloud Shell always uses the latest version of the Azure CLI. + + Alternatively, you can [install Azure CLI locally](/cli/azure/install-azure-cli) to run the commands. The steps in this article require Azure CLI version 2.0.20 or later. Run [az version](/cli/azure/reference-index?#az-version) to see your installed version and dependent libraries, and run [az upgrade](/cli/azure/reference-index?#az-upgrade) to upgrade. If you use a local installation, sign in to Azure by using the appropriate command. + +>[!NOTE] +>For some regions and subscription types, quota restrictions might cause Batch account or node creation to fail or not complete. In this situation, you can request a quota increase at no charge. For more information, see [Batch service quotas and limits](batch-quota-limit.md). + +## Create a resource group + +Run the following [az group create](/cli/azure/group#az-group-create) command to create an Azure resource group. The resource group is a logical container that holds the Azure resources for this quickstart. + +```azurecli-interactive +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="canadacentral" +export RESOURCE_GROUP="qsBatch$RANDOM_SUFFIX" + +az group create \ + --name $RESOURCE_GROUP \ + --location $REGION +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/qsBatchxxx", + "location": "eastus2", + "managedBy": null, + "name": "qsBatchxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create a storage account + +Use the [az storage account create](/cli/azure/storage/account#az-storage-account-create) command to create an Azure Storage account to link to your Batch account. Although this quickstart doesn't use the storage account, most real-world Batch workloads use a linked storage account to deploy applications and store input and output data. + +Run the following command to create a Standard_LRS SKU storage account in your resource group: + +```azurecli-interactive +export STORAGE_ACCOUNT="mybatchstorage$RANDOM_SUFFIX" + +az storage account create \ + --resource-group $RESOURCE_GROUP \ + --name $STORAGE_ACCOUNT \ + --location $REGION \ + --sku Standard_LRS +``` + +## Create a Batch account + +Run the following [az batch account create](/cli/azure/batch/account#az-batch-account-create) command to create a Batch account in your resource group and link it with the storage account. + +```azurecli-interactive +export BATCH_ACCOUNT="mybatchaccount$RANDOM_SUFFIX" + +az batch account create \ + --name $BATCH_ACCOUNT \ + --storage-account $STORAGE_ACCOUNT \ + --resource-group $RESOURCE_GROUP \ + --location $REGION +``` + +Sign in to the new Batch account by running the [az batch account login](/cli/azure/batch/account#az-batch-account-login) command. Once you authenticate your account with Batch, subsequent `az batch` commands in this session use this account context. + +```azurecli-interactive +az batch account login \ + --name $BATCH_ACCOUNT \ + --resource-group $RESOURCE_GROUP \ + --shared-key-auth +``` + +## Create a pool of compute nodes + +Run the [az batch pool create](/cli/azure/batch/pool#az-batch-pool-create) command to create a pool of Linux compute nodes in your Batch account. The following example creates a pool that consists of two Standard_A1_v2 size VMs running Ubuntu 20.04 LTS OS. This node size offers a good balance of performance versus cost for this quickstart example. + +```azurecli-interactive +export POOL_ID="myPool$RANDOM_SUFFIX" + +az batch pool create \ + --id $POOL_ID \ + --image canonical:0001-com-ubuntu-server-focal:20_04-lts \ + --node-agent-sku-id "batch.node.ubuntu 20.04" \ + --target-dedicated-nodes 2 \ + --vm-size Standard_A1_v2 +``` + +Batch creates the pool immediately, but takes a few minutes to allocate and start the compute nodes. To see the pool status, use the [az batch pool show](/cli/azure/batch/pool#az-batch-pool-show) command. This command shows all the properties of the pool, and you can query for specific properties. The following command queries for the pool allocation state: + +```azurecli-interactive +az batch pool show --pool-id $POOL_ID \ + --query "{allocationState: allocationState}" +``` + +Results: + + + +```JSON +{ + "allocationState": "resizing" +} +``` + +While Batch allocates and starts the nodes, the pool is in the `resizing` state. You can create a job and tasks while the pool state is still `resizing`. The pool is ready to run tasks when the allocation state is `steady` and all the nodes are running. + +## Create a job + +Use the [az batch job create](/cli/azure/batch/job#az-batch-job-create) command to create a Batch job to run on your pool. A Batch job is a logical group of one or more tasks. The job includes settings common to the tasks, such as the pool to run on. The following example creates a job that initially has no tasks. + +```azurecli-interactive +export JOB_ID="myJob$RANDOM_SUFFIX" + +az batch job create \ + --id $JOB_ID \ + --pool-id $POOL_ID +``` + +## Create job tasks + +Batch provides several ways to deploy apps and scripts to compute nodes. Use the [az batch task create](/cli/azure/batch/task#az-batch-task-create) command to create tasks to run in the job. Each task has a command line that specifies an app or script. + +The following Bash script creates four identical, parallel tasks called `myTask1` through `myTask4`. The task command line displays the Batch environment variables on the compute node, and then waits 90 seconds. + +```azurecli-interactive +for i in {1..4} +do + az batch task create \ + --task-id myTask$i \ + --job-id $JOB_ID \ + --command-line "/bin/bash -c 'printenv | grep AZ_BATCH; sleep 90s'" +done +``` + +Batch distributes the tasks to the compute nodes. + +## View task status + +After you create the tasks, Batch queues them to run on the pool. Once a node is available, a task runs on the node. + +Use the [az batch task show](/cli/azure/batch/task#az-batch-task-show) command to view the status of Batch tasks. The following example shows details about the status of `myTask1`: + +```azurecli-interactive +az batch task show \ + --job-id $JOB_ID \ + --task-id myTask1 +``` + +The command output includes many details. For example, an `exitCode` of `0` indicates that the task command completed successfully. The `nodeId` shows the name of the pool node that ran the task. + +## View task output + +Use the [az batch task file list](/cli/azure/batch/task#az-batch-task-file-show) command to list the files a task created on a node. The following command lists the files that `myTask1` created: + +```azurecli-interactive +# Wait for task to complete before downloading output +echo "Waiting for task to complete..." +while true; do + STATUS=$(az batch task show --job-id $JOB_ID --task-id myTask1 --query "state" -o tsv) + if [ "$STATUS" == "running" ]; then + break + fi + sleep 10 +done + +az batch task file list --job-id $JOB_ID --task-id myTask1 --output table +``` + +Results are similar to the following output: + +Results: + + + +```output +Name URL Is Directory Content Length +---------- ---------------------------------------------------------------------------------------- -------------- ---------------- +stdout.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stdout.txt False 695 +certs https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/certs True +wd https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/wd True +stderr.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stderr.txt False 0 +``` + +The [az batch task file download](/cli/azure/batch/task#az-batch-task-file-download) command downloads output files to a local directory. Run the following example to download the *stdout.txt* file: + +```azurecli-interactive +az batch task file download \ + --job-id $JOB_ID \ + --task-id myTask1 \ + --file-path stdout.txt \ + --destination ./stdout.txt +``` + +You can view the contents of the standard output file in a text editor. The following example shows a typical *stdout.txt* file. The standard output from this task shows the Azure Batch environment variables that are set on the node. You can refer to these environment variables in your Batch job task command lines, and in the apps and scripts the command lines run. + +```text +AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1 +AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup +AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/certs +AZ_BATCH_ACCOUNT_URL=https://mybatchaccount.eastus2.batch.azure.com/ +AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/wd +AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared +AZ_BATCH_TASK_USER=_azbatch +AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks +AZ_BATCH_JOB_ID=myJob +AZ_BATCH_NODE_IS_DEDICATED=true +AZ_BATCH_NODE_ID=tvm-257509324_2-20180703t215033z +AZ_BATCH_POOL_ID=myPool +AZ_BATCH_TASK_ID=myTask1 +AZ_BATCH_ACCOUNT_NAME=mybatchaccount +AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin +``` + +## Next steps + +In this quickstart, you created a Batch account and pool, created and ran a Batch job and tasks, and viewed task output from the nodes. Now that you understand the key concepts of the Batch service, you're ready to use Batch with more realistic, larger scale workloads. To learn more about Azure Batch, continue to the Azure Batch tutorials. + +> [!div class="nextstepaction"] +> [Tutorial: Run a parallel workload with Azure Batch](./tutorial-parallel-python.md) \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index efc983f17..8ec5d5342 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -997,5 +997,23 @@ } ] } + }, + { + "status": "active", + "key": "azure-docs/articles/batch/quick-create-cli.md", + "title": "Quickstart: Use the Azure CLI to create a Batch account and run a job", + "description": "Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool.", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/batch/quick-create-cli.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/batch/quick-create-cli", + "nextSteps": [ + { + "title": "Tutorial: Run a parallel workload with Azure Batch", + "url": "https://learn.microsoft.com/en-us/azure/batch/tutorial-parallel-python" + } + ], + "configurations": { + } } ] diff --git a/tools/converted_doc.md b/tools/converted_doc.md index d4aaeaaa9..ea6b37499 100644 --- a/tools/converted_doc.md +++ b/tools/converted_doc.md @@ -27,25 +27,15 @@ After you complete this quickstart, you understand the [key concepts of the Batc >[!NOTE] >For some regions and subscription types, quota restrictions might cause Batch account or node creation to fail or not complete. In this situation, you can request a quota increase at no charge. For more information, see [Batch service quotas and limits](batch-quota-limit.md). -## Setup Environment Variables - -Below, we declare environment variables that will be used throughout this Exec Doc. We include a random suffix to uniquely name resources and avoid collisions on repeated executions. - -```bash -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export REGION="eastus2" -export RESOURCE_GROUP="qsBatch$RANDOM_SUFFIX" -export STORAGE_ACCOUNT="mybatchstorage$RANDOM_SUFFIX" -export BATCH_ACCOUNT="mybatchaccount$RANDOM_SUFFIX" -export POOL_ID="myPool$RANDOM_SUFFIX" -export JOB_ID="myJob$RANDOM_SUFFIX" -``` - ## Create a resource group Run the following [az group create](/cli/azure/group#az-group-create) command to create an Azure resource group. The resource group is a logical container that holds the Azure resources for this quickstart. ```azurecli-interactive +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="canadacentral" +export RESOURCE_GROUP="qsBatch$RANDOM_SUFFIX" + az group create \ --name $RESOURCE_GROUP \ --location $REGION @@ -76,6 +66,8 @@ Use the [az storage account create](/cli/azure/storage/account#az-storage-accoun Run the following command to create a Standard_LRS SKU storage account in your resource group: ```azurecli-interactive +export STORAGE_ACCOUNT="mybatchstorage$RANDOM_SUFFIX" + az storage account create \ --resource-group $RESOURCE_GROUP \ --name $STORAGE_ACCOUNT \ @@ -88,6 +80,8 @@ az storage account create \ Run the following [az batch account create](/cli/azure/batch/account#az-batch-account-create) command to create a Batch account in your resource group and link it with the storage account. ```azurecli-interactive +export BATCH_ACCOUNT="mybatchaccount$RANDOM_SUFFIX" + az batch account create \ --name $BATCH_ACCOUNT \ --storage-account $STORAGE_ACCOUNT \ @@ -109,6 +103,8 @@ az batch account login \ Run the [az batch pool create](/cli/azure/batch/pool#az-batch-pool-create) command to create a pool of Linux compute nodes in your Batch account. The following example creates a pool that consists of two Standard_A1_v2 size VMs running Ubuntu 20.04 LTS OS. This node size offers a good balance of performance versus cost for this quickstart example. ```azurecli-interactive +export POOL_ID="myPool$RANDOM_SUFFIX" + az batch pool create \ --id $POOL_ID \ --image canonical:0001-com-ubuntu-server-focal:20_04-lts \ @@ -121,7 +117,7 @@ Batch creates the pool immediately, but takes a few minutes to allocate and star ```azurecli-interactive az batch pool show --pool-id $POOL_ID \ - --query "allocationState" + --query "{allocationState: allocationState}" ``` Results: @@ -141,6 +137,8 @@ While Batch allocates and starts the nodes, the pool is in the `resizing` state. Use the [az batch job create](/cli/azure/batch/job#az-batch-job-create) command to create a Batch job to run on your pool. A Batch job is a logical group of one or more tasks. The job includes settings common to the tasks, such as the pool to run on. The following example creates a job that initially has no tasks. ```azurecli-interactive +export JOB_ID="myJob$RANDOM_SUFFIX" + az batch job create \ --id $JOB_ID \ --pool-id $POOL_ID @@ -183,14 +181,25 @@ The command output includes many details. For example, an `exitCode` of `0` indi Use the [az batch task file list](/cli/azure/batch/task#az-batch-task-file-show) command to list the files a task created on a node. The following command lists the files that `myTask1` created: ```azurecli-interactive -az batch task file list \ - --job-id $JOB_ID \ - --task-id myTask1 \ - --output table +# Wait for task to complete before downloading output +echo "Waiting for task to complete..." +while true; do + STATUS=$(az batch task show --job-id $JOB_ID --task-id myTask1 --query "state" -o tsv) + if [ "$STATUS" == "running" ]; then + break + fi + sleep 10 +done + +az batch task file list --job-id $JOB_ID --task-id myTask1 --output table ``` Results are similar to the following output: +Results: + + + ```output Name URL Is Directory Content Length ---------- ---------------------------------------------------------------------------------------- -------------- ---------------- diff --git a/tools/stdout.txt b/tools/stdout.txt new file mode 100644 index 000000000..d606b3d0c --- /dev/null +++ b/tools/stdout.txt @@ -0,0 +1,20 @@ +AZ_BATCH_NODE_MOUNTS_DIR=/mnt/batch/tasks/fsmounts +AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJobadb33d/job-1/myTask1/wd +AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJobadb33d/job-1/myTask1 +AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared +AZ_BATCH_TASK_USER=_azbatch +AZ_BATCH_NODE_IS_DEDICATED=true +AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup +AZ_BATCH_JOB_ID=myJobadb33d +AZ_BATCH_NODE_STARTUP_WORKING_DIR=/mnt/batch/tasks/startup/wd +AZ_BATCH_TASK_ID=myTask1 +AZ_BATCH_ACCOUNT_NAME=mybatchaccountadb33d +AZ_BATCH_RESERVED_EPHEMERAL_DISK_SPACE_BYTES=1000000000 +AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks +AZ_BATCH_POOL_ID=myPooladb33d +AZ_BATCH_RESERVED_DISK_SPACE_BYTES=1000000000 +AZ_BATCH_ACCOUNT_URL=https://mybatchaccountadb33d.canadacentral.batch.azure.com/ +AZ_BATCH_NODE_ID=tvmps_1b25c614520a9192d5e81007e1880adf7012f74bc13ba2733718a8d77878cc5b_d +AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin +AZ_BATCH_OS_RESERVED_EPHEMERAL_DISK_SPACE_BYTES=1000000000 +AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJobadb33d/job-1/myTask1/certs From 4a4fb716c1e36cfca87b348ef66790b59d89f10a Mon Sep 17 00:00:00 2001 From: naman-msft Date: Tue, 4 Mar 2025 00:09:16 -0800 Subject: [PATCH 07/87] added 4 new ai generated conversion docs --- ...edirection.virtual-machine-scale-sets.json | 0 .../virtual-machine-scale-sets/TOC.yml | 0 .../breadcrumb/toc.yml | 0 ...flexible-virtual-machine-scale-sets-cli.md | 0 .../virtual-machine-scale-sets/index.yml | 0 .../tutorial-autoscale-cli.md | 147 ++++++ .../tutorial-modify-scale-sets-cli.md | 441 ++++++++++++++++++ .../tutorial-use-custom-image-cli.md | 0 .../virtual-machine-scale-sets-faq.yml | 0 .../disks-enable-performance.md | 331 +++++++++++++ .../linux/quick-create-cli.md | 0 .../linux/tutorial-lemp-stack.md | 0 .../linux/tutorial-manage-vm.md | 330 +++++++++++++ scenarios/metadata.json | 86 +++- tools/ada.py | 69 ++- tools/converted_doc.md | 247 ---------- tools/doc.md | 211 --------- tools/execution_log.csv | 35 ++ tools/stdout.txt | 20 - 19 files changed, 1411 insertions(+), 506 deletions(-) rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machine-scale-sets/.openpublishing.redirection.virtual-machine-scale-sets.json (100%) rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machine-scale-sets/TOC.yml (100%) rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machine-scale-sets/breadcrumb/toc.yml (100%) rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md (100%) rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machine-scale-sets/index.yml (100%) create mode 100644 scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md create mode 100644 scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md (100%) rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machine-scale-sets/virtual-machine-scale-sets-faq.yml (100%) create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machines/linux/quick-create-cli.md (100%) rename scenarios/{azure-docs => azure-compute-docs}/articles/virtual-machines/linux/tutorial-lemp-stack.md (100%) create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md delete mode 100644 tools/converted_doc.md delete mode 100644 tools/doc.md delete mode 100644 tools/stdout.txt diff --git a/scenarios/azure-docs/articles/virtual-machine-scale-sets/.openpublishing.redirection.virtual-machine-scale-sets.json b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/.openpublishing.redirection.virtual-machine-scale-sets.json similarity index 100% rename from scenarios/azure-docs/articles/virtual-machine-scale-sets/.openpublishing.redirection.virtual-machine-scale-sets.json rename to scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/.openpublishing.redirection.virtual-machine-scale-sets.json diff --git a/scenarios/azure-docs/articles/virtual-machine-scale-sets/TOC.yml b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/TOC.yml similarity index 100% rename from scenarios/azure-docs/articles/virtual-machine-scale-sets/TOC.yml rename to scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/TOC.yml diff --git a/scenarios/azure-docs/articles/virtual-machine-scale-sets/breadcrumb/toc.yml b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/breadcrumb/toc.yml similarity index 100% rename from scenarios/azure-docs/articles/virtual-machine-scale-sets/breadcrumb/toc.yml rename to scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/breadcrumb/toc.yml diff --git a/scenarios/azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md similarity index 100% rename from scenarios/azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md rename to scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md diff --git a/scenarios/azure-docs/articles/virtual-machine-scale-sets/index.yml b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/index.yml similarity index 100% rename from scenarios/azure-docs/articles/virtual-machine-scale-sets/index.yml rename to scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/index.yml diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md new file mode 100644 index 000000000..65e4bdc85 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md @@ -0,0 +1,147 @@ +--- +title: Tutorial - Autoscale a scale set with the Azure CLI +description: Learn how to use the Azure CLI to automatically scale a Virtual Machine Scale Set as CPU demands increases and decreases +author: ju-shim +ms.author: jushiman +ms.topic: tutorial +ms.service: azure-virtual-machine-scale-sets +ms.subservice: autoscale +ms.date: 06/14/2024 +ms.reviewer: mimckitt +ms.custom: avverma, devx-track-azurecli, linux-related-content, innovation-engine +--- + +# Tutorial: Automatically scale a Virtual Machine Scale Set with the Azure CLI + +When you create a scale set, you define the number of VM instances that you wish to run. As your application demand changes, you can automatically increase or decrease the number of VM instances. The ability to autoscale lets you keep up with customer demand or respond to application performance changes throughout the lifecycle of your app. In this tutorial you learn how to: + +> [!div class="checklist"] +> * Use autoscale with a scale set +> * Create and use autoscale rules +> * Simulate CPU load to trigger autoscale rules +> * Monitor autoscale actions as demand changes + +[!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] + +[!INCLUDE [azure-cli-prepare-your-environment.md](~/reusable-content/azure-cli/azure-cli-prepare-your-environment.md)] + +- This tutorial requires version 2.0.32 or later of the Azure CLI. If using Azure Cloud Shell, the latest version is already installed. + +## Create a scale set +Create a resource group with [az group create](/cli/azure/group). + +```azurecli-interactive +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="WestUS2" +export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" +az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION +``` + +Now create a Virtual Machine Scale Set with [az vmss create](/cli/azure/vmss). The following example creates a scale set with an instance count of 2, generates SSH keys if they don't exist, and uses a valid image "Ubuntu2204". + +```azurecli-interactive +export MY_SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" +az vmss create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_SCALE_SET_NAME \ + --image Ubuntu2204 \ + --orchestration-mode Flexible \ + --instance-count 2 \ + --admin-username azureuser \ + --generate-ssh-keys +``` + +## Define an autoscale profile +To enable autoscale on a scale set, you first define an autoscale profile. This profile defines the default, minimum, and maximum scale set capacity. These limits let you control cost by not continually creating VM instances, and balance acceptable performance with a minimum number of instances that remain in a scale-in event. Create an autoscale profile with [az monitor autoscale create](/cli/azure/monitor/autoscale#az-monitor-autoscale-create). The following example sets the default and minimum capacity of 2 VM instances, and a maximum of 10: + +```azurecli-interactive +az monitor autoscale create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --resource $MY_SCALE_SET_NAME \ + --resource-type Microsoft.Compute/virtualMachineScaleSets \ + --name autoscale \ + --min-count 2 \ + --max-count 10 \ + --count 2 +``` + +## Create a rule to autoscale out +If your application demand increases, the load on the VM instances in your scale set increases. If this increased load is consistent, rather than just a brief demand, you can configure autoscale rules to increase the number of VM instances. When these instances are created and your application is deployed, the scale set starts to distribute traffic to them through the load balancer. You control which metrics to monitor, how long the load must meet a given threshold, and how many VM instances to add. + +Create a rule with [az monitor autoscale rule create](/cli/azure/monitor/autoscale/rule#az-monitor-autoscale-rule-create) that increases the number of VM instances when the average CPU load is greater than 70% over a 5-minute period. When the rule triggers, the number of VM instances is increased by three. + +```azurecli-interactive +az monitor autoscale rule create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --autoscale-name autoscale \ + --condition "Percentage CPU > 70 avg 5m" \ + --scale out 3 +``` + +## Create a rule to autoscale in +When application demand decreases, the load on the VM instances drops. If this decreased load persists over a period of time, you can configure autoscale rules to decrease the number of VM instances in the scale set. This scale-in action helps reduce costs by running only the necessary number of instances required to meet current demand. + +Create another rule with [az monitor autoscale rule create](/cli/azure/monitor/autoscale/rule#az-monitor-autoscale-rule-create) that decreases the number of VM instances when the average CPU load drops below 30% over a 5-minute period. The following example scales in the number of VM instances by one. + +```azurecli-interactive +az monitor autoscale rule create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --autoscale-name autoscale \ + --condition "Percentage CPU < 30 avg 5m" \ + --scale in 1 +``` + +## Simulate CPU load on scale set +To test the autoscale rules, you need to simulate sustained CPU load on the VM instances in the scale set. In this minimalist approach, we avoid installing additional packages by using the built-in "yes" command to generate CPU load. The following command starts 3 background processes that continuously output data to /dev/null for 60 seconds and then terminates them. + +```bash +for i in {1..3}; do + yes > /dev/null & +done +sleep 60 +pkill yes +``` + +This command simulates CPU load without introducing package installation errors. + +## Monitor the active autoscale rules +To monitor the number of VM instances in your scale set, use the watch command. It may take up to 5 minutes for the autoscale rules to begin the scale-out process in response to the CPU load. However, once it happens, you can exit watch with Ctrl-c. + +By then, the scale set will automatically increase the number of VM instances to meet the demand. The following command shows the list of VM instances in the scale set: + +```azurecli-interactive +az vmss list-instances \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_SCALE_SET_NAME \ + --output table +``` + +Once the CPU threshold has been met, the autoscale rules increase the number of VM instances in the scale set. The output will show the list of VM instances as new ones are created. + +```output + InstanceId LatestModelApplied Location Name ProvisioningState ResourceGroup VmId +------------ -------------------- ---------- --------------- ------------------- -------------------- ------------------------------------ + 1 True WestUS2 myScaleSet_1 Succeeded myResourceGroupxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + 2 True WestUS2 myScaleSet_2 Succeeded myResourceGroupxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + 4 True WestUS2 myScaleSet_4 Creating myResourceGroupxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + 5 True WestUS2 myScaleSet_5 Creating myResourceGroupxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + 6 True WestUS2 myScaleSet_6 Creating myResourceGroupxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +Once the CPU load subsides, the average CPU load returns to normal. After another 5 minutes, the autoscale rules then scale in the number of VM instances. Scale-in actions remove VM instances with the highest IDs first. When a scale set uses Availability Sets or Availability Zones, scale-in actions are evenly distributed across the VM instances. The following sample output shows one VM instance being deleted as the scale set autoscales in: + +```output +6 True WestUS2 myScaleSet_6 Deleting myResourceGroupxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +## Clean up resources +To remove your scale set and associated resources, please manually delete the resource group using your preferred method. + +## Next steps +In this tutorial, you learned how to automatically scale in or out a scale set with the Azure CLI: + +> [!div class="checklist"] +> * Use autoscale with a scale set +> * Create and use autoscale rules +> * Simulate CPU load to trigger autoscale rules +> * Monitor autoscale actions as demand changes \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md new file mode 100644 index 000000000..1ffa46d52 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md @@ -0,0 +1,441 @@ +--- +title: Modify an Azure Virtual Machine Scale Set using Azure CLI +description: Learn how to modify and update an Azure Virtual Machine Scale Set using Azure CLI +author: ju-shim +ms.author: jushiman +ms.topic: how-to +ms.service: azure-virtual-machine-scale-sets +ms.date: 06/14/2024 +ms.reviewer: mimckitt +ms.custom: mimckitt, devx-track-azurecli, linux-related-content, innovation-engine +--- + +# Tutorial: Modify a Virtual Machine Scale Set using Azure CLI +Throughout the lifecycle of your applications, you may need to modify or update your Virtual Machine Scale Set. These updates may include how to update the configuration of the scale set, or change the application configuration. This article describes how to modify an existing scale set using the Azure CLI. + +Below, we declare environment variables that will be used throughout this document. A random suffix is appended to resource names that need to be unique for each deployment. The REGION is set to WestUS2. + +```bash +export RANDOM_SUFFIX=adcc95 +export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" +export SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" +export NEW_INSTANCE_NAME="myNewInstance$RANDOM_SUFFIX" +export REGION="WestUS2" +``` + +## Setup Resource Group +Before proceeding, ensure the resource group exists. This step creates the resource group if it does not already exist. + +```bash +az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION +``` + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx", + "location": "WestUS2", + "managedBy": null, + "name": "myResourceGroupxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create the Virtual Machine Scale Set +To ensure that subsequent update and query commands have a valid resource to work on, create a Virtual Machine Scale Set. In this step, we deploy a basic scale set using a valid image ("Ubuntu2204") and set the instance count to 5 so that instance-specific updates can target an existing instance ID. + +```azurecli-interactive +az vmss create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $SCALE_SET_NAME \ + --image Ubuntu2204 \ + --upgrade-policy-mode manual \ + --instance-count 5 \ + --admin-username azureuser \ + --generate-ssh-keys +``` + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetxxx", + "location": "WestUS2", + "name": "myScaleSetxxx", + "provisioningState": "Succeeded" +} +``` + +## Update the scale set model +A scale set has a "scale set model" that captures the *desired* state of the scale set as a whole. To query the model for a scale set, you can use [az vmss show](/cli/azure/vmss#az-vmss-show): + +```azurecli +az vmss show --resource-group $MY_RESOURCE_GROUP_NAME --name $SCALE_SET_NAME +``` + +The exact presentation of the output depends on the options you provide to the command. The following example shows condensed sample output from the Azure CLI: + +```output +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetxxx", + "location": "WestUS2", + "name": "myScaleSetxxx", + "orchestrationMode": "Flexible", + "platformFaultDomainCount": 1, + "resourceGroup": "myResourceGroupxxx", + "sku": { + "capacity": 5, + "name": "Standard_DS1_v2", + "tier": "Standard" + }, + "timeCreated": "2022-11-29T22:16:43.250912+00:00", + "type": "Microsoft.Compute/virtualMachineScaleSets", + "networkProfile": { + "networkApiVersion": "2020-11-01", + "networkInterfaceConfigurations": [ + { + "deleteOption": "Delete", + "disableTcpStateTracking": false, + "dnsSettings": { + "dnsServers": [] + }, + "enableIpForwarding": false, + "ipConfigurations": [ + { + "applicationGatewayBackendAddressPools": [], + "applicationSecurityGroups": [], + "loadBalancerBackendAddressPools": [ + { + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Network/loadBalancers/myScaleSetLB/backendAddressPools/myScaleSetLBBEPool", + "resourceGroup": "myResourceGroupxxx" + } + ], + "name": "mysca2215IPConfig", + "privateIpAddressVersion": "IPv4", + "subnet": { + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Network/virtualNetworks/myScaleSetVNET/subnets/myScaleSetSubnet", + "resourceGroup": "myResourceGroupxxx" + } + } + ], + "name": "mysca2215Nic", + "networkSecurityGroup": { + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Network/networkSecurityGroups/myScaleSetNSG", + "resourceGroup": "myResourceGroupxxx" + }, + "primary": true + } + ] + }, + "osProfile": { + "allowExtensionOperations": true, + "computerNamePrefix": "myScaleS", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "enableVmAgentPlatformUpdates": false, + "patchSettings": { + "assessmentMode": "ImageDefault", + "patchMode": "ImageDefault" + }, + "provisionVmAgent": true + } + }, + "storageProfile": { + "imageReference": { + "offer": "UbuntuServer", + "publisher": "Canonical", + "sku": "22_04-lts", + "version": "latest" + }, + "osDisk": { + "caching": "ReadWrite", + "createOption": "FromImage", + "deleteOption": "Delete", + "diskSizeGb": 30, + "managedDisk": { + "storageAccountType": "Premium_LRS" + }, + "osType": "Linux" + } + } +} +``` + +You can use [az vmss update](/cli/azure/vmss#az-vmss-update) to update various properties of your scale set. For example, updating your license type or a VM's instance protection policy. Note that the allowed license type value is "RHEL_BYOS" rather than "Windows_Server." + +```azurecli-interactive +az vmss update --name $SCALE_SET_NAME --resource-group $MY_RESOURCE_GROUP_NAME --license-type RHEL_BYOS +``` + +```azurecli-interactive +export INSTANCE_ID=$(az vmss list-instances \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $SCALE_SET_NAME \ + --query "[0].instanceId" \ + -o tsv) + +az vmss update \ + --name $SCALE_SET_NAME \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --instance-id "$INSTANCE_ID" \ + --protect-from-scale-set-actions False \ + --protect-from-scale-in +``` + +Additionally, if you previously deployed the scale set with the az vmss create command, you can run the az vmss create command again to update the scale set. Make sure that all properties in the az vmss create command are the same as before, except for the properties that you wish to modify. For example, below we're increasing the instance count to five. + +> [!IMPORTANT] +>Starting November 2023, VM scale sets created using PowerShell and Azure CLI will default to Flexible Orchestration Mode if no orchestration mode is specified. For more information about this change and what actions you should take, go to [Breaking Change for VMSS PowerShell/CLI Customers - Microsoft Community Hub](https://techcommunity.microsoft.com/t5/azure-compute-blog/breaking-change-for-vmss-powershell-cli-customers/ba-p/3818295) + +```azurecli-interactive +az vmss create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $SCALE_SET_NAME \ + --orchestration-mode flexible \ + --image RHELRaw8LVMGen2 \ + --admin-username azureuser \ + --generate-ssh-keys \ + --instance-count 5 \ + --os-disk-size-gb 64 +``` + +## Updating individual VM instances in a scale set +Similar to how a scale set has a model view, each VM instance in the scale set has its own model view. To query the model view for a particular VM instance in a scale set, you can use [az vm show](/cli/azure/vm#az-vm-show). + +```azurecli +export INSTANCE_NAME=$(az vmss list-instances \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $SCALE_SET_NAME \ + --query "[0].name" \ + -o tsv) + +az vm show --resource-group $MY_RESOURCE_GROUP_NAME --name $INSTANCE_NAME +``` + +The exact presentation of the output depends on the options you provide to the command. The following example shows condensed sample output from the Azure CLI: + +```output +{ + "hardwareProfile": { + "vmSize": "Standard_DS1_v2" + }, + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/virtualMachines/myScaleSet_Instance1", + "location": "WestUS2", + "name": "myScaleSet_Instance1", + "networkProfile": { + "networkInterfaces": [ + { + "deleteOption": "Delete", + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Network/networkInterfaces/mysca2215Nic-5cf164f7", + "primary": true, + "resourceGroup": "myResourceGroupxxx" + } + ] + }, + "osProfile": { + "allowExtensionOperations": true, + "computerName": "myScaleset_Computer1", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "enableVmAgentPlatformUpdates": false, + "patchSettings": { + "assessmentMode": "ImageDefault", + "patchMode": "ImageDefault" + }, + "provisionVmAgent": true + } + }, + "provisioningState": "Succeeded", + "resourceGroup": "myResourceGroupxxx", + "storageProfile": { + "dataDisks": [], + "imageReference": { + "exactVersion": "22.04.202204200", + "offer": "0001-com-ubuntu-server-jammy", + "publisher": "Canonical", + "sku": "22_04-lts", + "version": "latest" + }, + "osDisk": { + "caching": "ReadWrite", + "createOption": "FromImage", + "deleteOption": "Delete", + "diskSizeGb": 30, + "managedDisk": { + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/disks/myScaleSet_Instance1_disk1_xxx", + "resourceGroup": "myResourceGroupxxx", + "storageAccountType": "Premium_LRS" + }, + "name": "myScaleSet_Instance1_disk1_xxx", + "osType": "Linux" + } + }, + "timeCreated": "2022-11-29T22:16:44.500895+00:00", + "type": "Microsoft.Compute/virtualMachines", + "virtualMachineScaleSet": { + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetxxx", + "resourceGroup": "myResourceGroupxxx" + } +} +``` + +These properties describe the configuration of a VM instance within a scale set, not the configuration of the scale set as a whole. + +You can perform updates to individual VM instances in a scale set just like you would a standalone VM. For example, attaching a new data disk to instance 1: + +```azurecli-interactive +az vm disk attach --resource-group $MY_RESOURCE_GROUP_NAME --vm-name $INSTANCE_NAME --name disk_name1 --new +``` + +Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the VM instance has the new disk attached. + +```output +{ + "storageProfile": { + "dataDisks": [ + { + "caching": "None", + "createOption": "Empty", + "deleteOption": "Detach", + "diskSizeGb": 1023, + "lun": 0, + "managedDisk": { + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/disks/disk_name1", + "resourceGroup": "myResourceGroupxxx", + "storageAccountType": "Premium_LRS" + }, + "name": "disk_name1", + "toBeDetached": false + } + ] + } +} +``` + +## Add an Instance to your scale set +There are times where you might want to add a new VM to your scale set but want different configuration options than those listed in the scale set model. VMs can be added to a scale set during creation by using the [az vm create](/cli/azure/vmss#az-vmss-create) command and specifying the scale set name you want the instance added to. + +```azurecli-interactive +az vm create --name $NEW_INSTANCE_NAME --resource-group $MY_RESOURCE_GROUP_NAME --vmss $SCALE_SET_NAME --image RHELRaw8LVMGen2 +``` + +```output +{ + "fqdns": "", + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/virtualMachines/myNewInstancexxx", + "location": "WestUS2", + "macAddress": "60-45-BD-D7-13-DD", + "powerState": "VM running", + "privateIpAddress": "10.0.0.6", + "publicIpAddress": "20.172.144.96", + "resourceGroup": "myResourceGroupxxx", + "zones": "" +} +``` + +If we then check our scale set, we'll see the new instance added. + +```azurecli-interactive +az vm list --resource-group $MY_RESOURCE_GROUP_NAME --output table +``` + +```output +Name ResourceGroup Location +-------------------- --------------- ---------- +myNewInstancexxx myResourceGroupxxx WestUS2 +myScaleSet_Instance1 myResourceGroupxxx WestUS2 +myScaleSet_Instance1 myResourceGroupxxx WestUS2 +``` + +## Bring VMs up-to-date with the latest scale set model + +> [!NOTE] +> Upgrade modes are not currently supported on Virtual Machine Scale Sets using Flexible orchestration mode. + +Scale sets have an "upgrade policy" that determine how VMs are brought up-to-date with the latest scale set model. The three modes for the upgrade policy are: + +- **Automatic** - In this mode, the scale set makes no guarantees about the order of VMs being brought down. The scale set may take down all VMs at the same time. +- **Rolling** - In this mode, the scale set rolls out the update in batches with an optional pause time between batches. +- **Manual** - In this mode, when you update the scale set model, nothing happens to existing VMs until a manual update is triggered. + +If your scale set is set to manual upgrades, you can trigger a manual upgrade using [az vmss update](/cli/azure/vmss#az-vmss-update). + +```azurecli +az vmss update --resource-group $MY_RESOURCE_GROUP_NAME --name $SCALE_SET_NAME +``` + +>[!NOTE] +> Service Fabric clusters can only use *Automatic* mode, but the update is handled differently. For more information, see [Service Fabric application upgrades](../service-fabric/service-fabric-application-upgrade.md). + +## Reimage a scale set +Virtual Machine Scale Sets will generate a unique name for each VM in the scale set. The naming convention differs by orchestration mode: + +- Flexible orchestration Mode: {scale-set-name}_{8-char-guid} +- Uniform orchestration mode: {scale-set-name}_{instance-id} + +In the cases where you need to reimage a specific instance, use [az vmss reimage](/cli/azure/vmss#az-vmss-reimage) and specify the instance id. Another option is to use [az vm redeploy](/cli/azure/vm#az-vm-redeploy) to reimage the VM directly. This command is useful if you want to reimage a VM without having to specify the instance ID. + +```azurecli +# Get the VM name first +VM_NAME=$(az vmss list-instances \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $SCALE_SET_NAME \ + --query "[0].name" \ + -o tsv) + +# Reimage the VM directly +az vm redeploy \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $VM_NAME +``` + +## Update the OS image for your scale set +You may have a scale set that runs an old version of Ubuntu. You want to update to a newer version of Ubuntu, such as the latest version. The image reference version property isn't part of a list, so you can directly modify these properties using [az vmss update](/cli/azure/vmss#az-vmss-update). + +```azurecli +az vmss update --resource-group $MY_RESOURCE_GROUP_NAME --name $SCALE_SET_NAME --set virtualMachineProfile.storageProfile.imageReference.version=latest +``` + +Alternatively, you may want to change the image your scale set uses. For example, you may want to update or change a custom image used by your scale set. You can change the image your scale set uses by updating the image reference ID property. The image reference ID property isn't part of a list, so you can directly modify this property using [az vmss update](/cli/azure/vmss#az-vmss-update). + +If you use Azure platform images, you can update the image by modifying the *imageReference* (more information, see the [REST API documentation](/rest/api/compute/virtualmachinescalesets/createorupdate)). + +>[!NOTE] +> With platform images, it is common to specify "latest" for the image reference version. When you create, scale out, and reimage, VMs are created with the latest available version. However, it **does not** mean that the OS image is automatically updated over time as new image versions are released. A separate feature provides automatic OS upgrades. For more information, see the [Automatic OS Upgrades documentation](virtual-machine-scale-sets-automatic-upgrade.md). + +If you use custom images, you can update the image by updating the *imageReference* ID (more information, see the [REST API documentation](/rest/api/compute/virtualmachinescalesets/createorupdate)). + +## Update the load balancer for your scale set +Let's say you have a scale set with an Azure Load Balancer, and you want to replace the Azure Load Balancer with an Azure Application Gateway. The load balancer and Application Gateway properties for a scale set are part of a list, so you can use the commands to remove or add list elements instead of modifying the properties directly. + +```text +# Remove the load balancer backend pool from the scale set model +az vmss update --resource-group $MY_RESOURCE_GROUP_NAME --name $SCALE_SET_NAME --remove virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0].loadBalancerBackendAddressPools 0 + +# Remove the load balancer backend pool from the scale set model; only necessary if you have NAT pools configured on the scale set +az vmss update --resource-group $MY_RESOURCE_GROUP_NAME --name $SCALE_SET_NAME --remove virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0].loadBalancerInboundNatPools 0 + +# Add the application gateway backend pool to the scale set model +az vmss update --resource-group $MY_RESOURCE_GROUP_NAME --name $SCALE_SET_NAME --add virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0].ApplicationGatewayBackendAddressPools '{"id": "/subscriptions/xxxxx/resourceGroups/'$MY_RESOURCE_GROUP_NAME'/providers/Microsoft.Network/applicationGateways/{applicationGatewayName}/backendAddressPools/{applicationGatewayBackendPoolName}"}' +``` + +>[!NOTE] +> These commands assume there is only one IP configuration and load balancer on the scale set. If there are multiple, you may need to use a list index other than *0*. + +## Next steps +In this tutorial, you learned how to modify various aspects of your scale set and individual instances. + +> [!div class="checklist"] +> * Update the scale set model +> * Update an individual VM instance in a scale set +> * Add an instance to your scale set +> * Bring VMs up-to-date with the latest scale set model +> * Reimage a scale set +> * Update the OS image for your scale set +> * Update the load balancer for your scale set + +> [!div class="nextstepaction"] +> [Use data disks with scale sets](tutorial-use-disks-powershell.md) \ No newline at end of file diff --git a/scenarios/azure-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md similarity index 100% rename from scenarios/azure-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md rename to scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md diff --git a/scenarios/azure-docs/articles/virtual-machine-scale-sets/virtual-machine-scale-sets-faq.yml b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/virtual-machine-scale-sets-faq.yml similarity index 100% rename from scenarios/azure-docs/articles/virtual-machine-scale-sets/virtual-machine-scale-sets-faq.yml rename to scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/virtual-machine-scale-sets-faq.yml diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md b/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md new file mode 100644 index 000000000..12c533f61 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md @@ -0,0 +1,331 @@ +--- +title: Preview - Increase performance of Premium SSDs and Standard SSD/HDDs +description: Increase the performance of Azure Premium SSDs and Standard SSD/HDDs using performance plus. +author: roygara +ms.service: azure-disk-storage +ms.topic: how-to +ms.date: 12/09/2024 +ms.author: rogarana +ms.custom: devx-track-azurepowershell, innovation-engine +--- + +# Preview - Increase IOPS and throughput limits for Azure Premium SSDs and Standard SSD/HDDs + +The Input/Output Operations Per Second (IOPS) and throughput limits for Azure Premium solid-state drives (SSD), Standard SSDs, and Standard hard disk drives (HDD) that are 513 GiB and larger can be increased by enabling performance plus. Enabling performance plus (preview) improves the experience for workloads that require high IOPS and throughput, such as database and transactional workloads. There's no extra charge for enabling performance plus on a disk. + +Once enabled, the IOPS and throughput limits for an eligible disk increase to the higher maximum limits. To see the new IOPS and throughput limits for eligible disks, consult the columns that begin with "*Expanded" in the [Scalability and performance targets for VM disks](disks-scalability-targets.md) article. + +## Limitations + +- Can only be enabled on Standard HDD, Standard SSD, and Premium SSD managed disks that are 513 GiB or larger. +- Can only be enabled on new disks. + - To work around this, create a snapshot of your disk, then create a new disk from the snapshot. +- Not supported for disks recovered with Azure Site Recovery or Azure Backup. +- Can't be enabled in the Azure portal. + +## Prerequisites + +Either use the Azure Cloud Shell to run your commands or install a version of the [Azure PowerShell module](/powershell/azure/install-azure-powershell) 9.5 or newer, or a version of the [Azure CLI](/cli/azure/install-azure-cli) that is 2.44.0 or newer. + +## Enable performance plus + +You need to create a new disk to use performance plus. The following scripts show how to create a disk with performance plus enabled and, if desired, attach it to a VM. The commands have been organized into self-contained steps for reliability. + +# [Azure CLI](#tab/azure-cli) + +### Create a resource group + +This step creates a resource group with a unique name. + +```azurecli +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export MY_RG="PerfPlusRG$RANDOM_SUFFIX" +export REGION="WestUS2" +az group create -g $MY_RG -l $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/PerfPlusRGxxx", + "location": "WestUS2", + "name": "PerfPlusRGxxx", + "properties": { + "provisioningState": "Succeeded" + } +} +``` + +### Create a new disk with performance plus enabled + +This step creates a new disk of 513 GiB (or larger) with performance plus enabled using a valid SKU value. + +```azurecli +export MY_DISK="PerfPlusDisk$RANDOM_SUFFIX" +export SKU="Premium_LRS" +export DISK_SIZE=513 +az disk create -g $MY_RG -n $MY_DISK --size-gb $DISK_SIZE --sku $SKU -l $REGION --performance-plus true +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/PerfPlusRGxxx/providers/Microsoft.Compute/disks/PerfPlusDiskxxx", + "location": "WestUS2", + "name": "PerfPlusDiskxxx", + "properties": { + "provisioningState": "Succeeded", + "diskSizeGb": 513, + "sku": "Premium_LRS", + "performancePlus": true + }, + "type": "Microsoft.Compute/disks" +} +``` + +### Attempt to attach the disk to a VM + +This optional step attempts to attach the disk to an existing VM. It first checks if the VM exists and then proceeds accordingly. + +```azurecli +export MY_VM="NonExistentVM" +if az vm show -g $MY_RG -n $MY_VM --query "name" --output tsv >/dev/null 2>&1; then + az vm disk attach --vm-name $MY_VM --name $MY_DISK --resource-group $MY_RG +else + echo "VM $MY_VM not found. Skipping disk attachment." +fi +``` + +Results: + + +```text +VM NonExistentVM not found. Skipping disk attachment. +``` + +### Create a new disk from an existing disk or snapshot with performance plus enabled + +This series of steps creates a separate resource group and then creates a new disk from an existing disk or snapshot. Replace the SOURCE_URI with a valid source blob URI that belongs to the same region (WestUS2) as the disk. + +#### Create a resource group for migration + +```azurecli +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export MY_MIG_RG="PerfPlusMigrRG$RANDOM_SUFFIX" +export REGION="WestUS2" +az group create -g $MY_MIG_RG -l $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/PerfPlusMigrRGxxx", + "location": "WestUS2", + "name": "PerfPlusMigrRGxxx", + "properties": { + "provisioningState": "Succeeded" + } +} +``` + +#### Create the disk from an existing snapshot or disk + +```azurecli +# Create a snapshot from the original disk +export MY_SNAPSHOT_NAME="PerfPlusSnapshot$RANDOM_SUFFIX" +echo "Creating snapshot from original disk..." +az snapshot create \ + --name $MY_SNAPSHOT_NAME \ + --resource-group $MY_RG \ + --source $MY_DISK + +# Get the snapshot ID for use as source +SNAPSHOT_ID=$(az snapshot show \ + --name $MY_SNAPSHOT_NAME \ + --resource-group $MY_RG \ + --query id \ + --output tsv) + +echo "Using snapshot ID: $SNAPSHOT_ID" + +# Create the new disk using the snapshot as source +export MY_MIG_DISK="PerfPlusMigrDisk$RANDOM_SUFFIX" +export SKU="Premium_LRS" +export DISK_SIZE=513 + +az disk create \ + --name $MY_MIG_DISK \ + --resource-group $MY_MIG_RG \ + --size-gb $DISK_SIZE \ + --performance-plus true \ + --sku $SKU \ + --source $SNAPSHOT_ID \ + --location $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/PerfPlusMigrRGxxx/providers/Microsoft.Compute/disks/PerfPlusMigrDiskxxx", + "location": "WestUS2", + "name": "PerfPlusMigrDiskxxx", + "properties": { + "provisioningState": "Succeeded", + "diskSizeGb": 513, + "sku": "Premium_LRS", + "performancePlus": true, + "source": "https://examplestorageaccount.blob.core.windows.net/snapshots/sample-westus2.vhd" + }, + "type": "Microsoft.Compute/disks" +} +``` + +# [Azure PowerShell](#tab/azure-powershell) + +### Create a resource group + +This step creates a resource group with a unique name. + +```azurepowershell +$RANDOM_SUFFIX = (New-Guid).Guid.Substring(0,6) +$myRG = "PerfPlusRG$RANDOM_SUFFIX" +$region = "WestUS2" +New-AzResourceGroup -Name $myRG -Location $region +``` + +Results: + + +```JSON +{ + "ResourceGroupName": "PerfPlusRGxxx", + "Location": "WestUS2", + "ProvisioningState": "Succeeded" +} +``` + +### Create a new disk with performance plus enabled + +This step creates a new disk with performance plus enabled using a valid SKU value. + +```azurepowershell +$myDisk = "PerfPlusDisk$RANDOM_SUFFIX" +$sku = "Premium_LRS" +$size = 513 +$diskConfig = New-AzDiskConfig -Location $region -CreateOption Empty -DiskSizeGB $size -SkuName $sku -PerformancePlus $true +$dataDisk = New-AzDisk -ResourceGroupName $myRG -DiskName $myDisk -Disk $diskConfig +``` + +Results: + + +```JSON +{ + "ResourceGroup": "PerfPlusRGxxx", + "Name": "PerfPlusDiskxxx", + "Location": "WestUS2", + "Sku": "Premium_LRS", + "DiskSizeGB": 513, + "PerformancePlus": true, + "ProvisioningState": "Succeeded" +} +``` + +### Attempt to attach the disk to a VM + +This optional step checks whether the specified VM exists before attempting the disk attachment. + +```azurepowershell +$myVM = "NonExistentVM" +if (Get-AzVM -ResourceGroupName $myRG -Name $myVM -ErrorAction SilentlyContinue) { + Add-AzVMDataDisk -VMName $myVM -ResourceGroupName $myRG -DiskName $myDisk -Lun 0 -CreateOption Empty -ManagedDiskId $dataDisk.Id +} else { + Write-Output "VM $myVM not found. Skipping disk attachment." +} +``` + +Results: + + +```text +VM NonExistentVM not found. Skipping disk attachment. +``` + +### Create a new disk from an existing disk or snapshot with performance plus enabled + +This series of steps creates a separate resource group and then creates a new disk from an existing disk or snapshot. Replace the $sourceURI with a valid source blob URI that belongs to the same region (WestUS2) as the disk. + +#### Create a resource group for migration + +```azurepowershell +$RANDOM_SUFFIX = (New-Guid).Guid.Substring(0,6) +$myMigrRG = "PerfPlusMigrRG$RANDOM_SUFFIX" +$region = "WestUS2" +New-AzResourceGroup -Name $myMigrRG -Location $region +``` + +Results: + + +```JSON +{ + "ResourceGroupName": "PerfPlusMigrRGxxx", + "Location": "WestUS2", + "ProvisioningState": "Succeeded" +} +``` + +#### Create the disk from an existing snapshot or disk + +```azurepowershell +$myDisk = "PerfPlusMigrDisk$RANDOM_SUFFIX" +$sku = "Premium_LRS" +$size = 513 +$sourceURI = "https://examplestorageaccount.blob.core.windows.net/snapshots/sample-westus2.vhd" # Replace with a valid source blob URI in WestUS2 +$diskConfig = New-AzDiskConfig -Location $region -CreateOption Copy -DiskSizeGB $size -SkuName $sku -PerformancePlus $true -SourceResourceID $sourceURI +$dataDisk = New-AzDisk -ResourceGroupName $myMigrRG -DiskName $myDisk -Disk $diskConfig +``` + +Results: + + +```JSON +{ + "ResourceGroup": "PerfPlusMigrRGxxx", + "Name": "PerfPlusMigrDiskxxx", + "Location": "WestUS2", + "Sku": "Premium_LRS", + "DiskSizeGB": 513, + "PerformancePlus": true, + "SourceResourceID": "https://examplestorageaccount.blob.core.windows.net/snapshots/sample-westus2.vhd", + "ProvisioningState": "Succeeded" +} +``` + +#### Attempt to attach the migrated disk to a VM + +This optional step verifies the existence of the specified VM before attempting disk attachment. + +```azurepowershell +$myVM = "NonExistentVM" +if (Get-AzVM -ResourceGroupName $myMigrRG -Name $myVM -ErrorAction SilentlyContinue) { + Add-AzVMDataDisk -VMName $myVM -ResourceGroupName $myMigrRG -DiskName $myDisk -Lun 0 -CreateOption Empty -ManagedDiskId $dataDisk.Id +} else { + Write-Output "VM $myVM not found. Skipping disk attachment." +} +``` + +Results: + + +```text +VM NonExistentVM not found. Skipping disk attachment. +``` \ No newline at end of file diff --git a/scenarios/azure-docs/articles/virtual-machines/linux/quick-create-cli.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md similarity index 100% rename from scenarios/azure-docs/articles/virtual-machines/linux/quick-create-cli.md rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md diff --git a/scenarios/azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md similarity index 100% rename from scenarios/azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md new file mode 100644 index 000000000..b0830b8af --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md @@ -0,0 +1,330 @@ +--- +title: Tutorial - Create and manage Linux VMs with the Azure CLI +description: In this tutorial, you learn how to use the Azure CLI to create and manage Linux VMs in Azure +author: ju-shim +ms.service: azure-virtual-machines +ms.collection: linux +ms.topic: tutorial +ms.date: 03/23/2023 +ms.author: jushiman +ms.custom: mvc, devx-track-azurecli, linux-related-content, innovation-engine +#Customer intent: As an IT administrator, I want to learn about common maintenance tasks so that I can create and manage Linux VMs in Azure +--- + +# Tutorial: Create and Manage Linux VMs with the Azure CLI + +**Applies to:** :heavy_check_mark: Linux VMs :heavy_check_mark: Flexible scale sets + +Azure virtual machines provide a fully configurable and flexible computing environment. This tutorial covers basic Azure virtual machine deployment items such as selecting a VM size, selecting a VM image, and deploying a VM. You learn how to: + +> [!div class="checklist"] +> * Create and connect to a VM +> * Select and use VM images +> * View and use specific VM sizes +> * Resize a VM +> * View and understand VM state + +This tutorial uses the CLI within the [Azure Cloud Shell](/azure/cloud-shell/overview), which is constantly updated to the latest version. + +If you choose to install and use the CLI locally, this tutorial requires that you are running the Azure CLI version 2.0.30 or later. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI]( /cli/azure/install-azure-cli). + +## Create resource group + +Below, we declare environment variables. A random suffix is appended to resource names that need to be unique for each deployment. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="eastus2" +export MY_RESOURCE_GROUP_NAME="myResourceGroupVM$RANDOM_SUFFIX" +az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupVMxxx", + "location": "eastus2", + "name": "myResourceGroupVMxxx", + "properties": { + "provisioningState": "Succeeded" + } +} +``` + +An Azure resource group is a logical container into which Azure resources are deployed and managed. A resource group must be created before a virtual machine. In this example, a resource group named *myResourceGroupVM* is created in the *eastus2* region. + +The resource group is specified when creating or modifying a VM, which can be seen throughout this tutorial. + +## Create virtual machine + +When you create a virtual machine, several options are available such as operating system image, disk sizing, and administrative credentials. The following example creates a VM named *myVM* that runs SUSE Linux Enterprise Server (SLES). A user account named *azureuser* is created on the VM, and SSH keys are generated if they do not exist in the default key location (*~/.ssh*). + +```bash +export MY_VM_NAME="myVM$RANDOM_SUFFIX" +az vm create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_VM_NAME \ + --image SuseSles15SP5 \ + --public-ip-sku Standard \ + --admin-username azureuser \ + --generate-ssh-keys +``` + +It may take a few minutes to create the VM. Once the VM has been created, the Azure CLI outputs information about the VM. Take note of the `publicIpAddress`; this address can be used to access the virtual machine. + +```JSON +{ + "fqdns": "", + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupVMxxx/providers/Microsoft.Compute/virtualMachines/myVMxxx", + "location": "eastus2", + "macAddress": "00-0D-3A-23-9A-49", + "powerState": "VM running", + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "52.174.34.95", + "resourceGroup": "myResourceGroupVMxxx" +} +``` + +## Connect to VM + +The original tutorial includes commands to connect to the VM via SSH. For non-interactive automated execution, the SSH command is not executed. Instead, use the provided public IP address output from VM creation to manually connect if needed. + +To connect to the VM, first retrieve the public IP address using the Azure CLI. Execute the following command to store the IP address in a variable: ```export IP_ADDRESS=$(az vm show --show-details --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --query publicIps --output tsv)``` + +Once you have the IP address, use SSH to connect to the VM. The following command connects to the VM using the `azureuser` account and the retrieved IP address: ```ssh -o StrictHostKeyChecking=no azureuser@$IP_ADDRESS``` + +## Understand VM images + +The Azure Marketplace includes many images that can be used to create VMs. In the previous steps, a virtual machine was created using a SUSE image. In this step, the Azure CLI is used to search the marketplace for an Ubuntu image, which is then used to deploy a second virtual machine. + +To see a list of the most commonly used images, use the [az vm image list](/cli/azure/vm/image) command. + +```bash +az vm image list --output table +``` + +The command output returns the most popular VM images on Azure. + +```output +Architecture Offer Publisher Sku Urn UrnAlias Version +-------------- ---------------------------- ---------------------- ---------------------------------- ------------------------------------------------------------------------------ ----------------------- --------- +x64 debian-10 Debian 10 Debian:debian-10:10:latest Debian latest +x64 flatcar-container-linux-free kinvolk stable kinvolk:flatcar-container-linux-free:stable:latest Flatcar latest +x64 opensuse-leap-15-3 SUSE gen2 SUSE:opensuse-leap-15-3:gen2:latest openSUSE-Leap latest +x64 RHEL RedHat 7-LVM RedHat:RHEL:7-LVM:latest RHEL latest +x64 sles-15-sp3 SUSE gen2 SUSE:sles-15-sp3:gen2:latest SLES latest +x64 0001-com-ubuntu-server-jammy Canonical 18.04-LTS Canonical:UbuntuServer:18.04-LTS:latest UbuntuLTS latest +x64 WindowsServer MicrosoftWindowsServer 2022-Datacenter MicrosoftWindowsServer:WindowsServer:2022-Datacenter:latest Win2022Datacenter latest +x64 WindowsServer MicrosoftWindowsServer 2022-datacenter-azure-edition-core MicrosoftWindowsServer:WindowsServer:2022-datacenter-azure-edition-core:latest Win2022AzureEditionCore latest +x64 WindowsServer MicrosoftWindowsServer 2019-Datacenter MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest Win2019Datacenter latest +x64 WindowsServer MicrosoftWindowsServer 2016-Datacenter MicrosoftWindowsServer:WindowsServer:2016-Datacenter:latest Win2016Datacenter latest +x64 WindowsServer MicrosoftWindowsServer 2012-R2-Datacenter MicrosoftWindowsServer:WindowsServer:2012-R2-Datacenter:latest Win2012R2Datacenter latest +x64 WindowsServer MicrosoftWindowsServer 2012-Datacenter MicrosoftWindowsServer:WindowsServer:2012-Datacenter:latest Win2012Datacenter latest +x64 WindowsServer MicrosoftWindowsServer 2008-R2-SP1 MicrosoftWindowsServer:WindowsServer:2008-R2-SP1:latest Win2008R2SP1 latest +``` + +A full list can be seen by adding the `--all` parameter. The image list can also be filtered by `--publisher` or `–-offer`. In this example, the list is filtered for all images, published by OpenLogic, with an offer that matches *0001-com-ubuntu-server-jammy*. + +```bash +az vm image list --offer 0001-com-ubuntu-server-jammy --publisher Canonical --all --output table +``` + +Example partial output: + +```output +Architecture Offer Publisher Sku Urn Version +-------------- --------------------------------- ----------- --------------- ------------------------------------------------------------------------ --------------- +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202204200 22.04.202204200 +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202205060 22.04.202205060 +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202205280 22.04.202205280 +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202206040 22.04.202206040 +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202206090 22.04.202206090 +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202206160 22.04.202206160 +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202206220 22.04.202206220 +x64 0001-com-ubuntu-server-jammy Canonical 22_04-lts Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202207060 22.04.202207060 +``` + +> [!NOTE] +> Canonical has changed the **Offer** names they use for the most recent versions. Before Ubuntu 20.04, the **Offer** name is UbuntuServer. For Ubuntu 20.04 the **Offer** name is `0001-com-ubuntu-server-focal` and for Ubuntu 22.04 it's `0001-com-ubuntu-server-jammy`. + +To deploy a VM using a specific image, take note of the value in the *Urn* column, which consists of the publisher, offer, SKU, and optionally a version number to [identify](cli-ps-findimage.md#terminology) the image. When specifying the image, the image version number can be replaced with `latest`, which selects the latest version of the distribution. In this example, the `--image` parameter is used to specify the latest version of a Ubuntu 22.04. + +```bash +export MY_VM2_NAME="myVM2$RANDOM_SUFFIX" +az vm create --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM2_NAME --image Canonical:0001-com-ubuntu-server-jammy:22_04-lts:latest --generate-ssh-keys +``` + +## Understand VM sizes + +A virtual machine size determines the amount of compute resources such as CPU, GPU, and memory that are made available to the virtual machine. Virtual machines need to be sized appropriately for the expected work load. If workload increases, an existing virtual machine can be resized. + +### VM Sizes + +The following table categorizes sizes into use cases. + +| Type | Description | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| [General purpose](../sizes-general.md) | Balanced CPU-to-memory. Ideal for dev / test and small to medium applications and data solutions. | +| [Compute optimized](../sizes-compute.md) | High CPU-to-memory. Good for medium traffic applications, network appliances, and batch processes. | +| [Memory optimized](../sizes-memory.md) | High memory-to-core. Great for relational databases, medium to large caches, and in-memory analytics. | +| [Storage optimized](../sizes-storage.md) | High disk throughput and IO. Ideal for Big Data, SQL, and NoSQL databases. | +| [GPU](../sizes-gpu.md) | Specialized VMs targeted for heavy graphic rendering and video editing. | +| [High performance](../sizes-hpc.md) | Our most powerful CPU VMs with optional high-throughput network interfaces (RDMA). | + +### Find available VM sizes + +To see a list of VM sizes available in a particular region, use the [az vm list-sizes](/cli/azure/vm) command. + +```bash +az vm list-sizes --location $REGION --output table +``` + +Example partial output: + +```output + MaxDataDiskCount MemoryInMb Name NumberOfCores OsDiskSizeInMb ResourceDiskSizeInMb +------------------ ------------ ---------------------- --------------- ---------------- ---------------------- +4 8192 Standard_D2ds_v4 2 1047552 76800 +8 16384 Standard_D4ds_v4 4 1047552 153600 +16 32768 Standard_D8ds_v4 8 1047552 307200 +32 65536 Standard_D16ds_v4 16 1047552 614400 +32 131072 Standard_D32ds_v4 32 1047552 1228800 +32 196608 Standard_D48ds_v4 48 1047552 1843200 +32 262144 Standard_D64ds_v4 64 1047552 2457600 +4 8192 Standard_D2ds_v5 2 1047552 76800 +8 16384 Standard_D4ds_v5 4 1047552 153600 +16 32768 Standard_D8ds_v5 8 1047552 307200 +32 65536 Standard_D16ds_v5 16 1047552 614400 +32 131072 Standard_D32ds_v5 32 1047552 1228800 +32 196608 Standard_D48ds_v5 48 1047552 1843200 +32 262144 Standard_D64ds_v5 64 1047552 2457600 +32 393216 Standard_D96ds_v5 96 1047552 3686400 +``` + +### Create VM with specific size + +In the previous VM creation example, a size was not provided, which results in a default size. A VM size can be selected at creation time using [az vm create](/cli/azure/vm) and the `--size` parameter. + +```bash +export MY_VM3_NAME="myVM3$RANDOM_SUFFIX" +az vm create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_VM3_NAME \ + --image SuseSles15SP5 \ + --size Standard_D2ds_v4 \ + --generate-ssh-keys +``` + +### Resize a VM + +After a VM has been deployed, it can be resized to increase or decrease resource allocation. You can view the current size of a VM with [az vm show](/cli/azure/vm): + +```bash +az vm show --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --query hardwareProfile.vmSize +``` + +Before resizing a VM, check if the desired size is available on the current Azure cluster. The [az vm list-vm-resize-options](/cli/azure/vm) command returns the list of sizes. + +```bash +az vm list-vm-resize-options --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --query [].name +``` + +If the desired size is available, the VM can be resized from a powered-on state, although it will be rebooted during the operation. Use the [az vm resize]( /cli/azure/vm) command to perform the resize. + +```bash +az vm resize --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --size Standard_D4s_v3 +``` + +If the desired size is not available on the current cluster, the VM needs to be deallocated before the resize operation can occur. Use the [az vm deallocate]( /cli/azure/vm) command to stop and deallocate the VM. Note that when the VM is powered back on, any data on the temporary disk may be removed. The public IP address also changes unless a static IP address is being used. Once deallocated, the resize can occur. + +After the resize, the VM can be started. + +```bash +az vm start --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME +``` + +## VM power states + +An Azure VM can have one of many power states. This state represents the current state of the VM from the standpoint of the hypervisor. + +### Power states + +| Power State | Description | +|-------------|-------------| +| Starting | Indicates the virtual machine is being started. | +| Running | Indicates that the virtual machine is running. | +| Stopping | Indicates that the virtual machine is being stopped. | +| Stopped | Indicates that the virtual machine is stopped. Virtual machines in the stopped state still incur compute charges. | +| Deallocating| Indicates that the virtual machine is being deallocated. | +| Deallocated | Indicates that the virtual machine is removed from the hypervisor but still available in the control plane. Virtual machines in the Deallocated state do not incur compute charges. | +| - | Indicates that the power state of the virtual machine is unknown. | + +### Find the power state + +To retrieve the state of a particular VM, use the [az vm get-instance-view](/cli/azure/vm) command. Be sure to specify a valid name for a virtual machine and resource group. + +```bash +az vm get-instance-view \ + --name $MY_VM_NAME \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --query instanceView.statuses[1] --output table +``` + +Output: + +```output +Code Level DisplayStatus +------------------ ------- --------------- +PowerState/running Info VM running +``` + +To retrieve the power state of all the VMs in your subscription, use the [Virtual Machines - List All API](/rest/api/compute/virtualmachines/listall) with parameter **statusOnly** set to *true*. + +## Management tasks + +During the life-cycle of a virtual machine, you may want to run management tasks such as starting, stopping, or deleting a virtual machine. Additionally, you may want to create scripts to automate repetitive or complex tasks. Using the Azure CLI, many common management tasks can be run from the command line or in scripts. + +### Get IP address + +This command returns the private and public IP addresses of a virtual machine. + +```bash +az vm list-ip-addresses --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --output table +``` + +### Stop virtual machine + +```bash +az vm stop --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME +``` + +### Start virtual machine + +```bash +az vm start --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME +``` + +### Deleting VM resources + +Depending on how you delete a VM, it may only delete the VM resource, not the networking and disk resources. You can change the default behavior to delete other resources when you delete the VM. For more information, see [Delete a VM and attached resources](../delete.md). + +Deleting a resource group also deletes all resources in the resource group, like the VM, virtual network, and disk. The `--no-wait` parameter returns control to the prompt without waiting for the operation to complete. The `--yes` parameter confirms that you wish to delete the resources without an additional prompt to do so. + +## Next steps + +In this tutorial, you learned about basic VM creation and management such as how to: + +> [!div class="checklist"] +> * Create and connect to a VM +> * Select and use VM images +> * View and use specific VM sizes +> * Resize a VM +> * View and understand VM state + +Advance to the next tutorial to learn about VM disks. + +> [!div class="nextstepaction"] +> [Create and Manage VM disks](./tutorial-manage-disks.md) \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 8ec5d5342..7cc74619e 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -159,11 +159,11 @@ }, { "status": "active", - "key": "azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", + "key": "azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", "title": "Create virtual machines in a Flexible scale set using Azure CLI", "description": "Learn how to create a Virtual Machine Scale Set in Flexible orchestration mode using Azure CLI.", "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli", "nextSteps": [ { @@ -184,7 +184,7 @@ }, { "status": "active", - "key": "azure-docs/articles/virtual-machines/linux/quick-create-cli.md", + "key": "azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md", "title": "Deploy a Linux virtual machine", "description": "In this quickstart, you learn how to use the Azure CLI to create a Linux virtual machine", "stackDetails": [ @@ -193,7 +193,7 @@ "Network interface with public IP and network security group", "Port 22 will be opened" ], - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/virtual-machines/linux/quick-create-cli.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/quick-create-cli", "nextSteps": [ { @@ -230,11 +230,11 @@ }, { "status": "active", - "key": "azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", + "key": "azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", "title": "Tutorial - Deploy a LEMP stack using WordPress on a VM", "description": "In this tutorial, you learn how to install the LEMP stack, and WordPress, on a Linux virtual machine in Azure.", "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-lemp-stack", "nextSteps": [ { @@ -763,7 +763,7 @@ }, { "status": "active", - "key": "azure-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md", + "key": "azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md", "title": "Tutorial - Use a custom VM image in a scale set with Azure CLI", "description": "Learn how to use the Azure CLI to create a custom VM image that you can use to deploy a Virtual Machine Scale Set", "stackDetails": [], @@ -1015,5 +1015,77 @@ ], "configurations": { } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md", + "title": "Tutorial - Create and manage Linux VMs with the Azure CLI", + "description": "In this tutorial, you learn how to use the Azure CLI to create and manage Linux VMs in Azure", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-manage-vm", + "nextSteps": [ + { + "title": "Create and Manage VM Disks", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-manage-disks" + } + ], + "configurations": { + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md", + "title": "Tutorial - Autoscale a scale set with the Azure CLI", + "description": "Learn how to use the Azure CLI to automatically scale a Virtual Machine Scale Set as CPU demands increases and decreases", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-autoscale-cli?tabs=Ubuntu", + "nextSteps": [ + ], + "configurations": { + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md", + "title": "Modify an Azure Virtual Machine Scale Set using Azure CLI", + "description": "Learn how to modify and update an Azure Virtual Machine Scale Set using Azure CLI", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli", + "nextSteps": [ + { + "title": "Use data disks with scale sets", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-use-disks-powershell" + } + ], + "configurations": { + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/disks-enable-performance.md", + "title": "Preview - Increase performance of Premium SSDs and Standard SSD/HDDs", + "description": "Increase the performance of Azure Premium SSDs and Standard SSD/HDDs using performance plus.", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/disks-enable-performance?tabs=azure-cli", + "nextSteps": [ + { + "title": "Create an incremental snapshot for managed disks", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/disks-incremental-snapshots" + }, + { + "title": "Expand virtual hard disks on a Linux VM", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/expand-disks" + } + ], + "configurations": { + } } ] diff --git a/tools/ada.py b/tools/ada.py index 03f2477a4..29e75e161 100644 --- a/tools/ada.py +++ b/tools/ada.py @@ -102,7 +102,7 @@ 7. Ensure that the Exec Doc does not require any user interaction during its execution. The document should not include any commands or scripts that prompt the user for input or expect interaction with the terminal. All inputs must be predefined and handled automatically within the script. -7. Appropriately add metadata at the start of the Exec Doc. Here are some mandatory fields: +8. Appropriately add metadata at the start of the Exec Doc. Here are some mandatory fields: - title = the title of the Exec Doc - description = the description of the Exec Doc @@ -126,13 +126,13 @@ --- ``` -7. Ensure the environment variable names are not placeholders i.e. <> but have a certain generic, useful name. For the location/region parameter, default to "WestUS2" or "centralindia". Additionally, appropriately add descriptions below every section explaining what is happening in that section in crisp but necessary detail so that the user can learn as they go. +9. Ensure the environment variable names are not placeholders i.e. <> but have a certain generic, useful name. For the location/region parameter, default to "WestUS2" or "centralindia". Additionally, appropriately add descriptions below every section explaining what is happening in that section in crisp but necessary detail so that the user can learn as they go. -8. Don't start and end your answer with ``` backticks!!! Don't add backticks to the metadata at the top!!!. +10. Don't start and end your answer with ``` backticks!!! Don't add backticks to the metadata at the top!!!. -8. Ensure that any info, literally any info whether it is a comment, tag, description, etc., which is not within a code block remains unchanged. Preserve ALL details of the doc. +11. Ensure that any info, literally any info whether it is a comment, tag, description, etc., which is not within a code block remains unchanged. Preserve ALL details of the doc. -8. Environment variables are dynamic values that store configuration settings, system paths, and other information that can be accessed throughout a doc. By using environment variables, you can separate configuration details from the code, making it easier to manage and deploy applications in an environment like Exec Docs. +12. Environment variables are dynamic values that store configuration settings, system paths, and other information that can be accessed throughout a doc. By using environment variables, you can separate configuration details from the code, making it easier to manage and deploy applications in an environment like Exec Docs. Declare environment variables _as they are being used_ in the Exec Doc using the export command. This is a best practice to ensure that the variables are accessible throughout the doc. @@ -170,7 +170,7 @@ >**Note:** Don't have any spaces around the equal sign when declaring environment variables. -9. A major component of Exec Docs is automated infrastructure deployment on the cloud. While testing the doc, if you do not update relevant environment variable names, the doc will fail when run/executed more than once as the resource group or other resources will already exist from the previous runs. +13. A major component of Exec Docs is automated infrastructure deployment on the cloud. While testing the doc, if you do not update relevant environment variable names, the doc will fail when run/executed more than once as the resource group or other resources will already exist from the previous runs. Add a random suffix at the end of _relevant_ environment variable(s). The example below shows how this would work when you are creating a resource group. @@ -186,7 +186,7 @@ >**Note:** You can generate your own random suffix or use the one provided in the example above. The `openssl rand -hex 3` command generates a random 3-character hexadecimal string. This string is then appended to the resource group name to ensure that the resource group name is unique for each deployment. -10. In Exec Docs, result blocks are distinguished by a custom expected_similarity comment tag followed by a code block. These result blocks indicate to Innovation Engine what the minimum degree of similarity should be between the actual and the expected output of a code block (one which returns something in the terminal that is relevant to benchmark against). Learn More: [Result Blocks](https://github.com/Azure/InnovationEngine/blob/main/README.md#result-blocks). +14. In Exec Docs, result blocks are distinguished by a custom expected_similarity comment tag followed by a code block. These result blocks indicate to Innovation Engine what the minimum degree of similarity should be between the actual and the expected output of a code block (one which returns something in the terminal that is relevant to benchmark against). Learn More: [Result Blocks](https://github.com/Azure/InnovationEngine/blob/main/README.md#result-blocks). Add result block(s) below code block(s) that you would want Innovation Engine to verify i.e. code block(s) which produce an output in the terminal that is relevant to benchmark against. Follow these steps when adding a result block below a code block for the first time: @@ -227,7 +227,7 @@ >**Note:** Result blocks are not required but recommended for commands that return some output in the terminal. They help Innovation Engine verify the output of a command and act as checkpoints to ensure that the doc is moving in the right direction. -11. Redacting PII from the output helps protect sensitive information from being inadvertently shared or exposed. This is crucial for maintaining privacy, complying with data protection regulations, and furthering the company's security posture. +15. Redacting PII from the output helps protect sensitive information from being inadvertently shared or exposed. This is crucial for maintaining privacy, complying with data protection regulations, and furthering the company's security posture. Ensure result block(s) have all the PII (Personally Identifiable Information) stricken out from them and replaced with x’s. @@ -257,7 +257,7 @@ >**Note:** Here are some examples of PII in result blocks: Unique identifiers for resources, Email Addresses, Phone Numbers, IP Addresses, Credit Card Numbers, Social Security Numbers (SSNs), Usernames, Resource Names, Subscription IDs, Resource Group Names, Tenant IDs, Service Principal Names, Client IDs, Secrets and Keys. -12. If you are converting an existing Azure Doc to an Exec Doc and if the existing doc contains a "Delete Resources" (or equivalent section) comprising resource/other deletion command(s), remove the code blocks in that section or remove that section entirely +16. If you are converting an existing Azure Doc to an Exec Doc and if the existing doc contains a "Delete Resources" (or equivalent section) comprising resource/other deletion command(s), remove the code blocks in that section or remove that section entirely >**Note:** We remove commands from this section ***only*** in Exec Docs. This is because Innovation Engine executes all relevant command(s) that it encounters, inlcuding deleting the resources. That would be counterproductive to automated deployment of cloud infrastructure @@ -320,19 +320,46 @@ def log_data_to_csv(data): writer.writerow(data) def main(): - print("\nWelcome to ADA - AI Documentation Assistant!\n") - print("This tool helps you write and troubleshoot Executable Documents efficiently!\n") - - user_input = input("Please enter the path to your markdown file for conversion or describe your intended workload: ") - - if os.path.isfile(user_input) and user_input.endswith('.md'): - input_type = 'file' - with open(user_input, "r", encoding='latin-1') as f: - input_content = f.read() - input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" + print("\nWelcome to ADA - AI Documentation Assistant!") + print("\nThis tool helps you write and troubleshoot Executable Documents efficiently!") + print("\nPlease select one of the following options:") + print(" 1. Enter path to markdown file for conversion") + print(" 2. Describe workload for new Exec Doc") + print(" 3. Generate description for shell script") + print(" 4. Redact PII from an existing Exec Doc") + choice = input("Enter the number corresponding to your choice: ") + + if choice == "1": + user_input = input("Enter the path to your markdown file: ") + if os.path.isfile(user_input) and user_input.endswith('.md'): + input_type = 'file' + with open(user_input, "r") as f: + input_content = f.read() + input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" + else: + print("Invalid file path or file type. Please provide a valid markdown file.") + sys.exit(1) + elif choice == "2": + user_input = input("Describe your workload for the new Exec Doc: ") + if os.path.isfile(user_input): + input_type = 'workload_description' + input_content = user_input + elif choice == "3": + user_input = input("Enter the path to your shell script (provide context and details): ") + elif choice == "4": + user_input = input("Enter the path to your Exec Doc for PII redaction: ") else: - input_type = 'workload_description' - input_content = user_input + print("Invalid choice. Exiting.") + sys.exit(1) + + # if os.path.isfile(user_input) and user_input.endswith('.md'): + # input_type = 'file' + # with open(user_input, "r") as f: + # input_content = f.read() + # input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" + # else: + # input_type = 'workload_description' + # input_content = user_input install_innovation_engine() diff --git a/tools/converted_doc.md b/tools/converted_doc.md deleted file mode 100644 index ea6b37499..000000000 --- a/tools/converted_doc.md +++ /dev/null @@ -1,247 +0,0 @@ ---- -title: 'Quickstart: Use the Azure CLI to create a Batch account and run a job' -description: Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool. -ms.topic: quickstart -ms.date: 04/12/2023 -ms.custom: mvc, devx-track-azurecli, mode-api, linux-related-content, innovation-engine -author: (preserved) -ms.author: (preserved) ---- - -# Quickstart: Use the Azure CLI to create a Batch account and run a job - -This quickstart shows you how to get started with Azure Batch by using Azure CLI commands and scripts to create and manage Batch resources. You create a Batch account that has a pool of virtual machines, or compute nodes. You then create and run a job with tasks that run on the pool nodes. - -After you complete this quickstart, you understand the [key concepts of the Batch service](batch-service-workflow-features.md) and are ready to use Batch with more realistic, larger scale workloads. - -## Prerequisites - -- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] - -- Azure Cloud Shell or Azure CLI. - - You can run the Azure CLI commands in this quickstart interactively in Azure Cloud Shell. To run the commands in the Cloud Shell, select **Open Cloudshell** at the upper-right corner of a code block. Select **Copy** to copy the code, and paste it into Cloud Shell to run it. You can also [run Cloud Shell from within the Azure portal](https://shell.azure.com). Cloud Shell always uses the latest version of the Azure CLI. - - Alternatively, you can [install Azure CLI locally](/cli/azure/install-azure-cli) to run the commands. The steps in this article require Azure CLI version 2.0.20 or later. Run [az version](/cli/azure/reference-index?#az-version) to see your installed version and dependent libraries, and run [az upgrade](/cli/azure/reference-index?#az-upgrade) to upgrade. If you use a local installation, sign in to Azure by using the appropriate command. - ->[!NOTE] ->For some regions and subscription types, quota restrictions might cause Batch account or node creation to fail or not complete. In this situation, you can request a quota increase at no charge. For more information, see [Batch service quotas and limits](batch-quota-limit.md). - -## Create a resource group - -Run the following [az group create](/cli/azure/group#az-group-create) command to create an Azure resource group. The resource group is a logical container that holds the Azure resources for this quickstart. - -```azurecli-interactive -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export REGION="canadacentral" -export RESOURCE_GROUP="qsBatch$RANDOM_SUFFIX" - -az group create \ - --name $RESOURCE_GROUP \ - --location $REGION -``` - -Results: - - - -```JSON -{ - "id": "/subscriptions/xxxxx/resourceGroups/qsBatchxxx", - "location": "eastus2", - "managedBy": null, - "name": "qsBatchxxx", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" -} -``` - -## Create a storage account - -Use the [az storage account create](/cli/azure/storage/account#az-storage-account-create) command to create an Azure Storage account to link to your Batch account. Although this quickstart doesn't use the storage account, most real-world Batch workloads use a linked storage account to deploy applications and store input and output data. - -Run the following command to create a Standard_LRS SKU storage account in your resource group: - -```azurecli-interactive -export STORAGE_ACCOUNT="mybatchstorage$RANDOM_SUFFIX" - -az storage account create \ - --resource-group $RESOURCE_GROUP \ - --name $STORAGE_ACCOUNT \ - --location $REGION \ - --sku Standard_LRS -``` - -## Create a Batch account - -Run the following [az batch account create](/cli/azure/batch/account#az-batch-account-create) command to create a Batch account in your resource group and link it with the storage account. - -```azurecli-interactive -export BATCH_ACCOUNT="mybatchaccount$RANDOM_SUFFIX" - -az batch account create \ - --name $BATCH_ACCOUNT \ - --storage-account $STORAGE_ACCOUNT \ - --resource-group $RESOURCE_GROUP \ - --location $REGION -``` - -Sign in to the new Batch account by running the [az batch account login](/cli/azure/batch/account#az-batch-account-login) command. Once you authenticate your account with Batch, subsequent `az batch` commands in this session use this account context. - -```azurecli-interactive -az batch account login \ - --name $BATCH_ACCOUNT \ - --resource-group $RESOURCE_GROUP \ - --shared-key-auth -``` - -## Create a pool of compute nodes - -Run the [az batch pool create](/cli/azure/batch/pool#az-batch-pool-create) command to create a pool of Linux compute nodes in your Batch account. The following example creates a pool that consists of two Standard_A1_v2 size VMs running Ubuntu 20.04 LTS OS. This node size offers a good balance of performance versus cost for this quickstart example. - -```azurecli-interactive -export POOL_ID="myPool$RANDOM_SUFFIX" - -az batch pool create \ - --id $POOL_ID \ - --image canonical:0001-com-ubuntu-server-focal:20_04-lts \ - --node-agent-sku-id "batch.node.ubuntu 20.04" \ - --target-dedicated-nodes 2 \ - --vm-size Standard_A1_v2 -``` - -Batch creates the pool immediately, but takes a few minutes to allocate and start the compute nodes. To see the pool status, use the [az batch pool show](/cli/azure/batch/pool#az-batch-pool-show) command. This command shows all the properties of the pool, and you can query for specific properties. The following command queries for the pool allocation state: - -```azurecli-interactive -az batch pool show --pool-id $POOL_ID \ - --query "{allocationState: allocationState}" -``` - -Results: - - - -```JSON -{ - "allocationState": "resizing" -} -``` - -While Batch allocates and starts the nodes, the pool is in the `resizing` state. You can create a job and tasks while the pool state is still `resizing`. The pool is ready to run tasks when the allocation state is `steady` and all the nodes are running. - -## Create a job - -Use the [az batch job create](/cli/azure/batch/job#az-batch-job-create) command to create a Batch job to run on your pool. A Batch job is a logical group of one or more tasks. The job includes settings common to the tasks, such as the pool to run on. The following example creates a job that initially has no tasks. - -```azurecli-interactive -export JOB_ID="myJob$RANDOM_SUFFIX" - -az batch job create \ - --id $JOB_ID \ - --pool-id $POOL_ID -``` - -## Create job tasks - -Batch provides several ways to deploy apps and scripts to compute nodes. Use the [az batch task create](/cli/azure/batch/task#az-batch-task-create) command to create tasks to run in the job. Each task has a command line that specifies an app or script. - -The following Bash script creates four identical, parallel tasks called `myTask1` through `myTask4`. The task command line displays the Batch environment variables on the compute node, and then waits 90 seconds. - -```azurecli-interactive -for i in {1..4} -do - az batch task create \ - --task-id myTask$i \ - --job-id $JOB_ID \ - --command-line "/bin/bash -c 'printenv | grep AZ_BATCH; sleep 90s'" -done -``` - -Batch distributes the tasks to the compute nodes. - -## View task status - -After you create the tasks, Batch queues them to run on the pool. Once a node is available, a task runs on the node. - -Use the [az batch task show](/cli/azure/batch/task#az-batch-task-show) command to view the status of Batch tasks. The following example shows details about the status of `myTask1`: - -```azurecli-interactive -az batch task show \ - --job-id $JOB_ID \ - --task-id myTask1 -``` - -The command output includes many details. For example, an `exitCode` of `0` indicates that the task command completed successfully. The `nodeId` shows the name of the pool node that ran the task. - -## View task output - -Use the [az batch task file list](/cli/azure/batch/task#az-batch-task-file-show) command to list the files a task created on a node. The following command lists the files that `myTask1` created: - -```azurecli-interactive -# Wait for task to complete before downloading output -echo "Waiting for task to complete..." -while true; do - STATUS=$(az batch task show --job-id $JOB_ID --task-id myTask1 --query "state" -o tsv) - if [ "$STATUS" == "running" ]; then - break - fi - sleep 10 -done - -az batch task file list --job-id $JOB_ID --task-id myTask1 --output table -``` - -Results are similar to the following output: - -Results: - - - -```output -Name URL Is Directory Content Length ----------- ---------------------------------------------------------------------------------------- -------------- ---------------- -stdout.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stdout.txt False 695 -certs https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/certs True -wd https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/wd True -stderr.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stderr.txt False 0 -``` - -The [az batch task file download](/cli/azure/batch/task#az-batch-task-file-download) command downloads output files to a local directory. Run the following example to download the *stdout.txt* file: - -```azurecli-interactive -az batch task file download \ - --job-id $JOB_ID \ - --task-id myTask1 \ - --file-path stdout.txt \ - --destination ./stdout.txt -``` - -You can view the contents of the standard output file in a text editor. The following example shows a typical *stdout.txt* file. The standard output from this task shows the Azure Batch environment variables that are set on the node. You can refer to these environment variables in your Batch job task command lines, and in the apps and scripts the command lines run. - -```text -AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1 -AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup -AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/certs -AZ_BATCH_ACCOUNT_URL=https://mybatchaccount.eastus2.batch.azure.com/ -AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/wd -AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared -AZ_BATCH_TASK_USER=_azbatch -AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks -AZ_BATCH_JOB_ID=myJob -AZ_BATCH_NODE_IS_DEDICATED=true -AZ_BATCH_NODE_ID=tvm-257509324_2-20180703t215033z -AZ_BATCH_POOL_ID=myPool -AZ_BATCH_TASK_ID=myTask1 -AZ_BATCH_ACCOUNT_NAME=mybatchaccount -AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin -``` - -## Next steps - -In this quickstart, you created a Batch account and pool, created and ran a Batch job and tasks, and viewed task output from the nodes. Now that you understand the key concepts of the Batch service, you're ready to use Batch with more realistic, larger scale workloads. To learn more about Azure Batch, continue to the Azure Batch tutorials. - -> [!div class="nextstepaction"] -> [Tutorial: Run a parallel workload with Azure Batch](./tutorial-parallel-python.md) \ No newline at end of file diff --git a/tools/doc.md b/tools/doc.md deleted file mode 100644 index dcc7b4b61..000000000 --- a/tools/doc.md +++ /dev/null @@ -1,211 +0,0 @@ ---- -title: 'Quickstart: Use the Azure CLI to create a Batch account and run a job' -description: Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool. -ms.topic: quickstart -ms.date: 04/12/2023 -ms.custom: mvc, devx-track-azurecli, mode-api, linux-related-content ---- - -# Quickstart: Use the Azure CLI to create a Batch account and run a job - -This quickstart shows you how to get started with Azure Batch by using Azure CLI commands and scripts to create and manage Batch resources. You create a Batch account that has a pool of virtual machines, or compute nodes. You then create and run a job with tasks that run on the pool nodes. - -After you complete this quickstart, you understand the [key concepts of the Batch service](batch-service-workflow-features.md) and are ready to use Batch with more realistic, larger scale workloads. - -## Prerequisites - -- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] - -- Azure Cloud Shell or Azure CLI. - - You can run the Azure CLI commands in this quickstart interactively in Azure Cloud Shell. To run the commands in the Cloud Shell, select **Open Cloudshell** at the upper-right corner of a code block. Select **Copy** to copy the code, and paste it into Cloud Shell to run it. You can also [run Cloud Shell from within the Azure portal](https://shell.azure.com). Cloud Shell always uses the latest version of the Azure CLI. - - Alternatively, you can [install Azure CLI locally](/cli/azure/install-azure-cli) to run the commands. The steps in this article require Azure CLI version 2.0.20 or later. Run [az version](/cli/azure/reference-index?#az-version) to see your installed version and dependent libraries, and run [az upgrade](/cli/azure/reference-index?#az-upgrade) to upgrade. If you use a local installation, sign in to Azure by using the [az login](/cli/azure/reference-index#az-login) command. - ->[!NOTE] ->For some regions and subscription types, quota restrictions might cause Batch account or node creation to fail or not complete. In this situation, you can request a quota increase at no charge. For more information, see [Batch service quotas and limits](batch-quota-limit.md). - -## Create a resource group - -Run the following [az group create](/cli/azure/group#az-group-create) command to create an Azure resource group named `qsBatch` in the `eastus2` Azure region. The resource group is a logical container that holds the Azure resources for this quickstart. - -```azurecli-interactive -az group create \ - --name qsBatch \ - --location eastus2 -``` - -## Create a storage account - -Use the [az storage account create](/cli/azure/storage/account#az-storage-account-create) command to create an Azure Storage account to link to your Batch account. Although this quickstart doesn't use the storage account, most real-world Batch workloads use a linked storage account to deploy applications and store input and output data. - -Run the following command to create a Standard_LRS SKU storage account named `mybatchstorage` in your resource group: - -```azurecli-interactive -az storage account create \ - --resource-group qsBatch \ - --name mybatchstorage \ - --location eastus2 \ - --sku Standard_LRS -``` - -## Create a Batch account - -Run the following [az batch account create](/cli/azure/batch/account#az-batch-account-create) command to create a Batch account named `mybatchaccount` in your resource group and link it with the `mybatchstorage` storage account. - -```azurecli-interactive -az batch account create \ - --name mybatchaccount \ - --storage-account mybatchstorage \ - --resource-group qsBatch \ - --location eastus2 -``` - -Sign in to the new Batch account by running the [az batch account login](/cli/azure/batch/account#az-batch-account-login) command. Once you authenticate your account with Batch, subsequent `az batch` commands in this session use this account context. - -```azurecli-interactive -az batch account login \ - --name mybatchaccount \ - --resource-group qsBatch \ - --shared-key-auth -``` - -## Create a pool of compute nodes - -Run the [az batch pool create](/cli/azure/batch/pool#az-batch-pool-create) command to create a pool of Linux compute nodes in your Batch account. The following example creates a pool named `myPool` that consists of two Standard_A1_v2 size VMs running Ubuntu 20.04 LTS OS. This node size offers a good balance of performance versus cost for this quickstart example. - -```azurecli-interactive -az batch pool create \ - --id myPool \ - --image canonical:0001-com-ubuntu-server-focal:20_04-lts \ - --node-agent-sku-id "batch.node.ubuntu 20.04" \ - --target-dedicated-nodes 2 \ - --vm-size Standard_A1_v2 -``` - -Batch creates the pool immediately, but takes a few minutes to allocate and start the compute nodes. To see the pool status, use the [az batch pool show](/cli/azure/batch/pool#az-batch-pool-show) command. This command shows all the properties of the pool, and you can query for specific properties. The following command queries for the pool allocation state: - -```azurecli-interactive -az batch pool show --pool-id myPool \ - --query "allocationState" -``` - -While Batch allocates and starts the nodes, the pool is in the `resizing` state. You can create a job and tasks while the pool state is still `resizing`. The pool is ready to run tasks when the allocation state is `steady` and all the nodes are running. - -## Create a job - -Use the [az batch job create](/cli/azure/batch/job#az-batch-job-create) command to create a Batch job to run on your pool. A Batch job is a logical group of one or more tasks. The job includes settings common to the tasks, such as the pool to run on. The following example creates a job called `myJob` on `myPool` that initially has no tasks. - -```azurecli-interactive -az batch job create \ - --id myJob \ - --pool-id myPool -``` - -## Create job tasks - -Batch provides several ways to deploy apps and scripts to compute nodes. Use the [az batch task create](/cli/azure/batch/task#az-batch-task-create) command to create tasks to run in the job. Each task has a command line that specifies an app or script. - -The following Bash script creates four identical, parallel tasks called `myTask1` through `myTask4`. The task command line displays the Batch environment variables on the compute node, and then waits 90 seconds. - -```azurecli-interactive -for i in {1..4} -do - az batch task create \ - --task-id myTask$i \ - --job-id myJob \ - --command-line "/bin/bash -c 'printenv | grep AZ_BATCH; sleep 90s'" -done -``` - -The command output shows the settings for each task. Batch distributes the tasks to the compute nodes. - -## View task status - -After you create the task, Batch queues the task to run on the pool. Once a node is available, the task runs on the node. - -Use the [az batch task show](/cli/azure/batch/task#az-batch-task-show) command to view the status of Batch tasks. The following example shows details about the status of `myTask1`: - -```azurecli-interactive -az batch task show \ - --job-id myJob \ - --task-id myTask1 -``` - -The command output includes many details. For example, an `exitCode` of `0` indicates that the task command completed successfully. The `nodeId` shows the name of the pool node that ran the task. - -## View task output - -Use the [az batch task file list](/cli/azure/batch/task#az-batch-task-file-show) command to list the files a task created on a node. The following command lists the files that `myTask1` created: - -```azurecli-interactive -az batch task file list \ - --job-id myJob \ - --task-id myTask1 \ - --output table -``` - -Results are similar to the following output: - -```output -Name URL Is Directory Content Length ----------- ---------------------------------------------------------------------------------------- -------------- ---------------- -stdout.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stdout.txt False 695 -certs https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/certs True -wd https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/wd True -stderr.txt https://mybatchaccount.eastus2.batch.azure.com/jobs/myJob/tasks/myTask1/files/stderr.txt False 0 - -``` - -The [az batch task file download](/cli/azure/batch/task#az-batch-task-file-download) command downloads output files to a local directory. Run the following example to download the *stdout.txt* file: - -```azurecli-interactive -az batch task file download \ - --job-id myJob \ - --task-id myTask1 \ - --file-path stdout.txt \ - --destination ./stdout.txt -``` - -You can view the contents of the standard output file in a text editor. The following example shows a typical *stdout.txt* file. The standard output from this task shows the Azure Batch environment variables that are set on the node. You can refer to these environment variables in your Batch job task command lines, and in the apps and scripts the command lines run. - -```text -AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1 -AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup -AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/certs -AZ_BATCH_ACCOUNT_URL=https://mybatchaccount.eastus2.batch.azure.com/ -AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJob/job-1/myTask1/wd -AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared -AZ_BATCH_TASK_USER=_azbatch -AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks -AZ_BATCH_JOB_ID=myJob -AZ_BATCH_NODE_IS_DEDICATED=true -AZ_BATCH_NODE_ID=tvm-257509324_2-20180703t215033z -AZ_BATCH_POOL_ID=myPool -AZ_BATCH_TASK_ID=myTask1 -AZ_BATCH_ACCOUNT_NAME=mybatchaccount -AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin -``` - -## Clean up resources - -If you want to continue with Batch tutorials and samples, you can use the Batch account and linked storage account that you created in this quickstart. There's no charge for the Batch account itself. - -Pools and nodes incur charges while the nodes are running, even if they aren't running jobs. When you no longer need a pool, use the [az batch pool delete](/cli/azure/batch/pool#az-batch-pool-delete) command to delete it. Deleting a pool deletes all task output on the nodes, and the nodes themselves. - -```azurecli-interactive -az batch pool delete --pool-id myPool -``` - -When you no longer need any of the resources you created for this quickstart, you can use the [az group delete](/cli/azure/group#az-group-delete) command to delete the resource group and all its resources. To delete the resource group and the storage account, Batch account, node pools, and all related resources, run the following command: - -```azurecli-interactive -az group delete --name qsBatch -``` - -## Next steps - -In this quickstart, you created a Batch account and pool, created and ran a Batch job and tasks, and viewed task output from the nodes. Now that you understand the key concepts of the Batch service, you're ready to use Batch with more realistic, larger scale workloads. To learn more about Azure Batch, continue to the Azure Batch tutorials. - -> [!div class="nextstepaction"] -> [Tutorial: Run a parallel workload with Azure Batch](./tutorial-parallel-python.md) \ No newline at end of file diff --git a/tools/execution_log.csv b/tools/execution_log.csv index c4bb91b1a..d54b48d7a 100644 --- a/tools/execution_log.csv +++ b/tools/execution_log.csv @@ -185,3 +185,38 @@ Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{" ' StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_OiVkDXNnyLGZHtL9M3XLZKPNnaMATBx6' is not valid according to the validation procedure. The tracking id is '26b03af6-8be7-4272-a5a5-6c59aad9b563'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}}",957.5963819026947,Failure 2025-02-28 00:31:26,file,doc1.md,converted_doc1.md,0,,91.56127834320068,Success +2025-03-03 21:35:51,file,doc2.md,converted_doc2.md,11,"time=2025-03-03T20:03:34-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""SuseSles15SP3"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. +' +StdErr: ERROR: Invalid image ""SuseSles15SP3"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. + + time=2025-03-03T20:07:31-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. +Error: command exited with 'exit status 255' and the message 'Pseudo-terminal will not be allocated because stdin is not a terminal. +ssh: connect to host 52.174.34.95 port 22: Connection timed out +' +StdErr: Pseudo-terminal will not be allocated because stdin is not a terminal. +ssh: connect to host 52.174.34.95 port 22: Connection timed out + + The 'ie test' command timed out after 11 minutes. + + The 'ie test' command timed out after 11 minutes. + + The 'ie test' command timed out after 11 minutes. + + The 'ie test' command timed out after 11 minutes. + + The 'ie test' command timed out after 11 minutes. + + The 'ie test' command timed out after 11 minutes. + + time=2025-03-03T21:23:19-08:00 level=error msg=Error testing scenario: failed to execute code block 2 on step 7. +Error: %!s() +StdErr: + + time=2025-03-03T21:24:06-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 0. +Error: %!s() +StdErr: + + The 'ie test' command timed out after 11 minutes.",5596.252681970596,Failure diff --git a/tools/stdout.txt b/tools/stdout.txt deleted file mode 100644 index d606b3d0c..000000000 --- a/tools/stdout.txt +++ /dev/null @@ -1,20 +0,0 @@ -AZ_BATCH_NODE_MOUNTS_DIR=/mnt/batch/tasks/fsmounts -AZ_BATCH_TASK_WORKING_DIR=/mnt/batch/tasks/workitems/myJobadb33d/job-1/myTask1/wd -AZ_BATCH_TASK_DIR=/mnt/batch/tasks/workitems/myJobadb33d/job-1/myTask1 -AZ_BATCH_NODE_SHARED_DIR=/mnt/batch/tasks/shared -AZ_BATCH_TASK_USER=_azbatch -AZ_BATCH_NODE_IS_DEDICATED=true -AZ_BATCH_NODE_STARTUP_DIR=/mnt/batch/tasks/startup -AZ_BATCH_JOB_ID=myJobadb33d -AZ_BATCH_NODE_STARTUP_WORKING_DIR=/mnt/batch/tasks/startup/wd -AZ_BATCH_TASK_ID=myTask1 -AZ_BATCH_ACCOUNT_NAME=mybatchaccountadb33d -AZ_BATCH_RESERVED_EPHEMERAL_DISK_SPACE_BYTES=1000000000 -AZ_BATCH_NODE_ROOT_DIR=/mnt/batch/tasks -AZ_BATCH_POOL_ID=myPooladb33d -AZ_BATCH_RESERVED_DISK_SPACE_BYTES=1000000000 -AZ_BATCH_ACCOUNT_URL=https://mybatchaccountadb33d.canadacentral.batch.azure.com/ -AZ_BATCH_NODE_ID=tvmps_1b25c614520a9192d5e81007e1880adf7012f74bc13ba2733718a8d77878cc5b_d -AZ_BATCH_TASK_USER_IDENTITY=PoolNonAdmin -AZ_BATCH_OS_RESERVED_EPHEMERAL_DISK_SPACE_BYTES=1000000000 -AZ_BATCH_CERTIFICATES_DIR=/mnt/batch/tasks/workitems/myJobadb33d/job-1/myTask1/certs From 11aba81738205f3e1548956903a9e7a3dba18353 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Tue, 4 Mar 2025 00:15:17 -0800 Subject: [PATCH 08/87] added toms vnet doc --- .../aks/learn/aks-store-quickstart.yaml | 0 .../aks/learn/quick-kubernetes-deploy-cli.md | 0 .../container-instances-vnet.md | 402 ++++++++++++++++++ scenarios/metadata.json | 24 +- 4 files changed, 425 insertions(+), 1 deletion(-) rename scenarios/{azure-docs => azure-aks-docs}/articles/aks/learn/aks-store-quickstart.yaml (100%) rename scenarios/{azure-docs => azure-aks-docs}/articles/aks/learn/quick-kubernetes-deploy-cli.md (100%) create mode 100644 scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md diff --git a/scenarios/azure-docs/articles/aks/learn/aks-store-quickstart.yaml b/scenarios/azure-aks-docs/articles/aks/learn/aks-store-quickstart.yaml similarity index 100% rename from scenarios/azure-docs/articles/aks/learn/aks-store-quickstart.yaml rename to scenarios/azure-aks-docs/articles/aks/learn/aks-store-quickstart.yaml diff --git a/scenarios/azure-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md b/scenarios/azure-aks-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md similarity index 100% rename from scenarios/azure-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md rename to scenarios/azure-aks-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md new file mode 100644 index 000000000..06ca507fd --- /dev/null +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md @@ -0,0 +1,402 @@ +--- +title: Deploy container group to Azure virtual network +description: Learn how to deploy a container group to a new or existing Azure virtual network via the Azure CLI. +ms.topic: how-to +ms.author: tomcassidy +author: tomvcassidy +ms.service: azure-container-instances +services: container-instances +ms.date: 09/09/2024 +ms.custom: devx-track-azurecli, innovation-engine +--- + +# Deploy container instances into an Azure virtual network + +[Azure Virtual Network](/azure/virtual-network/virtual-networks-overview) provides secure, private networking for your Azure and on-premises resources. By deploying container groups into an Azure virtual network, your containers can communicate securely with other resources in the virtual network. + +This article shows how to use the [az container create][az-container-create] command in the Azure CLI to deploy container groups to either a new virtual network or an existing virtual network. + +> [!IMPORTANT] +> * Subnets must be delegated before using a virtual network +> * Before deploying container groups in virtual networks, we suggest checking the limitation first. For networking scenarios and limitations, see [Virtual network scenarios and resources for Azure Container Instances](container-instances-virtual-network-concepts.md). +> * Container group deployment to a virtual network is generally available for Linux and Windows containers, in most regions where Azure Container Instances is available. For details, see [available-regions][available-regions]. + +[!INCLUDE [network profile callout](./includes/network-profile-callout.md)] + +Examples in this article are formatted for the Bash shell. If you prefer another shell such as PowerShell or Command Prompt, adjust the line continuation characters accordingly. + +## Prerequisites + +### Define environment variables + +The automated deployment pathway uses the following environment variables and resource names throughout this guide. Users proceeding through the guide manually can use their own variables and names as preferred. + +```azurecli-interactive +export RANDOM_ID="$(openssl rand -hex 3)" +export MY_RESOURCE_GROUP_NAME="myACIResourceGroup$RANDOM_ID" +export MY_VNET_NAME="aci-vnet" +export MY_SUBNET_NAME="aci-subnet" +export MY_SUBNET_ID="/subscriptions/$(az account show --query id --output tsv)/resourceGroups/$MY_RESOURCE_GROUP_NAME/providers/Microsoft.Network/virtualNetworks/$MY_VNET_NAME/subnets/$MY_SUBNET_NAME" +export MY_APP_CONTAINER_NAME="appcontainer" +export MY_COMM_CHECKER_NAME="commchecker" +export MY_YAML_APP_CONTAINER_NAME="appcontaineryaml" +``` + +### Create a resource group + +You need a resource group to manage all the resources used in the following examples. To create a resource group, use [az group create][az-group-create]: + +```azurecli-interactive +az group create --name $MY_RESOURCE_GROUP_NAME --location eastus +``` + +A successful operation should produce output similar to the following JSON: + +Results: + + + +```json +{ + "id": "/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx/resourceGroups/myACIResourceGroup123abc", + "location": "abcdef", + "managedBy": null, + "name": "myACIResourceGroup123", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Deploy to new virtual network + +> [!NOTE] +> If you are using subnet IP range /29 to have only 3 IP addresses. we recommend always to go one range above (never below). For example, use subnet IP range /28 so you can have at least 1 or more IP buffer per container group. By doing this, you can avoid containers in stuck, not able to start, restart or even not able to stop states. + +To deploy to a new virtual network and have Azure create the network resources for you automatically, specify the following when you execute [az container create][az-container-create]: + +* Virtual network name +* Virtual network address prefix in CIDR format +* Subnet name +* Subnet address prefix in CIDR format + +The virtual network and subnet address prefixes specify the address spaces for the virtual network and subnet, respectively. These values are represented in Classless Inter-Domain Routing (CIDR) notation, for example `10.0.0.0/16`. For more information about working with subnets, see [Add, change, or delete a virtual network subnet](/azure/virtual-network/virtual-network-manage-subnet). + +Once you deploy your first container group with this method, you can deploy to the same subnet by specifying the virtual network and subnet names, or the network profile that Azure automatically creates for you. Because Azure delegates the subnet to Azure Container Instances, you can deploy *only* container groups to the subnet. + +### Example + +The following [az container create][az-container-create] command specifies settings for a new virtual network and subnet. Provide the name of a resource group that was created in a region where container group deployments in a virtual network are [available](container-instances-region-availability.md). This command deploys the public Microsoft aci-helloworld container that runs a small Node.js webserver serving a static web page. In the next section, you'll deploy a second container group to the same subnet, and test communication between the two container instances. + +```azurecli-interactive +az container create \ + --name $MY_APP_CONTAINER_NAME \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --image mcr.microsoft.com/azuredocs/aci-helloworld \ + --vnet $MY_VNET_NAME \ + --vnet-address-prefix 10.0.0.0/16 \ + --subnet $MY_SUBNET_NAME \ + --subnet-address-prefix 10.0.0.0/24 +``` + +A successful operation should produce output similar to the following JSON: + +Results: + + + +```json +{ + "confidentialComputeProperties": null, + "containers": [ + { + "command": null, + "environmentVariables": [], + "image": "mcr.microsoft.com/azuredocs/aci-helloworld", + "instanceView": { + "currentState": { + "detailStatus": "", + "exitCode": null, + "finishTime": null, + "startTime": "0000-00-00T00:00:00.000000+00:00", + "state": "Running" + }, + "events": [ + { + "count": 1, + "firstTimestamp": "0000-00-00T00:00:00+00:00", + "lastTimestamp": "0000-00-00T00:00:00+00:00", + "message": "Successfully pulled image \"mcr.microsoft.com/azuredocs/aci-helloworld@sha256:0000000000000000000000000000000000000000000000000000000000000000\"", + "name": "Pulled", + "type": "Normal" + }, + { + "count": 1, + "firstTimestamp": "0000-00-00T00:00:00+00:00", + "lastTimestamp": "0000-00-00T00:00:00+00:00", + "message": "pulling image \"mcr.microsoft.com/azuredocs/aci-helloworld@sha256:0000000000000000000000000000000000000000000000000000000000000000\"", + "name": "Pulling", + "type": "Normal" + }, + { + "count": 1, + "firstTimestamp": "0000-00-00T00:00:00+00:00", + "lastTimestamp": "0000-00-00T00:00:00+00:00", + "message": "Started container", + "name": "Started", + "type": "Normal" + } + ], + "previousState": null, + "restartCount": 0 + }, + "livenessProbe": null, + "name": "appcontainer", + "ports": [ + { + "port": 80, + "protocol": "TCP" + } + ], + "readinessProbe": null, + "resources": { + "limits": null, + "requests": { + "cpu": 1.0, + "gpu": null, + "memoryInGb": 1.5 + } + }, + "securityContext": null, + "volumeMounts": null + } + ], + "diagnostics": null, + "dnsConfig": null, + "encryptionProperties": null, + "extensions": null, + "id": "/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx/resourceGroups/myACIResourceGroup123/providers/Microsoft.ContainerInstance/containerGroups/appcontainer", + "identity": null, + "imageRegistryCredentials": null, + "initContainers": [], + "instanceView": { + "events": [], + "state": "Running" + }, + "ipAddress": { + "autoGeneratedDomainNameLabelScope": null, + "dnsNameLabel": null, + "fqdn": null, + "ip": "10.0.0.4", + "ports": [ + { + "port": 80, + "protocol": "TCP" + } + ], + "type": "Private" + }, + "location": "eastus", + "name": "appcontainer", + "osType": "Linux", + "priority": null, + "provisioningState": "Succeeded", + "resourceGroup": "myACIResourceGroup123abc", + "restartPolicy": "Always", + "sku": "Standard", + "subnetIds": [ + { + "id": "/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx/resourceGroups/myACIResourceGroup123/providers/Microsoft.Network/virtualNetworks/aci-vnet/subnets/aci-subnet", + "name": null, + "resourceGroup": "myACIResourceGroup123abc" + } + ], + "tags": {}, + "type": "Microsoft.ContainerInstance/containerGroups", + "volumes": null, + "zones": null +} +``` + +When you deploy to a new virtual network by using this method, the deployment can take a few minutes while the network resources are created. After the initial deployment, further container group deployments to the same subnet complete more quickly. + +## Deploy to existing virtual network + +To deploy a container group to an existing virtual network: + +1. Create a subnet within your existing virtual network, use an existing subnet in which a container group is already deployed, or use an existing subnet emptied of *all* other resources and configuration. The subnet that you use for container groups can contain only container groups. Before you deploy a container group to a subnet, you must explicitly delegate the subnet before provisioning. Once delegated, the subnet can be used only for container groups. If you attempt to deploy resources other than container groups to a delegated subnet, the operation fails. +1. Deploy a container group with [az container create][az-container-create] and specify one of the following: + * Virtual network name and subnet name + * Virtual network resource ID and subnet resource ID, which allows using a virtual network from a different resource group + +### Deploy using a YAML file + +You can also deploy a container group to an existing virtual network by using a YAML file, a [Resource Manager template](https://github.com/Azure/azure-quickstart-templates/tree/master/quickstarts/microsoft.containerinstance/aci-vnet), or another programmatic method such as with the Python SDK. + +For example, when using a YAML file, you can deploy to a virtual network with a subnet delegated to Azure Container Instances. Specify the following properties: + +* `ipAddress`: The private IP address settings for the container group. + * `ports`: The ports to open, if any. + * `protocol`: The protocol (TCP or UDP) for the opened port. +* `subnetIds`: The resource IDs of the subnets to be deployed to + * `id`: The resource ID of the subnet + * `name`: The name of the subnet + +This YAML creates a container group in your virtual network. Enter your container group name in the name fields and your subnet ID in the subnet ID field. We use *appcontaineryaml* for the name. If you need to find your subnet ID and no longer have access to previous outputs, you can use the [az container show][az-container-show] command to view it. Look for the `id` field under `subnetIds`. + +```YAML +apiVersion: '2021-07-01' +location: eastus +name: appcontaineryaml +properties: + containers: + - name: appcontaineryaml + properties: + image: mcr.microsoft.com/azuredocs/aci-helloworld + ports: + - port: 80 + protocol: TCP + resources: + requests: + cpu: 1.0 + memoryInGB: 1.5 + ipAddress: + type: Private + ports: + - protocol: tcp + port: '80' + osType: Linux + restartPolicy: Always + subnetIds: + - id: + name: default +tags: null +type: Microsoft.ContainerInstance/containerGroups +``` + +The following Bash command is for the automated deployment pathway. + +```bash +echo -e "apiVersion: '2021-07-01'\nlocation: eastus\nname: $MY_YAML_APP_CONTAINER_NAME\nproperties:\n containers:\n - name: $MY_YAML_APP_CONTAINER_NAME\n properties:\n image: mcr.microsoft.com/azuredocs/aci-helloworld\n ports:\n - port: 80\n protocol: TCP\n resources:\n requests:\n cpu: 1.0\n memoryInGB: 1.5\n ipAddress:\n type: Private\n ports:\n - protocol: tcp\n port: '80'\n osType: Linux\n restartPolicy: Always\n subnetIds:\n - id: $MY_SUBNET_ID\n name: default\ntags: null\ntype: Microsoft.ContainerInstance/containerGroups" > container-instances-vnet.yaml +``` + +Deploy the container group with the [az container create][az-container-create] command, specifying the YAML file name for the `--file` parameter: + +```azurecli-interactive +az container create --resource-group $MY_RESOURCE_GROUP_NAME \ + --file container-instances-vnet.yaml +``` + +The following Bash command is for the automated deployment pathway. + +```bash +rm container-instances-vnet.yaml +``` + +Once the deployment completes, run the [az container show][az-container-show] command to display its status: + +```azurecli-interactive +az container list --resource-group $MY_RESOURCE_GROUP_NAME --output table +``` + +The output should resemble the sample below: + +Results: + + + +```output +Name ResourceGroup Status Image IP:ports Network CPU/Memory OsType Location +---------------- ------------------------ --------- ------------------------------------------ -------------- --------- --------------- -------- ---------- +appcontainer myACIResourceGroup123abc Succeeded mcr.microsoft.com/azuredocs/aci-helloworld 10.0.0.4:80,80 Private 1.0 core/1.5 gb Linux abcdef +appcontaineryaml myACIResourceGroup123abc Succeeded mcr.microsoft.com/azuredocs/aci-helloworld 10.0.0.5:80,80 Private 1.0 core/1.5 gb Linux abcdef +``` + +### Demonstrate communication between container instances + +The following example deploys a third container group to the same subnet created previously. Using an Alpine Linux image, it verifies communication between itself and the first container instance. + +> [!NOTE] +> Due to rate limiting in effect for pulling public Docker images like the Alpine Linux one used here, you may receive an error in the form: +> +> (RegistryErrorResponse) An error response is received from the docker registry 'index.docker.io'. Please retry later. +> Code: RegistryErrorResponse +> Message: An error response is received from the docker registry 'index.docker.io'. Please retry later. + +The following Bash command is for the automated deployment pathway. + +```bash +echo -e "Due to rate limiting in effect for pulling public Docker images like the Alpine Linux one used here, you may receive an error in the form:\n\n(RegistryErrorResponse) An error response is received from the docker registry 'index.docker.io'. Please retry later.\nCode: RegistryErrorResponse\nMessage: An error response is received from the docker registry 'index.docker.io'. Please retry later.\n\nIf this occurs, the automated deployment will exit. You can try again or go to the end of the guide to see instructions for cleaning up your resources." +``` + +First, get the IP address of the first container group you deployed, the *appcontainer*: + +```azurecli-interactive +az container show --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_APP_CONTAINER_NAME \ + --query ipAddress.ip --output tsv +``` + +The output displays the IP address of the container group in the private subnet. For example: + +Results: + + + +```output +10.0.0.4 +``` + +Now, set `CONTAINER_GROUP_IP` to the IP you retrieved with the `az container show` command, and execute the following `az container create` command. This second container, *commchecker*, runs an Alpine Linux-based image and executes `wget` against the first container group's private subnet IP address. + +```azurecli-interactive +az container create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_COMM_CHECKER_NAME \ + --image alpine:3.4 \ + --command-line "wget 10.0.0.4" \ + --restart-policy never \ + --vnet $MY_VNET_NAME \ + --subnet $MY_SUBNET_NAME +``` + +After this second container deployment completes, pull its logs so you can see the output of the `wget` command it executed: + +```azurecli-interactive +az container logs --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_COMM_CHECKER_NAME +``` + +If the second container communicated successfully with the first, output is similar to: + +```output +Connecting to 10.0.0.4 (10.0.0.4:80) +index.html 100% |*******************************| 1663 0:00:00 ETA +``` + +The log output should show that `wget` was able to connect and download the index file from the first container using its private IP address on the local subnet. Network traffic between the two container groups remained within the virtual network. + +## Clean up resources + +If you don't plan to continue using these resources, you can delete them to avoid Azure charges. You can clean up all the resources you used in this guide by deleting the resource group with the [az group delete][az-group-delete] command. Once deleted, **these resources are unrecoverable**. + +## Next steps + +* To deploy a new virtual network, subnet, network profile, and container group using a Resource Manager template, see [Create an Azure container group with virtual network](https://github.com/Azure/azure-quickstart-templates/tree/master/quickstarts/microsoft.containerinstance/aci-vnet). + +* To deploy Azure Container Instances that can pull images from an Azure Container Registry through a private endpoint, see [Deploy to Azure Container Instances from Azure Container Registry using a managed identity](../container-instances/using-azure-container-registry-mi.md). + + +[aci-vnet-01]: ./media/container-instances-vnet/aci-vnet-01.png + + +[aci-helloworld]: https://hub.docker.com/_/microsoft-azuredocs-aci-helloworld + + +[az-group-create]: /cli/azure/group#az-group-create +[az-container-create]: /cli/azure/container#az_container_create +[az-container-show]: /cli/azure/container#az_container_show +[az-network-vnet-create]: /cli/azure/network/vnet#az_network_vnet_create +[az-group-delete]: /cli/azure/group#az-group-delete +[available-regions]: https://azure.microsoft.com/explore/global-infrastructure/products-by-region/?products=container-instances \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 7cc74619e..9da5bf6ac 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1,7 +1,7 @@ [ { "status": "active", - "key": "azure-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md", + "key": "azure-aks-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md", "title": "Deploy an Azure Kubernetes Service (AKS) cluster", "description": "Learn how to quickly deploy a Kubernetes cluster and deploy an application in Azure Kubernetes Service (AKS) using Azure CLI", "stackDetails": "", @@ -1087,5 +1087,27 @@ ], "configurations": { } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/container-instances/container-instances-vnet.md", + "title": "Deploy container group to Azure virtual network", + "description": "Learn how to deploy a container group to a new or existing Azure virtual network via the Azure CLI.", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/container-instances/container-instances-vnet", + "nextSteps": [ + { + "title": "Create an Azure container group with virtual network", + "url": "https://github.com/Azure/azure-quickstart-templates/tree/master/quickstarts/microsoft.containerinstance/aci-vnet" + }, + { + "title": " Deploy to Azure Container Instances from Azure Container Registry using a managed identity", + "url": "https://learn.microsoft.com/en-us/azure/container-instances/using-azure-container-registry-mi" + } + ], + "configurations": { + } } ] From aab66af470852210416aa87d518352ebef4154b9 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Tue, 4 Mar 2025 11:28:55 -0800 Subject: [PATCH 09/87] added 2 new ai docs --- .../virtual-machines/linux/multiple-nics.md | 268 ++++++++++++++++++ scenarios/metadata.json | 22 ++ tools/converted_doc.md | 168 +++++++++++ tools/doc.md | 136 +++++++++ 4 files changed, 594 insertions(+) create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md create mode 100644 tools/converted_doc.md create mode 100644 tools/doc.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md new file mode 100644 index 000000000..8f02ee1a8 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md @@ -0,0 +1,268 @@ +--- +title: Create a Linux VM in Azure with multiple NICs +description: Learn how to create a Linux VM with multiple NICs attached to it using the Azure CLI or Resource Manager templates. +author: mattmcinnes +ms.service: azure-virtual-machines +ms.subservice: networking +ms.topic: how-to +ms.custom: devx-track-azurecli, linux-related-content, innovation-engine +ms.date: 04/06/2023 +ms.author: mattmcinnes +ms.reviewer: cynthn +--- + +# How to create a Linux virtual machine in Azure with multiple network interface cards + +**Applies to:** :heavy_check_mark: Linux VMs :heavy_check_mark: Flexible scale sets + +This article details how to create a VM with multiple NICs with the Azure CLI. + +## Create supporting resources +Install the latest [Azure CLI](/cli/azure/install-az-cli2) and log in to an Azure account using [az login](/cli/azure/reference-index). + +In the following examples, replace example parameter names with your own values. Example parameter names included *myResourceGroup*, *mystorageaccount*, and *myVM*. + +First, create a resource group with [az group create](/cli/azure/group). The following example creates a resource group named *myResourceGroup* in the *eastus* location. In these examples, we declare environment variables as they are used and add a random suffix to unique resource names. + +```azurecli +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" +export REGION="WestUS2" +az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION +``` + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx", + "location": "WestUS2", + "managedBy": null, + "name": "myResourceGroupxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +Create the virtual network with [az network vnet create](/cli/azure/network/vnet). The following example creates a virtual network named *myVnet* and subnet named *mySubnetFrontEnd*: + +```azurecli +export VNET_NAME="myVnet" +export FRONTEND_SUBNET="mySubnetFrontEnd" +az network vnet create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $VNET_NAME \ + --address-prefix 10.0.0.0/16 \ + --subnet-name $FRONTEND_SUBNET \ + --subnet-prefix 10.0.1.0/24 +``` + +Create a subnet for the back-end traffic with [az network vnet subnet create](/cli/azure/network/vnet/subnet). The following example creates a subnet named *mySubnetBackEnd*: + +```azurecli +export BACKEND_SUBNET="mySubnetBackEnd" +az network vnet subnet create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --vnet-name $VNET_NAME \ + --name $BACKEND_SUBNET \ + --address-prefix 10.0.2.0/24 +``` + +Create a network security group with [az network nsg create](/cli/azure/network/nsg). The following example creates a network security group named *myNetworkSecurityGroup*: + +```azurecli +export NSG_NAME="myNetworkSecurityGroup" +az network nsg create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $NSG_NAME +``` + +## Create and configure multiple NICs +Create two NICs with [az network nic create](/cli/azure/network/nic). The following example creates two NICs, named *myNic1* and *myNic2*, connected to the network security group, with one NIC connecting to each subnet: + +```azurecli +export NIC1="myNic1" +export NIC2="myNic2" +az network nic create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $NIC1 \ + --vnet-name $VNET_NAME \ + --subnet $FRONTEND_SUBNET \ + --network-security-group $NSG_NAME +az network nic create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $NIC2 \ + --vnet-name $VNET_NAME \ + --subnet $BACKEND_SUBNET \ + --network-security-group $NSG_NAME +``` + +## Create a VM and attach the NICs +When you create the VM, specify the NICs you created with --nics. You also need to take care when you select the VM size. There are limits for the total number of NICs that you can add to a VM. Read more about [Linux VM sizes](../sizes.md). + +Create a VM with [az vm create](/cli/azure/vm). The following example creates a VM named *myVM*: + +```azurecli +export VM_NAME="myVM" +az vm create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $VM_NAME \ + --image Ubuntu2204 \ + --size Standard_DS3_v2 \ + --admin-username azureuser \ + --generate-ssh-keys \ + --nics $NIC1 $NIC2 +``` + +Add routing tables to the guest OS by completing the steps in [Configure the guest OS for multiple NICs](#configure-guest-os-for-multiple-nics). + +## Add a NIC to a VM +The previous steps created a VM with multiple NICs. You can also add NICs to an existing VM with the Azure CLI. Different [VM sizes](../sizes.md) support a varying number of NICs, so size your VM accordingly. If needed, you can [resize a VM](../resize-vm.md). + +Create another NIC with [az network nic create](/cli/azure/network/nic). The following example creates a NIC named *myNic3* connected to the back-end subnet and network security group created in the previous steps: + +```azurecli +export NIC3="myNic3" +az network nic create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $NIC3 \ + --vnet-name $VNET_NAME \ + --subnet $BACKEND_SUBNET \ + --network-security-group $NSG_NAME +``` + +To add a NIC to an existing VM, first deallocate the VM with [az vm deallocate](/cli/azure/vm). The following example deallocates the VM named *myVM*: + +```azurecli +az vm deallocate --resource-group $MY_RESOURCE_GROUP_NAME --name $VM_NAME +``` + +Add the NIC with [az vm nic add](/cli/azure/vm/nic). The following example adds *myNic3* to *myVM*: + +```azurecli +az vm nic add \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --vm-name $VM_NAME \ + --nics $NIC3 +``` + +Start the VM with [az vm start](/cli/azure/vm): + +```azurecli +az vm start --resource-group $MY_RESOURCE_GROUP_NAME --name $VM_NAME +``` + +Add routing tables to the guest OS by completing the steps in [Configure the guest OS for multiple NICs](#configure-guest-os-for-multiple-nics). + +## Remove a NIC from a VM +To remove a NIC from an existing VM, first deallocate the VM with [az vm deallocate](/cli/azure/vm). The following example deallocates the VM named *myVM*: + +```azurecli +az vm deallocate --resource-group $MY_RESOURCE_GROUP_NAME --name $VM_NAME +``` + +Remove the NIC with [az vm nic remove](/cli/azure/vm/nic). The following example removes *myNic3* from *myVM*: + +```azurecli +az vm nic remove \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --vm-name $VM_NAME \ + --nics $NIC3 +``` + +Start the VM with [az vm start](/cli/azure/vm): + +```azurecli +az vm start --resource-group $MY_RESOURCE_GROUP_NAME --name $VM_NAME +``` + +## Create multiple NICs using Resource Manager templates +Azure Resource Manager templates use declarative JSON files to define your environment. You can read an [overview of Azure Resource Manager](/azure/azure-resource-manager/management/overview). Resource Manager templates provide a way to create multiple instances of a resource during deployment, such as creating multiple NICs. You use *copy* to specify the number of instances to create: + +```json +"copy": { + "name": "multiplenics" + "count": "[parameters('count')]" +} +``` + +Read more about [creating multiple instances using *copy*](/azure/azure-resource-manager/templates/copy-resources). + +You can also use a copyIndex() to then append a number to a resource name, which allows you to create myNic1, myNic2, etc. The following shows an example of appending the index value: + +```json +"name": "[concat('myNic', copyIndex())]", +``` + +You can read a complete example of [creating multiple NICs using Resource Manager templates](/azure/virtual-network/template-samples). + +Add routing tables to the guest OS by completing the steps in [Configure the guest OS for multiple NICs](#configure-guest-os-for-multiple-nics). + +## Configure guest OS for multiple NICs + +The previous steps created a virtual network and subnet, attached NICs, then created a VM. A public IP address and network security group rules that allow SSH traffic were not created. To configure the guest OS for multiple NICs, you need to allow remote connections and run commands locally on the VM. + +To allow SSH traffic, create a network security group rule with [az network nsg rule create](/cli/azure/network/nsg/rule#az-network-nsg-rule-create) as follows: + +```azurecli +az network nsg rule create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --nsg-name $NSG_NAME \ + --name allow_ssh \ + --priority 101 \ + --destination-port-ranges 22 +``` + +Create a public IP address with [az network public-ip create](/cli/azure/network/public-ip#az-network-public-ip-create) and assign it to the first NIC with [az network nic ip-config update](/cli/azure/network/nic/ip-config#az-network-nic-ip-config-update): + +```azurecli +export PUBLIC_IP_NAME="myPublicIP" +az network public-ip create --resource-group $MY_RESOURCE_GROUP_NAME --name $PUBLIC_IP_NAME + +az network nic ip-config update \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --nic-name $NIC1 \ + --name ipconfig1 \ + --public-ip $PUBLIC_IP_NAME +``` + +To view the public IP address of the VM, use [az vm show](/cli/azure/vm#az-vm-show) as follows: + +```azurecli +az vm show --resource-group $MY_RESOURCE_GROUP_NAME --name $VM_NAME -d --query publicIps -o tsv +``` + +```TEXT +x.x.x.x +``` + +Now SSH to the public IP address of your VM. The default username provided in a previous step was *azureuser*. Provide your own username and public IP address: + +```bash +export IP_ADDRESS=$(az vm show --resource-group $MY_RESOURCE_GROUP_NAME --name $VM_NAME -d --query publicIps -o tsv) +ssh -o StrictHostKeyChecking=no azureuser@$IP_ADDRESS +``` +To send to or from a secondary network interface, you have to manually add persistent routes to the operating system for each secondary network interface. In this article, *eth1* is the secondary interface. Instructions for adding persistent routes to the operating system vary by distro. See documentation for your distro for instructions. + +When adding the route to the operating system, the gateway address is the first address of the subnet the network interface is in. For example, if the subnet has been assigned the range 10.0.2.0/24, the gateway you specify for the route is 10.0.2.1 or if the subnet has been assigned the range 10.0.2.128/25, the gateway you specify for the route is 10.0.2.129. You can define a specific network for the route's destination, or specify a destination of 0.0.0.0, if you want all traffic for the interface to go through the specified gateway. The gateway for each subnet is managed by the virtual network. + +Once you've added the route for a secondary interface, verify that the route is in your route table with `route -n`. The following example output is for the route table that has the two network interfaces added to the VM in this article: + +```output +Kernel IP routing table +Destination Gateway Genmask Flags Metric Ref Use Iface +0.0.0.0 10.0.1.1 0.0.0.0 UG 0 0 0 eth0 +0.0.0.0 10.0.2.1 0.0.0.0 UG 0 0 0 eth1 +10.0.1.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 +10.0.2.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1 +168.63.129.16 10.0.1.1 255.255.255.255 UGH 0 0 0 eth0 +169.254.169.254 10.0.1.1 255.255.255.255 UGH 0 0 0 eth0 +``` + +Confirm that the route you added persists across reboots by checking your route table again after a reboot. To test connectivity, you can enter the following command, for example, where *eth1* is the name of a secondary network interface: `ping bing.com -c 4 -I eth1` + +## Next steps +Review [Linux VM sizes](../sizes.md) when trying to creating a VM with multiple NICs. Pay attention to the maximum number of NICs each VM size supports. + +To further secure your VMs, use just in time VM access. This feature opens network security group rules for SSH traffic when needed, and for a defined period of time. For more information, see [Manage virtual machine access using just in time](/azure/security-center/security-center-just-in-time). \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 9da5bf6ac..d32af71ec 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1109,5 +1109,27 @@ ], "configurations": { } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md", + "title": "Deploy container group to Azure virtual network", + "description": "Learn how to deploy a container group to a new or existing Azure virtual network via the Azure CLI.", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/multiple-nics", + "nextSteps": [ + { + "title": "Review Linux VM Sizes", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/sizes" + }, + { + "title": "Manage virtual machine access using just in time", + "url": "https://learn.microsoft.com/en-us/azure/security-center/security-center-just-in-time" + } + ], + "configurations": { + } } ] diff --git a/tools/converted_doc.md b/tools/converted_doc.md new file mode 100644 index 000000000..283591c48 --- /dev/null +++ b/tools/converted_doc.md @@ -0,0 +1,168 @@ +--- +title: 'Quickstart: Use Terraform to create a Linux VM' +description: In this quickstart, you learn how to use Terraform to create a Linux virtual machine +author: tomarchermsft +ms.service: azure-virtual-machines +ms.collection: linux +ms.topic: quickstart +ms.date: 07/24/2023 +ms.author: tarcher +ms.custom: devx-track-terraform, linux-related-content, innovation-engine +content_well_notification: + - AI-contribution +ai-usage: ai-assisted +--- + +# Quickstart: Use Terraform to create a Linux VM + +**Applies to:** :heavy_check_mark: Linux VMs + +Article tested with the following Terraform and Terraform provider versions: + +This article shows you how to create a complete Linux environment and supporting resources with Terraform. Those resources include a virtual network, subnet, public IP address, and more. + +[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] + +In this article, you learn how to: +> [!div class="checklist"] +> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet). +> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group). +> * Create a virtual network (VNET) using [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network). +> * Create a subnet using [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet). +> * Create a public IP using [azurerm_public_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip). +> * Create a network security group using [azurerm_network_security_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_group). +> * Create a network interface using [azurerm_network_interface](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface). +> * Create an association between the network security group and the network interface using [azurerm_network_interface_security_group_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface_security_group_association). +> * Generate a random value for a unique storage account name using [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id). +> * Create a storage account for boot diagnostics using [azurerm_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account). +> * Create a Linux VM using [azurerm_linux_virtual_machine](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine) +> * Create an AzAPI resource [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource). +> * Create an AzAPI resource to generate an SSH key pair using [azapi_resource_action](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource_action). + +## Prerequisites + +- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) + +## Implement the Terraform code + +> [!NOTE] +> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure/TestRecord.md). +> +> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) + +1. Create a directory in which to test the sample Terraform code and make it the current directory. + +1. Create a file named providers.tf and insert the following code: + +:::bash +# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/providers.tf +::: + +1. Create a file named ssh.tf and insert the following code: + +:::bash +# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/ssh.tf +::: + +1. Create a file named main.tf and insert the following code: + +:::bash +# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/main.tf +::: + +1. Create a file named variables.tf and insert the following code: + +:::bash +# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/variables.tf +::: + +1. Create a file named outputs.tf and insert the following code: + +:::bash +# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/outputs.tf +::: + +## Initialize Terraform + +[!INCLUDE [terraform-init.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-init.md)] + +## Create a Terraform execution plan + +[!INCLUDE [terraform-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan.md)] + +## Apply a Terraform execution plan + +[!INCLUDE [terraform-apply-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-apply-plan.md)] + +Cost information isn't presented during the virtual machine creation process for Terraform like it is for the [Azure portal](quick-create-portal.md). If you want to learn more about how cost works for virtual machines, see the [Cost optimization Overview page](../plan-to-manage-costs.md). + +## Verify the results + +#### [Azure CLI](#tab/azure-cli) + +1. Get the Azure resource group name. + +```bash +export resource_group_name=$(terraform output -raw resource_group_name) +``` + +```JSON +{ + "resource_group_name": "RESOURCE_GROUP_NAMExxx" +} +``` + +1. Run az vm list with a JMESPath query to display the names of the virtual machines created in the resource group. + +```azurecli +az vm list \ + --resource-group $resource_group_name \ + --query "[].{\"VM Name\":name}" -o table +``` + +```JSON +[ + { + "VM Name": "myLinuxVMxxx" + } +] +``` + +#### [Azure PowerShell](#tab/azure-powershell) + +1. Get the Azure resource group name. + +```bash +$resource_group_name=$(terraform output -raw resource_group_name) +``` + +```JSON +{ + "resource_group_name": "RESOURCE_GROUP_NAMExxx" +} +``` + +1. Run Get-AzVm to display the names of all the virtual machines in the resource group. + +```bash +Get-AzVm -ResourceGroupName $resource_group_name +``` + +```JSON +[ + { + "Name": "myLinuxVMxxx" + } +] +``` + +## Troubleshoot Terraform on Azure + +[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) + +## Next steps + +In this quickstart, you deployed a simple virtual machine using Terraform. To learn more about Azure virtual machines, continue to the tutorial for Linux VMs. + +> [!div class="nextstepaction"] +> [Azure Linux virtual machine tutorials](./tutorial-manage-vm.md) \ No newline at end of file diff --git a/tools/doc.md b/tools/doc.md new file mode 100644 index 000000000..432e59958 --- /dev/null +++ b/tools/doc.md @@ -0,0 +1,136 @@ +--- +title: 'Quickstart: Use Terraform to create a Linux VM' +description: In this quickstart, you learn how to use Terraform to create a Linux virtual machine +author: tomarchermsft +ms.service: azure-virtual-machines +ms.collection: linux +ms.topic: quickstart +ms.date: 07/24/2023 +ms.author: tarcher +ms.custom: devx-track-terraform, linux-related-content +content_well_notification: + - AI-contribution +ai-usage: ai-assisted +--- + +# Quickstart: Use Terraform to create a Linux VM + +**Applies to:** :heavy_check_mark: Linux VMs + +Article tested with the following Terraform and Terraform provider versions: + +This article shows you how to create a complete Linux environment and supporting resources with Terraform. Those resources include a virtual network, subnet, public IP address, and more. + +[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] + +In this article, you learn how to: +> [!div class="checklist"] +> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet). +> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group). +> * Create a virtual network (VNET) using [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network). +> * Create a subnet using [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet). +> * Create a public IP using [azurerm_public_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip). +> * Create a network security group using [azurerm_network_security_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_group). +> * Create a network interface using [azurerm_network_interface](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface). +> * Create an association between the network security group and the network interface using [azurerm_network_interface_security_group_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface_security_group_association). +> * Generate a random value for a unique storage account name using [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id). +> * Create a storage account for boot diagnostics using [azurerm_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account). +> * Create a Linux VM using [azurerm_linux_virtual_machine](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine) +> * Create an AzAPI resource [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource). +> * Create an AzAPI resource to generate an SSH key pair using [azapi_resource_action](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource_action). + +## Prerequisites + +- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) + +## Implement the Terraform code + +> [!NOTE] +> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure/TestRecord.md). +> +> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) + +1. Create a directory in which to test the sample Terraform code and make it the current directory. + +1. Create a file named `providers.tf` and insert the following code: + + :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/providers.tf"::: + +1. Create a file named `ssh.tf` and insert the following code: + + :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/ssh.tf"::: + +1. Create a file named `main.tf` and insert the following code: + + :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/main.tf"::: + +1. Create a file named `variables.tf` and insert the following code: + + :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/variables.tf"::: + +1. Create a file named `outputs.tf` and insert the following code: + + :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/outputs.tf"::: + +## Initialize Terraform + +[!INCLUDE [terraform-init.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-init.md)] + +## Create a Terraform execution plan + +[!INCLUDE [terraform-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan.md)] + +## Apply a Terraform execution plan + +[!INCLUDE [terraform-apply-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-apply-plan.md)] + +Cost information isn't presented during the virtual machine creation process for Terraform like it is for the [Azure portal](quick-create-portal.md). If you want to learn more about how cost works for virtual machines, see the [Cost optimization Overview page](../plan-to-manage-costs.md). + +## Verify the results + +#### [Azure CLI](#tab/azure-cli) + +1. Get the Azure resource group name. + + ```console + resource_group_name=$(terraform output -raw resource_group_name) + ``` + +1. Run [az vm list](/cli/azure/vm#az-vm-list) with a [JMESPath](/cli/azure/query-azure-cli) query to display the names of the virtual machines created in the resource group. + + ```azurecli + az vm list \ + --resource-group $resource_group_name \ + --query "[].{\"VM Name\":name}" -o table + ``` + +#### [Azure PowerShell](#tab/azure-powershell) + +1. Get the Azure resource group name. + + ```console + $resource_group_name=$(terraform output -raw resource_group_name) + ``` + +1. Run [Get-AzVm](/powershell/module/az.compute/get-azvm) to display the names of all the virtual machines in the resource group. + + ```azurepowershell + Get-AzVm -ResourceGroupName $resource_group_name + ``` + +--- + +## Clean up resources + +[!INCLUDE [terraform-plan-destroy.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan-destroy.md)] + +## Troubleshoot Terraform on Azure + +[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) + +## Next steps + +In this quickstart, you deployed a simple virtual machine using Terraform. To learn more about Azure virtual machines, continue to the tutorial for Linux VMs. + +> [!div class="nextstepaction"] +> [Azure Linux virtual machine tutorials](./tutorial-manage-vm.md) \ No newline at end of file From 1e37504500550cfce163fbd21bde9487daaf04a7 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 5 Mar 2025 00:47:39 -0800 Subject: [PATCH 10/87] added a bunch of more functionalities in the ai tool and its readme --- tools/README.md | 79 +++-- tools/ada.py | 468 +++++++++++++++++++++++++-- tools/aks-store-quickstart.yaml | 286 +++++++++++++++++ tools/converted_doc.md | 168 ---------- tools/converted_test.md | 248 --------------- tools/doc.md | 543 +++++++++++++++++++++++++++----- tools/execution_log.csv | 6 + tools/generated_exec_doc.md | 3 - tools/generated_exec_doccc.md | 272 ---------------- 9 files changed, 1232 insertions(+), 841 deletions(-) create mode 100644 tools/aks-store-quickstart.yaml delete mode 100644 tools/converted_doc.md delete mode 100644 tools/converted_test.md delete mode 100644 tools/generated_exec_doc.md delete mode 100644 tools/generated_exec_doccc.md diff --git a/tools/README.md b/tools/README.md index 4b931a162..ddbaa4404 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,21 +1,23 @@ # ADA - AI Documentation Assistant -Welcome to ADA! This tool helps you convert documents and troubleshoot errors efficiently using OpenAI's Large Language Models and the Azure Innovation Engine. +Welcome to ADA! This tool helps you convert documents and troubleshoot errors efficiently using Azure OpenAI's Large Language Models and the Azure Innovation Engine. ## Features -- Converts input documents using OpenAI's LLMs. -- Automatically installs required packages and the Innovation Engine. +- Converts source markdown files to Exec Docs with proper formatting. +- Generates new Exec Docs from workload descriptions with auto-generated titles. +- Creates documentation for shell scripts while preserving the original code. +- Redacts Personally Identifiable Information (PII) from Exec Doc result blocks. +- Automatically identifies and generates dependency files referenced in documents. +- Performs comprehensive security vulnerability analysis on Exec Docs. - Runs tests on the converted document using the Innovation Engine. -- Provides detailed error logs and generates troubleshooting steps. -- Merges code blocks from the updated document with non-code content from the original document. - Logs execution data to a CSV file for analytics. ## Prerequisites - Python 3.6 or higher - An Azure OpenAI API key -- Required Python packages: `openai`, `azure-identity`, `requests` +- Required Python packages: `openai`, `azure-identity`, `requests`, `pyyaml` ## Installation @@ -27,7 +29,7 @@ Welcome to ADA! This tool helps you convert documents and troubleshoot errors ef 2. Install the required Python packages: ```bash - pip install openai azure-identity requests + pip install openai azure-identity requests pyyaml ``` 3. Ensure you have the Azure OpenAI API key and endpoint set as environment variables: @@ -49,7 +51,7 @@ Welcome to ADA! This tool helps you convert documents and troubleshoot errors ef - **Subscription**: Choose your Azure subscription. - **Resource Group**: Select an existing resource group or create a new one. - **Region**: Choose the region closest to your location. - - **Name**: Provide a unique name for your OpenAI resource. + - **Name**: Provide a unique name for your Azure OpenAI resource. - **Pricing Tier**: Select the appropriate pricing tier (e.g., Standard S0). - Click "Review + create" and then "Create" to deploy the resource. @@ -69,7 +71,7 @@ Welcome to ADA! This tool helps you convert documents and troubleshoot errors ef 5. **Set Environment Variables in Linux**: - Open your terminal. - - Edit the `.bashrc` file using a text editor, such as `nano`: + - Edit the [.bashrc](http://_vscodecontentref_/2) file using a text editor, such as `nano`: ```bash nano ~/.bashrc ``` @@ -79,7 +81,7 @@ Welcome to ADA! This tool helps you convert documents and troubleshoot errors ef export AZURE_OPENAI_ENDPOINT="" ``` - Save and exit the editor (`Ctrl + X`, then `Y`, and `Enter` for nano). - - Apply the changes by sourcing the `.bashrc` file: + - Apply the changes by sourcing the [.bashrc](http://_vscodecontentref_/3) file: ```bash source ~/.bashrc ``` @@ -100,49 +102,60 @@ Welcome to ADA! This tool helps you convert documents and troubleshoot errors ef python ada.py ``` -2. Enter the path to the input file or describe your intended workload when prompted. +2. Choose from the available options: + - Option 1: Convert an existing markdown file to an Exec Doc + - Option 2: Describe a workload to generate a new Exec Doc + - Option 3: Add descriptions to a shell script as an Exec Doc + - Option 4: Redact PII from an existing Exec Doc + - Option 5: Perform security vulnerability check on an Exec Doc -3. The script will process the file or description, convert it using OpenAI's GPT-4O model, and perform testing using the Innovation Engine. +3. Follow the prompts to provide the required information: + - For file conversion, provide the path to your input file + - For workload descriptions, describe your intended workload in detail + - For shell script documentation, provide the path to your script and optional context + - For PII redaction, provide the path to your Exec Doc + - For security checks, provide the path to your Exec Doc -4. If the tests fail, the script will generate troubleshooting steps and attempt to correct the document. +4. The tool will process your request based on the selected option: + - For options 1 and 2, it will convert or create an Exec Doc and run tests using Innovation Engine + - For options 3, 4, and 5, it will generate the requested output and save it to a file -5. If the tests pass successfully, the script will merge code blocks from the updated document with non-code content from the original document. - -6. The final merged document will be saved, and a summary will be displayed. +5. For document conversion or creation, if the tests pass successfully, the final document will be saved with proper formatting. ## Script Workflow 1. **Initialization**: The script initializes the Azure OpenAI client and checks for required packages. -2. **Input File or Workload Description**: Prompts the user to enter the path to the input file or describe their intended workload. - -3. **System Prompt**: Prepares the system prompt for the AI model. - -4. **File Content or Workload Description**: Reads the content of the input file or uses the provided workload description. +2. **Option Selection**: Prompts the user to select from available options for document processing. -5. **Install Innovation Engine**: Checks if the Innovation Engine is installed and installs it if necessary. +3. **Input Collection**: Collects necessary inputs based on the selected option. -6. **Conversion and Testing**: - - Attempts to convert the document using OpenAI's GPT-4O model. - - Runs tests on the converted document using the Innovation Engine. - - If tests fail, generates troubleshooting steps and attempts to correct the document. +4. **Processing Based on Option**: + - **Convert Markdown**: Converts an existing markdown file to an Exec Doc + - **Generate New Doc**: Creates an Exec Doc from a workload description + - **Document Script**: Adds detailed explanations to a shell script + - **Redact PII**: Removes personally identifiable information from result blocks + - **Security Check**: Performs comprehensive security analysis -7. **Merge Documents**: - - If tests pass successfully, merges code blocks from the updated document with non-code content from the original document. - - Ensures that anything not within code blocks remains unchanged from the original document. +5. **For Document Conversion and Generation**: + - Install Innovation Engine if needed + - Process the document using Azure OpenAI's model + - Run tests on the document using Innovation Engine + - If tests fail, generate troubleshooting steps and attempt corrections + - If tests pass, finalize the document -8. **Remove Backticks**: Ensures that backticks are properly handled in the document. +6. **Final Output**: Saves the processed document and provides the file path. -9. **Logging**: Logs execution data to `execution_log.csv`. +7. **Dependency Generation**: Optionally identifies and creates dependency files referenced in the document. -10. **Final Output**: Saves the final merged document and provides the path. +8. **Logging**: Logs execution data to `execution_log.csv`. ## Logging The script logs the following data to `execution_log.csv`: - Timestamp: The date and time when the script was run. -- Type: Whether the input was a file or a workload description. +- Type: The type of processing performed (file conversion, workload description, etc.). - Input: The path to the input file or the workload description. - Output: The path to the output file. - Number of Attempts: The number of attempts made to generate a successful document. diff --git a/tools/ada.py b/tools/ada.py index 29e75e161..ce6a8fd7b 100644 --- a/tools/ada.py +++ b/tools/ada.py @@ -10,6 +10,9 @@ from datetime import datetime from openai import AzureOpenAI from collections import defaultdict +import re +import json +import yaml # Add this import at the top of your file client = AzureOpenAI( api_key=os.getenv("AZURE_OPENAI_API_KEY"), @@ -289,6 +292,280 @@ def get_last_error_log(): return "".join(lines[error_index:]) return "No error log found." +def generate_script_description(script_path, context=""): + """Generate descriptions around a shell script without modifying the code.""" + if not os.path.isfile(script_path): + print(f"\nError: The file {script_path} does not exist.") + return None + + try: + with open(script_path, "r") as f: + script_content = f.read() + except Exception as e: + print(f"\nError reading script: {e}") + return None + + # Create output filename + script_name = os.path.splitext(os.path.basename(script_path))[0] + output_file = f"{script_name}_documented.md" + + print("\nGenerating documentation for shell script...") + + # Prepare prompt for the LLM + script_prompt = f"""Create an Exec Doc that explains this shell script in detail. + DO NOT CHANGE ANY CODE in the script. Instead: + 1. Add clear descriptions before and after each functional block + 2. Explain what each section does + 3. Format as a proper markdown document with appropriate headings and structure + 4. Include all the necessary metadata in the front matter + + Script context provided by user: {context} + + Here is the script content: + ``` + {script_content} + ``` + """ + + response = client.chat.completions.create( + model=deployment_name, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": script_prompt} + ] + ) + + doc_content = response.choices[0].message.content + + # Save the generated documentation + try: + with open(output_file, "w") as f: + f.write(doc_content) + print(f"\nScript documentation saved to: {output_file}") + return output_file + except Exception as e: + print(f"\nError saving documentation: {e}") + return None + +def redact_pii_from_doc(doc_path): + """Redact PII from result blocks in an Exec Doc.""" + if not os.path.isfile(doc_path): + print(f"\nError: The file {doc_path} does not exist.") + return None + + try: + with open(doc_path, "r") as f: + doc_content = f.read() + except Exception as e: + print(f"\nError reading document: {e}") + return None + + # Create output filename + doc_name = os.path.splitext(os.path.basename(doc_path))[0] + output_file = f"{doc_name}_redacted.md" + + print("\nRedacting PII from document...") + + # Use the LLM to identify and redact PII + redaction_prompt = """Redacting PII from the output helps protect sensitive information from being inadvertently shared or exposed. This is crucial for maintaining privacy, complying with data protection regulations, and furthering the company's security posture. + + Ensure result block(s) have all the PII (Personally Identifiable Information) stricken out from them and replaced with x’s. + + **Example:** + + ```markdown + Results: + + + + ```JSON + {{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyResourceGroupxxx", + "location": "eastus", + "managedBy": null, + "name": "MyResourceGroupxxx", + "properties": {{ + "provisioningState": "Succeeded" + }}, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" + }} + ``` + ``` + + >**Note:** The number of x's used to redact PII need not be the same as the number of characters in the original PII. Furthermore, it is recommended not to redact the key names in the output, only the values containing the PII (which are usually strings). + + >**Note:** Here are some examples of PII in result blocks: Unique identifiers for resources, Email Addresses, Phone Numbers, IP Addresses, Credit Card Numbers, Social Security Numbers (SSNs), Usernames, Resource Names, Subscription IDs, Resource Group Names, Tenant IDs, Service Principal Names, Client IDs, Secrets and Keys. + + Document content: + """ + + response = client.chat.completions.create( + model=deployment_name, + messages=[ + {"role": "system", "content": "You are an AI specialized in PII redaction. Either redact the PII or return the document as is - nothing els is acceptable."}, + {"role": "user", "content": redaction_prompt + "\n\n" + doc_content} + ] + ) + + redacted_content = response.choices[0].message.content + + # Save the redacted document + try: + with open(output_file, "w") as f: + f.write(redacted_content) + print(f"\nRedacted document saved to: {output_file}") + return output_file + except Exception as e: + print(f"\nError saving redacted document: {e}") + return None + +def generate_dependency_files(doc_path): + """Extract and generate dependency files referenced in an Exec Doc.""" + if not os.path.isfile(doc_path): + print(f"\nError: The file {doc_path} does not exist.") + return False + + try: + with open(doc_path, "r") as f: + doc_content = f.read() + except Exception as e: + print(f"\nError reading document: {e}") + return False + + # Directory where the doc is located + doc_dir = os.path.dirname(doc_path) or "." + + print("\nAnalyzing document for dependencies...") + + # Enhanced prompt for better dependency file identification + dependency_prompt = """Analyze this Exec Doc and identify ANY files that the user is instructed to create. + + Look specifically for: + 1. Files where the doc says "Create a file named X" or similar instructions + 2. Files that are referenced in commands (e.g., kubectl apply -f filename.yaml) + 3. YAML files (configuration, templates, manifests) + 4. JSON files (configuration, templates, API payloads) + 5. Shell scripts (.sh files) + 6. Any other files where content is provided and meant to be saved separately + + IMPORTANT: Include files even if their full content is provided in the document! + If the doc instructs the user to create a file and provides its content, this IS a dependency file. + + For each file you identify: + 1. Extract the exact filename with its extension + 2. Use the exact content provided in the document + 3. Format your response as a JSON list + """ + + response = client.chat.completions.create( + model=deployment_name, + messages=[ + {"role": "system", "content": "You are an AI specialized in extracting and generating dependency files."}, + {"role": "user", "content": dependency_prompt + "\n\n" + doc_content} + ] + ) + + try: + # Extract the JSON part from the response with improved robustness + response_text = response.choices[0].message.content + + # Find JSON content between triple backticks with more flexible pattern matching + json_match = re.search(r'```(?:json)?(.+?)```', response_text, re.DOTALL) + if json_match: + # Clean the extracted JSON content + json_content = json_match.group(1).strip() + try: + dependency_list = json.loads(json_content) + except json.JSONDecodeError: + # Try removing any non-JSON text at the beginning or end + json_content = re.search(r'(\[.+?\])', json_content, re.DOTALL) + if json_content: + dependency_list = json.loads(json_content.group(1)) + else: + raise ValueError("Could not extract valid JSON from response") + else: + # Try to parse the entire response as JSON + try: + dependency_list = json.loads(response_text) + except json.JSONDecodeError: + # Last resort: look for anything that looks like a JSON array + array_match = re.search(r'\[(.*?)\]', response_text.replace('\n', ''), re.DOTALL) + if array_match: + try: + dependency_list = json.loads('[' + array_match.group(1) + ']') + except: + raise ValueError("Could not extract valid JSON from response") + else: + raise ValueError("Response did not contain valid JSON") + + if not dependency_list: + print("\nNo dependency files identified.") + return True + + # Create each dependency file with type-specific handling + created_files = [] + for dep in dependency_list: + filename = dep.get("filename") + content = dep.get("content") + file_type = dep.get("type", "").lower() + + if not filename or not content: + continue + + file_path = os.path.join(doc_dir, filename) + + # Check if file already exists + if os.path.exists(file_path): + print(f"\nFile already exists: {filename} - Skipping") + continue + + # Validate and format content based on file type + try: + if filename.endswith('.json') or file_type == 'json': + # Validate JSON + try: + parsed = json.loads(content) + content = json.dumps(parsed, indent=2) # Pretty-print JSON + except json.JSONDecodeError: + print(f"\nWarning: Content for {filename} is not valid JSON. Saving as plain text.") + + elif filename.endswith('.yaml') or filename.endswith('.yml') or file_type == 'yaml': + # Validate YAML + try: + parsed = yaml.safe_load(content) + content = yaml.dump(parsed, default_flow_style=False) # Pretty-print YAML + except yaml.YAMLError: + print(f"\nWarning: Content for {filename} is not valid YAML. Saving as plain text.") + + elif filename.endswith('.sh') or file_type == 'shell': + # Ensure shell scripts are executable + is_executable = True + + # Write the file + with open(file_path, "w") as f: + f.write(content) + + # Make shell scripts executable if needed + if (filename.endswith('.sh') or file_type == 'shell') and is_executable: + os.chmod(file_path, os.stat(file_path).st_mode | 0o111) # Add executable bit + + created_files.append(filename) + except Exception as e: + print(f"\nError creating {filename}: {e}") + + if created_files: + print(f"\nCreated {len(created_files)} dependency files: {', '.join(created_files)}") + else: + print("\nNo new dependency files were created.") + + return True + except Exception as e: + print(f"\nError generating dependency files: {e}") + print("\nResponse from model was not valid JSON. Raw response:") + # print(response.choices[0].message.content[:500] + "..." if len(response.choices[0].message.content) > 500 else response.choices[0].message.content) + return False + def remove_backticks_from_file(file_path): with open(file_path, "r") as f: lines = f.readlines() @@ -319,56 +596,185 @@ def log_data_to_csv(data): writer.writeheader() writer.writerow(data) +def generate_title_from_description(description): + """Generate a title for the Exec Doc based on the workload description.""" + print("\nGenerating title for your Exec Doc...") + + title_prompt = """Create a concise, descriptive title for an Executable Document (Exec Doc) based on the following workload description. + The title should: + 1. Be clear and informative + 2. Start with an action verb (Deploy, Create, Configure, etc.) when appropriate + 3. Mention the main Azure service(s) involved + 4. Be formatted like a typical Azure quickstart or tutorial title + 5. Not exceed 10 words + + Return ONLY the title text, nothing else. + + Workload description: + """ + + try: + response = client.chat.completions.create( + model=deployment_name, + messages=[ + {"role": "system", "content": "You are an AI specialized in creating concise, descriptive titles."}, + {"role": "user", "content": title_prompt + description} + ] + ) + + title = response.choices[0].message.content.strip() + # Remove any quotes, backticks or other formatting that might be included + title = title.strip('"\'`') + print(f"\nGenerated title: {title}") + return title + except Exception as e: + print(f"\nError generating title: {e}") + return "Azure Executable Documentation Guide" # Default fallback title + +def perform_security_check(doc_path): + """Perform a comprehensive security vulnerability check on an Exec Doc.""" + if not os.path.isfile(doc_path): + print(f"\nError: The file {doc_path} does not exist.") + return None + + try: + with open(doc_path, "r") as f: + doc_content = f.read() + except Exception as e: + print(f"\nError reading document: {e}") + return None + + # Create output filename + doc_name = os.path.splitext(os.path.basename(doc_path))[0] + output_file = f"{doc_name}_security_report.md" + + print("\nPerforming comprehensive security vulnerability analysis...") + + # Use the LLM to analyze security vulnerabilities + security_prompt = """Conduct a thorough, state-of-the-art security vulnerability analysis of this Exec Doc. Analyze both static aspects (code review) and dynamic aspects (runtime behavior). + + Focus on: + 1. Authentication and authorization vulnerabilities + 2. Potential for privilege escalation + 3. Resource exposure risks + 4. Data handling and privacy concerns + 5. Network security considerations + 6. Input validation vulnerabilities + 7. Command injection risks + 8. Cloud-specific security threats + 9. Compliance issues with security best practices + 10. Secret management practices + + Structure your report with the following sections: + 1. Executive Summary - Overall risk assessment + 2. Methodology - How the analysis was performed + 3. Findings - Detailed description of each vulnerability found + 4. Recommendations - Specific remediation steps for each issue + 5. Best Practices - General security improvements + + For each vulnerability found, include: + - Severity (Critical, High, Medium, Low) + - Location in code + - Description of the vulnerability + - Potential impact + - Recommended fix with code example where appropriate + + Use the OWASP Top 10 and cloud security best practices as frameworks for your analysis. + Format the output as a professional Markdown document with appropriate headings, tables, and code blocks. + + Document content: + """ + + response = client.chat.completions.create( + model=deployment_name, + messages=[ + {"role": "system", "content": "You are an AI specialized in security vulnerability assessment and report generation."}, + {"role": "user", "content": security_prompt + "\n\n" + doc_content} + ] + ) + + report_content = response.choices[0].message.content + + # Save the security report + try: + with open(output_file, "w") as f: + f.write(report_content) + print(f"\nSecurity analysis report saved to: {output_file}") + return output_file + except Exception as e: + print(f"\nError saving security report: {e}") + return None + def main(): print("\nWelcome to ADA - AI Documentation Assistant!") print("\nThis tool helps you write and troubleshoot Executable Documents efficiently!") print("\nPlease select one of the following options:") - print(" 1. Enter path to markdown file for conversion") - print(" 2. Describe workload for new Exec Doc") - print(" 3. Generate description for shell script") + print(" 1. Enter path to markdown file for conversion to Exec Doc") + print(" 2. Describe workload to generate a new Exec Doc") + print(" 3. Add descriptions to a shell script as an Exec Doc") print(" 4. Redact PII from an existing Exec Doc") - choice = input("Enter the number corresponding to your choice: ") + print(" 5. Perform security vulnerability check on an Exec Doc") + choice = input("\nEnter the number corresponding to your choice: ") if choice == "1": - user_input = input("Enter the path to your markdown file: ") - if os.path.isfile(user_input) and user_input.endswith('.md'): - input_type = 'file' - with open(user_input, "r") as f: - input_content = f.read() - input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" - else: - print("Invalid file path or file type. Please provide a valid markdown file.") + user_input = input("\nEnter the path to your markdown file: ") + if not os.path.isfile(user_input) or not user_input.endswith('.md'): + print("\nInvalid file path or file type. Please provide a valid markdown file.") sys.exit(1) + input_type = 'file' + with open(user_input, "r") as f: + input_content = f.read() + input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" + if input("\nMake new files referenced in the doc for its execution? (y/n): ").lower() == 'y': + generate_dependency_files(user_input) elif choice == "2": - user_input = input("Describe your workload for the new Exec Doc: ") - if os.path.isfile(user_input): - input_type = 'workload_description' - input_content = user_input + user_input = input("\nDescribe your workload for the new Exec Doc: ") + if not user_input: + print("\nInvalid input. Please provide a workload description.") + sys.exit(1) + input_type = 'workload_description' + input_content = user_input elif choice == "3": - user_input = input("Enter the path to your shell script (provide context and details): ") + user_input = input("\nEnter the path to your shell script: ") + context = input("\nProvide additional context for the script (optional): ") + if not os.path.isfile(user_input): + print("\nInvalid file path. Please provide a valid shell script.") + sys.exit(1) + input_type = 'shell_script' + output_file = generate_script_description(user_input, context) + remove_backticks_from_file(output_file) + sys.exit(0) elif choice == "4": - user_input = input("Enter the path to your Exec Doc for PII redaction: ") + user_input = input("\nEnter the path to your Exec Doc for PII redaction: ") + if not os.path.isfile(user_input) or not user_input.endswith('.md'): + print("\nInvalid file path or file type. Please provide a valid markdown file.") + sys.exit(1) + input_type = 'pii_redaction' + output_file = redact_pii_from_doc(user_input) + remove_backticks_from_file(output_file) + sys.exit(0) + elif choice == "5": + user_input = input("\nEnter the path to your Exec Doc for security analysis: ") + if not os.path.isfile(user_input) or not user_input.endswith('.md'): + print("\nInvalid file path or file type. Please provide a valid markdown file.") + sys.exit(1) + input_type = 'security_check' + output_file = perform_security_check(user_input) + if output_file: + print(f"\nSecurity analysis complete. Report saved to: {output_file}") + sys.exit(0) else: - print("Invalid choice. Exiting.") + print("\nInvalid choice. Exiting.") sys.exit(1) - # if os.path.isfile(user_input) and user_input.endswith('.md'): - # input_type = 'file' - # with open(user_input, "r") as f: - # input_content = f.read() - # input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" - # else: - # input_type = 'workload_description' - # input_content = user_input - install_innovation_engine() max_attempts = 11 attempt = 1 if input_type == 'file': - output_file = f"converted_{os.path.splitext(os.path.basename(user_input))[0]}.md" + output_file = f"{os.path.splitext(os.path.basename(user_input))[0]}_converted.md" else: - output_file = "generated_exec_doccc.md" + output_file = f"{generate_title_from_description(user_input)}_ai_generated.md" start_time = time.time() errors_encountered = [] @@ -407,7 +813,7 @@ def main(): try: result = subprocess.run(["ie", "test", output_file], capture_output=True, text=True, timeout=660) except subprocess.TimeoutExpired: - print("The 'ie test' command timed out after 11 minutes.") + print("\nThe 'ie test' command timed out after 11 minutes.") errors_encountered.append("The 'ie test' command timed out after 11 minutes.") attempt += 1 continue # Proceed to the next attempt diff --git a/tools/aks-store-quickstart.yaml b/tools/aks-store-quickstart.yaml new file mode 100644 index 000000000..2aaaf609d --- /dev/null +++ b/tools/aks-store-quickstart.yaml @@ -0,0 +1,286 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rabbitmq +spec: + serviceName: rabbitmq + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: rabbitmq + image: mcr.microsoft.com/mirror/docker/library/rabbitmq:3.10-management-alpine + ports: + - containerPort: 5672 + name: rabbitmq-amqp + - containerPort: 15672 + name: rabbitmq-http + env: + - name: RABBITMQ_DEFAULT_USER + value: "username" + - name: RABBITMQ_DEFAULT_PASS + value: "password" + resources: + requests: + cpu: 10m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: rabbitmq-enabled-plugins + mountPath: /etc/rabbitmq/enabled_plugins + subPath: enabled_plugins + volumes: + - name: rabbitmq-enabled-plugins + configMap: + name: rabbitmq-enabled-plugins + items: + - key: rabbitmq_enabled_plugins + path: enabled_plugins +--- +apiVersion: v1 +data: + rabbitmq_enabled_plugins: | + [rabbitmq_management,rabbitmq_prometheus,rabbitmq_amqp1_0]. +kind: ConfigMap +metadata: + name: rabbitmq-enabled-plugins +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + selector: + app: rabbitmq + ports: + - name: rabbitmq-amqp + port: 5672 + targetPort: 5672 + - name: rabbitmq-http + port: 15672 + targetPort: 15672 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: order-service +spec: + replicas: 1 + selector: + matchLabels: + app: order-service + template: + metadata: + labels: + app: order-service + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: order-service + image: ghcr.io/azure-samples/aks-store-demo/order-service:latest + ports: + - containerPort: 3000 + env: + - name: ORDER_QUEUE_HOSTNAME + value: "rabbitmq" + - name: ORDER_QUEUE_PORT + value: "5672" + - name: ORDER_QUEUE_USERNAME + value: "username" + - name: ORDER_QUEUE_PASSWORD + value: "password" + - name: ORDER_QUEUE_NAME + value: "orders" + - name: FASTIFY_ADDRESS + value: "0.0.0.0" + resources: + requests: + cpu: 1m + memory: 50Mi + limits: + cpu: 75m + memory: 128Mi + startupProbe: + httpGet: + path: /health + port: 3000 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: 3000 + failureThreshold: 3 + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: 3000 + failureThreshold: 5 + initialDelaySeconds: 3 + periodSeconds: 3 + initContainers: + - name: wait-for-rabbitmq + image: busybox + command: ['sh', '-c', 'until nc -zv rabbitmq 5672; do echo waiting for rabbitmq; sleep 2; done;'] + resources: + requests: + cpu: 1m + memory: 50Mi + limits: + cpu: 75m + memory: 128Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: order-service +spec: + type: ClusterIP + ports: + - name: http + port: 3000 + targetPort: 3000 + selector: + app: order-service +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: product-service +spec: + replicas: 1 + selector: + matchLabels: + app: product-service + template: + metadata: + labels: + app: product-service + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: product-service + image: ghcr.io/azure-samples/aks-store-demo/product-service:latest + ports: + - containerPort: 3002 + env: + - name: AI_SERVICE_URL + value: "http://ai-service:5001/" + resources: + requests: + cpu: 1m + memory: 1Mi + limits: + cpu: 2m + memory: 20Mi + readinessProbe: + httpGet: + path: /health + port: 3002 + failureThreshold: 3 + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: 3002 + failureThreshold: 5 + initialDelaySeconds: 3 + periodSeconds: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: product-service +spec: + type: ClusterIP + ports: + - name: http + port: 3002 + targetPort: 3002 + selector: + app: product-service +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: store-front +spec: + replicas: 1 + selector: + matchLabels: + app: store-front + template: + metadata: + labels: + app: store-front + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: store-front + image: ghcr.io/azure-samples/aks-store-demo/store-front:latest + ports: + - containerPort: 8080 + name: store-front + env: + - name: VUE_APP_ORDER_SERVICE_URL + value: "http://order-service:3000/" + - name: VUE_APP_PRODUCT_SERVICE_URL + value: "http://product-service:3002/" + resources: + requests: + cpu: 1m + memory: 200Mi + limits: + cpu: 1000m + memory: 512Mi + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + initialDelaySeconds: 3 + periodSeconds: 3 + livenessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 5 + initialDelaySeconds: 3 + periodSeconds: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: store-front +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: store-front + type: LoadBalancer diff --git a/tools/converted_doc.md b/tools/converted_doc.md deleted file mode 100644 index 283591c48..000000000 --- a/tools/converted_doc.md +++ /dev/null @@ -1,168 +0,0 @@ ---- -title: 'Quickstart: Use Terraform to create a Linux VM' -description: In this quickstart, you learn how to use Terraform to create a Linux virtual machine -author: tomarchermsft -ms.service: azure-virtual-machines -ms.collection: linux -ms.topic: quickstart -ms.date: 07/24/2023 -ms.author: tarcher -ms.custom: devx-track-terraform, linux-related-content, innovation-engine -content_well_notification: - - AI-contribution -ai-usage: ai-assisted ---- - -# Quickstart: Use Terraform to create a Linux VM - -**Applies to:** :heavy_check_mark: Linux VMs - -Article tested with the following Terraform and Terraform provider versions: - -This article shows you how to create a complete Linux environment and supporting resources with Terraform. Those resources include a virtual network, subnet, public IP address, and more. - -[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] - -In this article, you learn how to: -> [!div class="checklist"] -> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet). -> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group). -> * Create a virtual network (VNET) using [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network). -> * Create a subnet using [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet). -> * Create a public IP using [azurerm_public_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip). -> * Create a network security group using [azurerm_network_security_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_group). -> * Create a network interface using [azurerm_network_interface](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface). -> * Create an association between the network security group and the network interface using [azurerm_network_interface_security_group_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface_security_group_association). -> * Generate a random value for a unique storage account name using [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id). -> * Create a storage account for boot diagnostics using [azurerm_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account). -> * Create a Linux VM using [azurerm_linux_virtual_machine](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine) -> * Create an AzAPI resource [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource). -> * Create an AzAPI resource to generate an SSH key pair using [azapi_resource_action](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource_action). - -## Prerequisites - -- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) - -## Implement the Terraform code - -> [!NOTE] -> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure/TestRecord.md). -> -> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) - -1. Create a directory in which to test the sample Terraform code and make it the current directory. - -1. Create a file named providers.tf and insert the following code: - -:::bash -# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/providers.tf -::: - -1. Create a file named ssh.tf and insert the following code: - -:::bash -# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/ssh.tf -::: - -1. Create a file named main.tf and insert the following code: - -:::bash -# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/main.tf -::: - -1. Create a file named variables.tf and insert the following code: - -:::bash -# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/variables.tf -::: - -1. Create a file named outputs.tf and insert the following code: - -:::bash -# Content from file: ~/terraform_samples/quickstart/101-vm-with-infrastructure/outputs.tf -::: - -## Initialize Terraform - -[!INCLUDE [terraform-init.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-init.md)] - -## Create a Terraform execution plan - -[!INCLUDE [terraform-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan.md)] - -## Apply a Terraform execution plan - -[!INCLUDE [terraform-apply-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-apply-plan.md)] - -Cost information isn't presented during the virtual machine creation process for Terraform like it is for the [Azure portal](quick-create-portal.md). If you want to learn more about how cost works for virtual machines, see the [Cost optimization Overview page](../plan-to-manage-costs.md). - -## Verify the results - -#### [Azure CLI](#tab/azure-cli) - -1. Get the Azure resource group name. - -```bash -export resource_group_name=$(terraform output -raw resource_group_name) -``` - -```JSON -{ - "resource_group_name": "RESOURCE_GROUP_NAMExxx" -} -``` - -1. Run az vm list with a JMESPath query to display the names of the virtual machines created in the resource group. - -```azurecli -az vm list \ - --resource-group $resource_group_name \ - --query "[].{\"VM Name\":name}" -o table -``` - -```JSON -[ - { - "VM Name": "myLinuxVMxxx" - } -] -``` - -#### [Azure PowerShell](#tab/azure-powershell) - -1. Get the Azure resource group name. - -```bash -$resource_group_name=$(terraform output -raw resource_group_name) -``` - -```JSON -{ - "resource_group_name": "RESOURCE_GROUP_NAMExxx" -} -``` - -1. Run Get-AzVm to display the names of all the virtual machines in the resource group. - -```bash -Get-AzVm -ResourceGroupName $resource_group_name -``` - -```JSON -[ - { - "Name": "myLinuxVMxxx" - } -] -``` - -## Troubleshoot Terraform on Azure - -[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) - -## Next steps - -In this quickstart, you deployed a simple virtual machine using Terraform. To learn more about Azure virtual machines, continue to the tutorial for Linux VMs. - -> [!div class="nextstepaction"] -> [Azure Linux virtual machine tutorials](./tutorial-manage-vm.md) \ No newline at end of file diff --git a/tools/converted_test.md b/tools/converted_test.md deleted file mode 100644 index b3a49e3ab..000000000 --- a/tools/converted_test.md +++ /dev/null @@ -1,248 +0,0 @@ ---- -title: 'Tutorial: Create & manage a Virtual Machine Scale Set – Azure CLI' -description: Learn how to use the Azure CLI to create a Virtual Machine Scale Set, along with some common management tasks such as how to start and stop an instance, or change the scale set capacity. -author: ju-shim -ms.author: jushiman -ms.topic: tutorial -ms.service: azure-virtual-machine-scale-sets -ms.date: 10/05/2023 -ms.reviewer: mimckitt -ms.custom: mimckitt, devx-track-azurecli, innovation-engine ---- - -# Tutorial: Create and manage a Virtual Machine Scale Set with Azure CLI - -A Virtual Machine Scale Set allows you to deploy and manage a set of virtual machines. Throughout the lifecycle of a Virtual Machine Scale Set, you may need to run one or more management tasks. In this tutorial, you will learn how to: - -- Create a resource group. -- Create a Virtual Machine Scale Set. -- Scale out and in. -- Stop, start, and restart VM instances. - -> [!div class="checklist"] -> * Create a resource group. -> * Create a Virtual Machine Scale Set. -> * Scale out and in. -> * Stop, Start, and restart VM instances. - -This article requires Azure CLI version 2.0.29 or later. If using Azure Cloud Shell, the latest version is already installed. - ---- - -## Create a resource group - -An Azure resource group is a container that holds related resources. A resource group must be created before a Virtual Machine Scale Set. This example uses a unique random suffix for the resource group name to avoid conflicts. Replace `` with a unique value. - -```bash -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export REGION="westus2" -export RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" - -az group create --name $RESOURCE_GROUP_NAME --location $REGION -``` - -The resource group name is used when you create or modify a scale set throughout this tutorial. - -Results: - - - -```json -{ - "id": "/subscriptions/xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupxxx", - "location": "westus2", - "managedBy": null, - "name": "myResourceGroupxxx", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" -} -``` - ---- - -## Create a Virtual Machine Scale Set - -> [!IMPORTANT] -> Starting November 2023, VM scale sets created using PowerShell and Azure CLI will default to Flexible Orchestration Mode if no orchestration mode is specified. For more information about this change and what actions you should take, go to [Breaking Change for VMSS PowerShell/CLI Customers - Microsoft Community Hub](https://techcommunity.microsoft.com/t5/azure-compute-blog/breaking-change-for-vmss-powershell-cli-customers/ba-p/3818295). - -A Virtual Machine Scale Set is created using the `az vmss create` command. Replace `` with a supported image such as `Ubuntu2204`. The VM SKU size is set to `Standard_B1s`. SSH keys are generated if they don’t exist. - -```bash -export SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" -export ADMIN_USERNAME="azureuser" -export VALID_IMAGE="Ubuntu2204" # Use a valid image from the supported list - -az vmss create \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $SCALE_SET_NAME \ - --orchestration-mode flexible \ - --image $VALID_IMAGE \ - --vm-sku "Standard_B1s" \ - --admin-username $ADMIN_USERNAME \ - --generate-ssh-keys -``` - -It takes a few minutes to create and configure the scale set resources and VM instances. A load balancer is also created to distribute traffic. - -Verify the scale set creation: - -```bash -az vmss list --resource-group $RESOURCE_GROUP_NAME --output table -``` - ---- - -## View information about VM instances - -To view a list of VM instances in your scale set, use the `az vmss list-instances` command. Flexible orchestration mode assigns dynamically generated instance names. - -```bash -az vmss list-instances \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $SCALE_SET_NAME \ - --output table -``` - -Results (example): - - - -```text -InstanceId ResourceGroup VmId ProvisioningState Location ------------ ----------------------- ------------------------------------ ----------------- ---------- -1 myResourceGroupxxx e768fb62-0d58-4173-978d-1f564e4a925a Succeeded westus2 -0 myResourceGroupxxx 5a2b34bd-1123-abcd-abcd-1623e0caf234 Succeeded westus2 -``` - -To see additional information about a specific VM instance, use the `az vm show` command: - -```bash -export INSTANCE_NAME=$(az vmss list-instances --resource-group $RESOURCE_GROUP_NAME --name $SCALE_SET_NAME --query "[0].name" -o tsv) - -az vm show --resource-group $RESOURCE_GROUP_NAME --name $INSTANCE_NAME -``` - ---- - -## Change the capacity of a scale set - -By default, two VM instances are created in the scale set. To increase or decrease instances, use the `az vmss scale` command. For example, scale to 3 instances: - -```bash -az vmss scale \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $SCALE_SET_NAME \ - --new-capacity 3 -``` - -Verify the updated instance count: - -```bash -az vmss list-instances \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $SCALE_SET_NAME \ - --output table -``` - -Results: - - - -```text -InstanceId ResourceGroup VmId ProvisioningState Location ------------ ----------------------- ------------------------------------ ----------------- ---------- -2 myResourceGroupxxx 54f68ce0-f123-abcd-abcd-4e6820cabccd Succeeded westus2 -1 myResourceGroupxxx e768fb62-0d58-4173-978d-1f564e4a925a Succeeded westus2 -0 myResourceGroupxxx 5a2b34bd-1123-abcd-abcd-1623e0caf234 Succeeded westus2 -``` - ---- - -## Stop instances in a scale set - -To stop individual VMs in Flexible orchestration mode, retrieve their unique names: - -```bash -export INSTANCE_NAME=$(az vmss list-instances --resource-group $RESOURCE_GROUP_NAME --name $SCALE_SET_NAME --query "[0].name" -o tsv) - -az vm stop \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $INSTANCE_NAME -``` - -For all instances, use: - -```bash -az vmss stop --resource-group $RESOURCE_GROUP_NAME --name $SCALE_SET_NAME -``` - ---- - -## Start instances in a scale set - -To start individual stopped VMs, use: - -```bash -az vm start \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $INSTANCE_NAME -``` - -To start all instances: - -```bash -az vmss start \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $SCALE_SET_NAME -``` - ---- - -## Restart instances in a scale set - -Restart specific instances: - -```bash -az vm restart \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $INSTANCE_NAME -``` - -Or restart all instances: - -```bash -az vmss restart \ - --resource-group $RESOURCE_GROUP_NAME \ - --name $SCALE_SET_NAME -``` - ---- - -## Clean up resources - -When you delete a resource group, all associated resources are deleted: - -```bash -az group delete --name $RESOURCE_GROUP_NAME --no-wait --yes -``` - ---- - -## Next steps - -In this tutorial, you learned how to perform common Virtual Machine Scale Set management tasks with Azure CLI: - -> [!div class="checklist"] -> * Create a resource group. -> * Create a scale set. -> * View and use specific VM sizes. -> * Manually scale a scale set. -> * Perform common management tasks such as stopping, starting, and restarting instances. - -Advance to the next tutorial to learn how to connect to scale set instances: - -> [!div class="nextstepaction"] -> [Use data disks with scale sets](tutorial-connect-to-instances-cli.md) \ No newline at end of file diff --git a/tools/doc.md b/tools/doc.md index 432e59958..8f3051a7e 100644 --- a/tools/doc.md +++ b/tools/doc.md @@ -1,136 +1,507 @@ --- -title: 'Quickstart: Use Terraform to create a Linux VM' -description: In this quickstart, you learn how to use Terraform to create a Linux virtual machine -author: tomarchermsft -ms.service: azure-virtual-machines -ms.collection: linux +title: 'Quickstart: Deploy an Azure Kubernetes Service (AKS) cluster using Azure CLI' +description: Learn how to quickly deploy a Kubernetes cluster and deploy an application in Azure Kubernetes Service (AKS) using Azure CLI. ms.topic: quickstart -ms.date: 07/24/2023 -ms.author: tarcher -ms.custom: devx-track-terraform, linux-related-content -content_well_notification: - - AI-contribution -ai-usage: ai-assisted +ms.date: 04/09/2024 +author: nickomang +ms.author: nickoman +ms.custom: H1Hack27Feb2017, mvc, devcenter, devx-track-azurecli, mode-api, innovation-engine, linux-related-content +#Customer intent: As a developer or cluster operator, I want to deploy an AKS cluster and deploy an application so I can see how to run applications using the managed Kubernetes service in Azure. --- -# Quickstart: Use Terraform to create a Linux VM +# Quickstart: Deploy an Azure Kubernetes Service (AKS) cluster using Azure CLI -**Applies to:** :heavy_check_mark: Linux VMs +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://go.microsoft.com/fwlink/?linkid=2286152) -Article tested with the following Terraform and Terraform provider versions: +Azure Kubernetes Service (AKS) is a managed Kubernetes service that lets you quickly deploy and manage clusters. In this quickstart, you learn how to: -This article shows you how to create a complete Linux environment and supporting resources with Terraform. Those resources include a virtual network, subnet, public IP address, and more. - -[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] - -In this article, you learn how to: -> [!div class="checklist"] -> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet). -> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group). -> * Create a virtual network (VNET) using [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network). -> * Create a subnet using [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet). -> * Create a public IP using [azurerm_public_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip). -> * Create a network security group using [azurerm_network_security_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_group). -> * Create a network interface using [azurerm_network_interface](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface). -> * Create an association between the network security group and the network interface using [azurerm_network_interface_security_group_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface_security_group_association). -> * Generate a random value for a unique storage account name using [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id). -> * Create a storage account for boot diagnostics using [azurerm_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account). -> * Create a Linux VM using [azurerm_linux_virtual_machine](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine) -> * Create an AzAPI resource [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource). -> * Create an AzAPI resource to generate an SSH key pair using [azapi_resource_action](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource_action). - -## Prerequisites - -- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) - -## Implement the Terraform code +- Deploy an AKS cluster using the Azure CLI. +- Run a sample multi-container application with a group of microservices and web front ends simulating a retail scenario. > [!NOTE] -> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure/TestRecord.md). -> -> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) - -1. Create a directory in which to test the sample Terraform code and make it the current directory. +> To get started with quickly provisioning an AKS cluster, this article includes steps to deploy a cluster with default settings for evaluation purposes only. Before deploying a production-ready cluster, we recommend that you familiarize yourself with our [baseline reference architecture][baseline-reference-architecture] to consider how it aligns with your business requirements. -1. Create a file named `providers.tf` and insert the following code: +## Before you begin - :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/providers.tf"::: +This quickstart assumes a basic understanding of Kubernetes concepts. For more information, see [Kubernetes core concepts for Azure Kubernetes Service (AKS)][kubernetes-concepts]. -1. Create a file named `ssh.tf` and insert the following code: +- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] - :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/ssh.tf"::: +[!INCLUDE [azure-cli-prepare-your-environment-no-header.md](~/reusable-content/azure-cli/azure-cli-prepare-your-environment-no-header.md)] -1. Create a file named `main.tf` and insert the following code: +- This article requires version 2.0.64 or later of the Azure CLI. If you're using Azure Cloud Shell, the latest version is already installed there. +- Make sure that the identity you're using to create your cluster has the appropriate minimum permissions. For more details on access and identity for AKS, see [Access and identity options for Azure Kubernetes Service (AKS)](../concepts-identity.md). +- If you have multiple Azure subscriptions, select the appropriate subscription ID in which the resources should be billed using the [az account set](/cli/azure/account#az-account-set) command. For more information, see [How to manage Azure subscriptions – Azure CLI](/cli/azure/manage-azure-subscriptions-azure-cli?tabs=bash#change-the-active-subscription). - :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/main.tf"::: +## Define environment variables -1. Create a file named `variables.tf` and insert the following code: +Define the following environment variables for use throughout this quickstart: - :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/variables.tf"::: +```azurecli-interactive +export RANDOM_ID="$(openssl rand -hex 3)" +export MY_RESOURCE_GROUP_NAME="myAKSResourceGroup$RANDOM_ID" +export REGION="westeurope" +export MY_AKS_CLUSTER_NAME="myAKSCluster$RANDOM_ID" +export MY_DNS_LABEL="mydnslabel$RANDOM_ID" +``` -1. Create a file named `outputs.tf` and insert the following code: +## Create a resource group - :::code language="Terraform" source="~/terraform_samples/quickstart/101-vm-with-infrastructure/outputs.tf"::: +An [Azure resource group][azure-resource-group] is a logical group in which Azure resources are deployed and managed. When you create a resource group, you're prompted to specify a location. This location is the storage location of your resource group metadata and where your resources run in Azure if you don't specify another region during resource creation. -## Initialize Terraform +Create a resource group using the [`az group create`][az-group-create] command. -[!INCLUDE [terraform-init.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-init.md)] +```azurecli-interactive +az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION +``` -## Create a Terraform execution plan +Results: + +```JSON +{ + "id": "/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroups/myAKSResourceGroupxxxxxx", + "location": "eastus", + "managedBy": null, + "name": "testResourceGroup", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` -[!INCLUDE [terraform-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan.md)] +## Create an AKS cluster -## Apply a Terraform execution plan +Create an AKS cluster using the [`az aks create`][az-aks-create] command. The following example creates a cluster with one node and enables a system-assigned managed identity. -[!INCLUDE [terraform-apply-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-apply-plan.md)] +```azurecli-interactive +az aks create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_AKS_CLUSTER_NAME \ + --node-count 1 \ + --generate-ssh-keys +``` -Cost information isn't presented during the virtual machine creation process for Terraform like it is for the [Azure portal](quick-create-portal.md). If you want to learn more about how cost works for virtual machines, see the [Cost optimization Overview page](../plan-to-manage-costs.md). +> [!NOTE] +> When you create a new cluster, AKS automatically creates a second resource group to store the AKS resources. For more information, see [Why are two resource groups created with AKS?](../faq.yml) -## Verify the results +## Connect to the cluster -#### [Azure CLI](#tab/azure-cli) +To manage a Kubernetes cluster, use the Kubernetes command-line client, [kubectl][kubectl]. `kubectl` is already installed if you use Azure Cloud Shell. To install `kubectl` locally, use the [`az aks install-cli`][az-aks-install-cli] command. -1. Get the Azure resource group name. +1. Configure `kubectl` to connect to your Kubernetes cluster using the [az aks get-credentials][az-aks-get-credentials] command. This command downloads credentials and configures the Kubernetes CLI to use them. - ```console - resource_group_name=$(terraform output -raw resource_group_name) + ```azurecli-interactive + az aks get-credentials --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_AKS_CLUSTER_NAME ``` -1. Run [az vm list](/cli/azure/vm#az-vm-list) with a [JMESPath](/cli/azure/query-azure-cli) query to display the names of the virtual machines created in the resource group. +1. Verify the connection to your cluster using the [kubectl get][kubectl-get] command. This command returns a list of the cluster nodes. - ```azurecli - az vm list \ - --resource-group $resource_group_name \ - --query "[].{\"VM Name\":name}" -o table + ```azurecli-interactive + kubectl get nodes ``` -#### [Azure PowerShell](#tab/azure-powershell) +## Deploy the application -1. Get the Azure resource group name. +To deploy the application, you use a manifest file to create all the objects required to run the [AKS Store application](https://github.com/Azure-Samples/aks-store-demo). A [Kubernetes manifest file][kubernetes-deployment] defines a cluster's desired state, such as which container images to run. The manifest includes the following Kubernetes deployments and services: - ```console - $resource_group_name=$(terraform output -raw resource_group_name) - ``` +:::image type="content" source="media/quick-kubernetes-deploy-portal/aks-store-architecture.png" alt-text="Screenshot of Azure Store sample architecture." lightbox="media/quick-kubernetes-deploy-portal/aks-store-architecture.png"::: -1. Run [Get-AzVm](/powershell/module/az.compute/get-azvm) to display the names of all the virtual machines in the resource group. +- **Store front**: Web application for customers to view products and place orders. +- **Product service**: Shows product information. +- **Order service**: Places orders. +- **Rabbit MQ**: Message queue for an order queue. - ```azurepowershell - Get-AzVm -ResourceGroupName $resource_group_name +> [!NOTE] +> We don't recommend running stateful containers, such as Rabbit MQ, without persistent storage for production. These are used here for simplicity, but we recommend using managed services, such as Azure CosmosDB or Azure Service Bus. + +1. Create a file named `aks-store-quickstart.yaml` and copy in the following manifest: + + ```yaml + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: rabbitmq + spec: + serviceName: rabbitmq + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: rabbitmq + image: mcr.microsoft.com/mirror/docker/library/rabbitmq:3.10-management-alpine + ports: + - containerPort: 5672 + name: rabbitmq-amqp + - containerPort: 15672 + name: rabbitmq-http + env: + - name: RABBITMQ_DEFAULT_USER + value: "username" + - name: RABBITMQ_DEFAULT_PASS + value: "password" + resources: + requests: + cpu: 10m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: rabbitmq-enabled-plugins + mountPath: /etc/rabbitmq/enabled_plugins + subPath: enabled_plugins + volumes: + - name: rabbitmq-enabled-plugins + configMap: + name: rabbitmq-enabled-plugins + items: + - key: rabbitmq_enabled_plugins + path: enabled_plugins + --- + apiVersion: v1 + data: + rabbitmq_enabled_plugins: | + [rabbitmq_management,rabbitmq_prometheus,rabbitmq_amqp1_0]. + kind: ConfigMap + metadata: + name: rabbitmq-enabled-plugins + --- + apiVersion: v1 + kind: Service + metadata: + name: rabbitmq + spec: + selector: + app: rabbitmq + ports: + - name: rabbitmq-amqp + port: 5672 + targetPort: 5672 + - name: rabbitmq-http + port: 15672 + targetPort: 15672 + type: ClusterIP + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: order-service + spec: + replicas: 1 + selector: + matchLabels: + app: order-service + template: + metadata: + labels: + app: order-service + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: order-service + image: ghcr.io/azure-samples/aks-store-demo/order-service:latest + ports: + - containerPort: 3000 + env: + - name: ORDER_QUEUE_HOSTNAME + value: "rabbitmq" + - name: ORDER_QUEUE_PORT + value: "5672" + - name: ORDER_QUEUE_USERNAME + value: "username" + - name: ORDER_QUEUE_PASSWORD + value: "password" + - name: ORDER_QUEUE_NAME + value: "orders" + - name: FASTIFY_ADDRESS + value: "0.0.0.0" + resources: + requests: + cpu: 1m + memory: 50Mi + limits: + cpu: 75m + memory: 128Mi + startupProbe: + httpGet: + path: /health + port: 3000 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: 3000 + failureThreshold: 3 + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: 3000 + failureThreshold: 5 + initialDelaySeconds: 3 + periodSeconds: 3 + initContainers: + - name: wait-for-rabbitmq + image: busybox + command: ['sh', '-c', 'until nc -zv rabbitmq 5672; do echo waiting for rabbitmq; sleep 2; done;'] + resources: + requests: + cpu: 1m + memory: 50Mi + limits: + cpu: 75m + memory: 128Mi + --- + apiVersion: v1 + kind: Service + metadata: + name: order-service + spec: + type: ClusterIP + ports: + - name: http + port: 3000 + targetPort: 3000 + selector: + app: order-service + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: product-service + spec: + replicas: 1 + selector: + matchLabels: + app: product-service + template: + metadata: + labels: + app: product-service + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: product-service + image: ghcr.io/azure-samples/aks-store-demo/product-service:latest + ports: + - containerPort: 3002 + env: + - name: AI_SERVICE_URL + value: "http://ai-service:5001/" + resources: + requests: + cpu: 1m + memory: 1Mi + limits: + cpu: 2m + memory: 20Mi + readinessProbe: + httpGet: + path: /health + port: 3002 + failureThreshold: 3 + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: 3002 + failureThreshold: 5 + initialDelaySeconds: 3 + periodSeconds: 3 + --- + apiVersion: v1 + kind: Service + metadata: + name: product-service + spec: + type: ClusterIP + ports: + - name: http + port: 3002 + targetPort: 3002 + selector: + app: product-service + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: store-front + spec: + replicas: 1 + selector: + matchLabels: + app: store-front + template: + metadata: + labels: + app: store-front + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: store-front + image: ghcr.io/azure-samples/aks-store-demo/store-front:latest + ports: + - containerPort: 8080 + name: store-front + env: + - name: VUE_APP_ORDER_SERVICE_URL + value: "http://order-service:3000/" + - name: VUE_APP_PRODUCT_SERVICE_URL + value: "http://product-service:3002/" + resources: + requests: + cpu: 1m + memory: 200Mi + limits: + cpu: 1000m + memory: 512Mi + startupProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 3 + initialDelaySeconds: 3 + periodSeconds: 3 + livenessProbe: + httpGet: + path: /health + port: 8080 + failureThreshold: 5 + initialDelaySeconds: 3 + periodSeconds: 3 + --- + apiVersion: v1 + kind: Service + metadata: + name: store-front + spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: store-front + type: LoadBalancer ``` ---- + For a breakdown of YAML manifest files, see [Deployments and YAML manifests](../concepts-clusters-workloads.md#deployments-and-yaml-manifests). -## Clean up resources + If you create and save the YAML file locally, then you can upload the manifest file to your default directory in CloudShell by selecting the **Upload/Download files** button and selecting the file from your local file system. -[!INCLUDE [terraform-plan-destroy.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan-destroy.md)] +1. Deploy the application using the [`kubectl apply`][kubectl-apply] command and specify the name of your YAML manifest. -## Troubleshoot Terraform on Azure + ```azurecli-interactive + kubectl apply -f aks-store-quickstart.yaml + ``` -[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) +## Test the application + +You can validate that the application is running by visiting the public IP address or the application URL. + +Get the application URL using the following commands: + +```azurecli-interactive +runtime="5 minutes" +endtime=$(date -ud "$runtime" +%s) +while [[ $(date -u +%s) -le $endtime ]] +do + STATUS=$(kubectl get pods -l app=store-front -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') + echo $STATUS + if [ "$STATUS" == 'True' ] + then + export IP_ADDRESS=$(kubectl get service store-front --output 'jsonpath={..status.loadBalancer.ingress[0].ip}') + echo "Service IP Address: $IP_ADDRESS" + break + else + sleep 10 + fi +done +``` + +```azurecli-interactive +curl $IP_ADDRESS +``` + +Results: + +```HTML + + + + + + + + store-front + + + + + +
+ + +``` + +```OUTPUT +echo "You can now visit your web server at $IP_ADDRESS" +``` + +:::image type="content" source="media/quick-kubernetes-deploy-cli/aks-store-application.png" alt-text="Screenshot of AKS Store sample application." lightbox="media/quick-kubernetes-deploy-cli/aks-store-application.png"::: + +## Delete the cluster + +If you don't plan on going through the [AKS tutorial][aks-tutorial], clean up unnecessary resources to avoid Azure charges. You can remove the resource group, container service, and all related resources using the [`az group delete`][az-group-delete] command. + +> [!NOTE] +> The AKS cluster was created with a system-assigned managed identity, which is the default identity option used in this quickstart. The platform manages this identity so you don't need to manually remove it. ## Next steps -In this quickstart, you deployed a simple virtual machine using Terraform. To learn more about Azure virtual machines, continue to the tutorial for Linux VMs. +In this quickstart, you deployed a Kubernetes cluster and then deployed a simple multi-container application to it. This sample application is for demo purposes only and doesn't represent all the best practices for Kubernetes applications. For guidance on creating full solutions with AKS for production, see [AKS solution guidance][aks-solution-guidance]. + +To learn more about AKS and walk through a complete code-to-deployment example, continue to the Kubernetes cluster tutorial. > [!div class="nextstepaction"] -> [Azure Linux virtual machine tutorials](./tutorial-manage-vm.md) \ No newline at end of file +> [AKS tutorial][aks-tutorial] + + +[kubectl]: https://kubernetes.io/docs/reference/kubectl/ +[kubectl-apply]: https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply +[kubectl-get]: https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#get + + +[kubernetes-concepts]: ../concepts-clusters-workloads.md +[aks-tutorial]: ../tutorial-kubernetes-prepare-app.md +[azure-resource-group]: /azure/azure-resource-manager/management/overview +[az-aks-create]: /cli/azure/aks#az-aks-create +[az-aks-get-credentials]: /cli/azure/aks#az-aks-get-credentials +[az-aks-install-cli]: /cli/azure/aks#az-aks-install-cli +[az-group-create]: /cli/azure/group#az-group-create +[az-group-delete]: /cli/azure/group#az-group-delete +[kubernetes-deployment]: ../concepts-clusters-workloads.md#deployments-and-yaml-manifests +[aks-solution-guidance]: /azure/architecture/reference-architectures/containers/aks-start-here?toc=/azure/aks/toc.json&bc=/azure/aks/breadcrumb/toc.json +[baseline-reference-architecture]: /azure/architecture/reference-architectures/containers/aks/baseline-aks?toc=/azure/aks/toc.json&bc=/azure/aks/breadcrumb/toc.json \ No newline at end of file diff --git a/tools/execution_log.csv b/tools/execution_log.csv index d54b48d7a..b3e251a79 100644 --- a/tools/execution_log.csv +++ b/tools/execution_log.csv @@ -220,3 +220,9 @@ Error: %!s() StdErr: The 'ie test' command timed out after 11 minutes.",5596.252681970596,Failure +2025-03-05 00:02:15,workload_description,create a linux vm and ssh into it,Deploy Linux VM and SSH into Instance.md,1,"time=2025-03-05T00:00:35-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. +' +StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image.",153.62026572227478,Success diff --git a/tools/generated_exec_doc.md b/tools/generated_exec_doc.md deleted file mode 100644 index 4018bcab8..000000000 --- a/tools/generated_exec_doc.md +++ /dev/null @@ -1,3 +0,0 @@ -It seems you've requested a workload titled "new.py," but no content or details are provided for this workload. Please provide more details or specify the objective of the Exec Doc you'd like me to create (e.g., Do you want to deploy a specific resource on Azure, set up a CI/CD pipeline, work with a particular Azure service like Virtual Machines, Kubernetes, Databases, etc.?). - -Once you provide this information, I can create a fully detailed and functional Exec Doc adherent to the rules mentioned above. \ No newline at end of file diff --git a/tools/generated_exec_doccc.md b/tools/generated_exec_doccc.md deleted file mode 100644 index 0c8a95dc5..000000000 --- a/tools/generated_exec_doccc.md +++ /dev/null @@ -1,272 +0,0 @@ ---- -title: Highly Available Kubernetes Cluster with AKS, Application Gateway, Monitor, and Key Vault -description: This Exec Doc demonstrates how to deploy a highly available Azure Kubernetes Service (AKS) cluster integrated with Azure Application Gateway for Ingress, Azure Monitor for observability, and Azure Key Vault for managing secrets. -ms.topic: quickstart -ms.date: 10/11/2023 -author: azureuser -ms.author: azurealias -ms.custom: innovation-engine, azurecli, kubernetes, monitoring ---- - -# Highly Available Kubernetes Cluster with AKS, Application Gateway, Monitor, and Key Vault - -This Exec Doc walks you through the deployment of a highly available AKS cluster integrated with an Azure Application Gateway used for Ingress, Azure Monitor for observability, and Azure Key Vault for securely managing secrets. Each section includes code blocks with environment variable declarations and inline explanations that automate the cloud infrastructure deployment and help you learn as you go. - -## Overview of the Deployment - -In this workflow, we perform the following steps: - -1. Create a resource group. -2. Create a dedicated virtual network and subnet for the Application Gateway. -3. Deploy an Azure Application Gateway. -4. Update the Application Gateway routing rule to assign an explicit priority. -5. Create an Azure Key Vault to manage secrets. -6. Retrieve the Application Gateway resource ID for integration. -7. Deploy an AKS cluster with: - - Ingress add-on integration with the Application Gateway. - - Monitoring add-on for Azure Monitor. -8. Enable the Azure Key Vault secrets provider add-on on the AKS cluster. - -For all resources that require unique names, a randomly generated suffix is appended. Code blocks are of type "bash" ensuring that they are executable via Innovation Engine. - -## Step 1: Create a Resource Group - -We start by defining our environment variables and creating a resource group to contain all the resources used in this deployment. - -```bash -export REGION="WestUS2" -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export RG_NAME="MyAKSResourceGroup$RANDOM_SUFFIX" -az group create --name $RG_NAME --location $REGION -``` - -Results: - - -```JSON -{ - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx", - "location": "westus2", - "managedBy": null, - "name": "MyAKSResourceGroupxxxxxx", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" -} -``` - -## Step 2: Create a Virtual Network for the Application Gateway - -Next, we create a virtual network and a dedicated subnet for our Application Gateway. This isolation ensures that the Application Gateway is deployed within its own network segment. - -```bash -export VNET_NAME="MyVnet$RANDOM_SUFFIX" -export SUBNET_NAME="AppGwSubnet" -az network vnet create --resource-group $RG_NAME --name $VNET_NAME --address-prefix 10.0.0.0/16 --subnet-name $SUBNET_NAME --subnet-prefix 10.0.1.0/24 -``` - -Results: - - -```JSON -{ - "newVNet": true, - "subnets": [ - { - "addressPrefix": "10.0.1.0/24", - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/virtualNetworks/MyVnetxxxxxx/subnets/AppGwSubnet", - "name": "AppGwSubnet" - } - ] -} -``` - -## Step 3: Deploy the Azure Application Gateway - -We deploy the Application Gateway using the Standard_V2 SKU for high availability and scalability. The default request routing rule "rule1" is automatically created but without a priority, which must be rectified for newer API versions. - -```bash -export AAGW_NAME="MyAppGateway$RANDOM_SUFFIX" -az network application-gateway create --name $AAGW_NAME --resource-group $RG_NAME --location $REGION --sku Standard_V2 --capacity 2 --vnet-name $VNET_NAME --subnet $SUBNET_NAME --http-settings-port 80 -``` - -Results: - - -```JSON -{ - "applicationGateway": { - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx", - "location": "westus2", - "name": "MyAppGatewayxxxxxx", - "provisioningState": "Succeeded", - "sku": { - "capacity": 2, - "name": "Standard_V2" - }, - "type": "Microsoft.Network/applicationGateways" - } -} -``` - -## Step 4: Update the Application Gateway Routing Rule Priority - -Instead of deleting and recreating the default rule, we update the existing request routing rule "rule1" to assign it an explicit priority. This addresses the error regarding an empty priority field required by API versions starting from 2021-08-01. - -```bash -# Wait until the Application Gateway is fully provisioned. -az network application-gateway wait --name $AAGW_NAME --resource-group $RG_NAME --created - -# Update the default request routing rule (rule1) with an explicit priority. -az network application-gateway rule update --resource-group $RG_NAME --gateway-name $AAGW_NAME --name rule1 --priority 1 -``` - -Results: - - -```JSON -{ - "name": "rule1", - "priority": 1, - "ruleType": "Basic", - "httpListener": { - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx/httpListeners/appGatewayHttpListener" - }, - "backendAddressPool": { - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx/backendAddressPools/BackendAddressPool_1" - }, - "backendHttpSettings": { - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx/backendHttpSettingsCollection/appGatewayBackendHttpSettings" - } -} -``` - -## Step 5: Create an Azure Key Vault - -Create an Azure Key Vault to securely store and manage application secrets and certificates. The Key Vault integration with AKS allows your cluster to securely retrieve secrets when needed. - -```bash -export KEYVAULT_NAME="myKeyVault$RANDOM_SUFFIX" -az keyvault create --name $KEYVAULT_NAME --resource-group $RG_NAME --location $REGION -``` - -Results: - - -```JSON -{ - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceVaults/myKeyVaultxxxxxx", - "location": "westus2", - "name": "myKeyVaultxxxxxx", - "properties": { - "sku": { - "family": "A", - "name": "standard" - }, - "tenantId": "xxxxx-xxxxx-xxxxx-xxxxx", - "accessPolicies": [] - }, - "type": "Microsoft.KeyVault/vaults" -} -``` - -## Step 6: Retrieve Application Gateway Resource ID - -Before deploying the AKS cluster, retrieve the Application Gateway resource ID. This ID is required for integrating the Application Gateway Ingress add-on with AKS. - -```bash -export AAGW_ID=$(az network application-gateway show --name $AAGW_NAME --resource-group $RG_NAME --query id -o tsv) -echo $AAGW_ID -``` - -Results: - - -```text -/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx -``` - -## Step 7: Deploy the AKS Cluster with Ingress and Monitoring Add-ons - -Deploy the AKS cluster using three nodes. The cluster is integrated with the Application Gateway Ingress add-on using the Application Gateway resource ID obtained in the previous step. Additionally, the monitoring add-on is enabled for integration with Azure Monitor. - -```bash -export AKS_CLUSTER_NAME="MyAKSCluster$RANDOM_SUFFIX" -az aks create --resource-group $RG_NAME --name $AKS_CLUSTER_NAME --node-count 3 --enable-addons ingress-appgw,monitoring --appgw-id $AAGW_ID --network-plugin azure --location $REGION --generate-ssh-keys -``` - -Results: - - -```JSON -{ - "aadProfile": null, - "addonProfiles": { - "ingressApplicationGateway": { - "config": { - "appgwId": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx" - }, - "enabled": true, - "identity": {} - }, - "omsagent": { - "config": { - "logAnalyticsWorkspaceResourceID": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.OperationalInsights/workspaces/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus" - }, - "enabled": true - } - }, - "dnsPrefix": "myaksclustercxxxxxx", - "enableRBAC": true, - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.ContainerService/managedClusters/MyAKSClusterxxxxxx", - "location": "westus2", - "name": "MyAKSClusterxxxxxx", - "provisioningState": "Succeeded", - "resourceGroup": "MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus", - "type": "Microsoft.ContainerService/managedClusters" -} -``` - -## Step 8: Enable Azure Key Vault Secrets Provider Add-on on AKS - -Integrate the AKS cluster with Azure Key Vault by enabling the Azure Key Vault secrets provider add-on. This add-on securely mounts secrets stored in Azure Key Vault as volumes within your pods. - -```bash -az aks enable-addons --addons azure-keyvault-secrets-provider --name $AKS_CLUSTER_NAME --resource-group $RG_NAME -``` - -Results: - - -```JSON -{ - "addonProfiles": { - "azureKeyvaultSecretsProvider": { - "config": {}, - "enabled": true - }, - "ingressApplicationGateway": { - "config": { - "appgwId": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/MyAKSResourceGroupxxxxxx/providers/Microsoft.Network/applicationGateways/MyAppGatewayxxxxxx" - }, - "enabled": true - }, - "omsagent": { - "config": { - "logAnalyticsWorkspaceResourceID": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.OperationalInsights/workspaces/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus" - }, - "enabled": true - } - }, - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourcegroups/MC_MyAKSResourceGroupxxxxxx_myaksclustercxxxxxx_eastus/providers/Microsoft.ContainerService/managedClusters/MyAKSClusterxxxxxx", - "name": "MyAKSClusterxxxxxx" -} -``` - -## Summary - -In this Exec Doc, you deployed a highly available AKS cluster integrated with an Application Gateway used for Ingress, Azure Monitor for observability, and Azure Key Vault for secure secret management. A dedicated virtual network was created for the Application Gateway, and after the gateway was provisioned, the default Application Gateway routing rule was updated to include a defined priority—thereby addressing the API validation requirement. With clearly defined environment variables and inline explanations, you can now deploy this production-grade infrastructure using the Innovation Engine without encountering deployment errors. - -Feel free to execute these commands step-by-step in your preferred Azure CLI environment. \ No newline at end of file From 4ef52ee54e2ed0818ebff3b5f1f16d0afa55de21 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 5 Mar 2025 21:01:53 -0800 Subject: [PATCH 11/87] updated tool --- scenarios/metadata.json | 2 +- tools/aks-store-quickstart.yaml | 286 --------- tools/doc.md | 757 ++++++++++-------------- tools/execution_log.csv | 995 ++++++++++++++++++++++++++++++++ 4 files changed, 1302 insertions(+), 738 deletions(-) delete mode 100644 tools/aks-store-quickstart.yaml diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 6fbc3ea19..64d3ef194 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -18,7 +18,7 @@ } ], "configurations": { - "permissions": [Spee + "permissions": [ "Microsoft.Resources/resourceGroups/write", "Microsoft.Resources/resourceGroups/read", "Microsoft.Network/virtualNetworks/write", diff --git a/tools/aks-store-quickstart.yaml b/tools/aks-store-quickstart.yaml deleted file mode 100644 index 2aaaf609d..000000000 --- a/tools/aks-store-quickstart.yaml +++ /dev/null @@ -1,286 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: rabbitmq -spec: - serviceName: rabbitmq - replicas: 1 - selector: - matchLabels: - app: rabbitmq - template: - metadata: - labels: - app: rabbitmq - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: rabbitmq - image: mcr.microsoft.com/mirror/docker/library/rabbitmq:3.10-management-alpine - ports: - - containerPort: 5672 - name: rabbitmq-amqp - - containerPort: 15672 - name: rabbitmq-http - env: - - name: RABBITMQ_DEFAULT_USER - value: "username" - - name: RABBITMQ_DEFAULT_PASS - value: "password" - resources: - requests: - cpu: 10m - memory: 128Mi - limits: - cpu: 250m - memory: 256Mi - volumeMounts: - - name: rabbitmq-enabled-plugins - mountPath: /etc/rabbitmq/enabled_plugins - subPath: enabled_plugins - volumes: - - name: rabbitmq-enabled-plugins - configMap: - name: rabbitmq-enabled-plugins - items: - - key: rabbitmq_enabled_plugins - path: enabled_plugins ---- -apiVersion: v1 -data: - rabbitmq_enabled_plugins: | - [rabbitmq_management,rabbitmq_prometheus,rabbitmq_amqp1_0]. -kind: ConfigMap -metadata: - name: rabbitmq-enabled-plugins ---- -apiVersion: v1 -kind: Service -metadata: - name: rabbitmq -spec: - selector: - app: rabbitmq - ports: - - name: rabbitmq-amqp - port: 5672 - targetPort: 5672 - - name: rabbitmq-http - port: 15672 - targetPort: 15672 - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: order-service -spec: - replicas: 1 - selector: - matchLabels: - app: order-service - template: - metadata: - labels: - app: order-service - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: order-service - image: ghcr.io/azure-samples/aks-store-demo/order-service:latest - ports: - - containerPort: 3000 - env: - - name: ORDER_QUEUE_HOSTNAME - value: "rabbitmq" - - name: ORDER_QUEUE_PORT - value: "5672" - - name: ORDER_QUEUE_USERNAME - value: "username" - - name: ORDER_QUEUE_PASSWORD - value: "password" - - name: ORDER_QUEUE_NAME - value: "orders" - - name: FASTIFY_ADDRESS - value: "0.0.0.0" - resources: - requests: - cpu: 1m - memory: 50Mi - limits: - cpu: 75m - memory: 128Mi - startupProbe: - httpGet: - path: /health - port: 3000 - failureThreshold: 5 - initialDelaySeconds: 20 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /health - port: 3000 - failureThreshold: 3 - initialDelaySeconds: 3 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /health - port: 3000 - failureThreshold: 5 - initialDelaySeconds: 3 - periodSeconds: 3 - initContainers: - - name: wait-for-rabbitmq - image: busybox - command: ['sh', '-c', 'until nc -zv rabbitmq 5672; do echo waiting for rabbitmq; sleep 2; done;'] - resources: - requests: - cpu: 1m - memory: 50Mi - limits: - cpu: 75m - memory: 128Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: order-service -spec: - type: ClusterIP - ports: - - name: http - port: 3000 - targetPort: 3000 - selector: - app: order-service ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: product-service -spec: - replicas: 1 - selector: - matchLabels: - app: product-service - template: - metadata: - labels: - app: product-service - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: product-service - image: ghcr.io/azure-samples/aks-store-demo/product-service:latest - ports: - - containerPort: 3002 - env: - - name: AI_SERVICE_URL - value: "http://ai-service:5001/" - resources: - requests: - cpu: 1m - memory: 1Mi - limits: - cpu: 2m - memory: 20Mi - readinessProbe: - httpGet: - path: /health - port: 3002 - failureThreshold: 3 - initialDelaySeconds: 3 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /health - port: 3002 - failureThreshold: 5 - initialDelaySeconds: 3 - periodSeconds: 3 ---- -apiVersion: v1 -kind: Service -metadata: - name: product-service -spec: - type: ClusterIP - ports: - - name: http - port: 3002 - targetPort: 3002 - selector: - app: product-service ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: store-front -spec: - replicas: 1 - selector: - matchLabels: - app: store-front - template: - metadata: - labels: - app: store-front - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: store-front - image: ghcr.io/azure-samples/aks-store-demo/store-front:latest - ports: - - containerPort: 8080 - name: store-front - env: - - name: VUE_APP_ORDER_SERVICE_URL - value: "http://order-service:3000/" - - name: VUE_APP_PRODUCT_SERVICE_URL - value: "http://product-service:3002/" - resources: - requests: - cpu: 1m - memory: 200Mi - limits: - cpu: 1000m - memory: 512Mi - startupProbe: - httpGet: - path: /health - port: 8080 - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 5 - readinessProbe: - httpGet: - path: /health - port: 8080 - failureThreshold: 3 - initialDelaySeconds: 3 - periodSeconds: 3 - livenessProbe: - httpGet: - path: /health - port: 8080 - failureThreshold: 5 - initialDelaySeconds: 3 - periodSeconds: 3 ---- -apiVersion: v1 -kind: Service -metadata: - name: store-front -spec: - ports: - - port: 80 - targetPort: 8080 - selector: - app: store-front - type: LoadBalancer diff --git a/tools/doc.md b/tools/doc.md index 8f3051a7e..63f5bf371 100644 --- a/tools/doc.md +++ b/tools/doc.md @@ -1,507 +1,362 @@ --- -title: 'Quickstart: Deploy an Azure Kubernetes Service (AKS) cluster using Azure CLI' -description: Learn how to quickly deploy a Kubernetes cluster and deploy an application in Azure Kubernetes Service (AKS) using Azure CLI. +title: 'Quickstart: Use Terraform to create a Linux VM' +description: In this quickstart, you learn how to use Terraform to create a Linux virtual machine +author: tomarchermsft +ms.service: azure-virtual-machines +ms.collection: linux ms.topic: quickstart -ms.date: 04/09/2024 -author: nickomang -ms.author: nickoman -ms.custom: H1Hack27Feb2017, mvc, devcenter, devx-track-azurecli, mode-api, innovation-engine, linux-related-content -#Customer intent: As a developer or cluster operator, I want to deploy an AKS cluster and deploy an application so I can see how to run applications using the managed Kubernetes service in Azure. +ms.date: 07/24/2023 +ms.author: tarcher +ms.custom: devx-track-terraform, linux-related-content +content_well_notification: + - AI-contribution +ai-usage: ai-assisted --- -# Quickstart: Deploy an Azure Kubernetes Service (AKS) cluster using Azure CLI +# Quickstart: Use Terraform to create a Linux VM -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://go.microsoft.com/fwlink/?linkid=2286152) +**Applies to:** :heavy_check_mark: Linux VMs -Azure Kubernetes Service (AKS) is a managed Kubernetes service that lets you quickly deploy and manage clusters. In this quickstart, you learn how to: +Article tested with the following Terraform and Terraform provider versions: -- Deploy an AKS cluster using the Azure CLI. -- Run a sample multi-container application with a group of microservices and web front ends simulating a retail scenario. +This article shows you how to create a complete Linux environment and supporting resources with Terraform. Those resources include a virtual network, subnet, public IP address, and more. -> [!NOTE] -> To get started with quickly provisioning an AKS cluster, this article includes steps to deploy a cluster with default settings for evaluation purposes only. Before deploying a production-ready cluster, we recommend that you familiarize yourself with our [baseline reference architecture][baseline-reference-architecture] to consider how it aligns with your business requirements. +[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] -## Before you begin +In this article, you learn how to: +> [!div class="checklist"] +> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet). +> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group). +> * Create a virtual network (VNET) using [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network). +> * Create a subnet using [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet). +> * Create a public IP using [azurerm_public_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip). +> * Create a network security group using [azurerm_network_security_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_group). +> * Create a network interface using [azurerm_network_interface](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface). +> * Create an association between the network security group and the network interface using [azurerm_network_interface_security_group_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface_security_group_association). +> * Generate a random value for a unique storage account name using [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id). +> * Create a storage account for boot diagnostics using [azurerm_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account). +> * Create a Linux VM using [azurerm_linux_virtual_machine](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine) +> * Create an AzAPI resource [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource). +> * Create an AzAPI resource to generate an SSH key pair using [azapi_resource_action](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource_action). -This quickstart assumes a basic understanding of Kubernetes concepts. For more information, see [Kubernetes core concepts for Azure Kubernetes Service (AKS)][kubernetes-concepts]. +## Prerequisites -- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] +- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) -[!INCLUDE [azure-cli-prepare-your-environment-no-header.md](~/reusable-content/azure-cli/azure-cli-prepare-your-environment-no-header.md)] +## Implement the Terraform code -- This article requires version 2.0.64 or later of the Azure CLI. If you're using Azure Cloud Shell, the latest version is already installed there. -- Make sure that the identity you're using to create your cluster has the appropriate minimum permissions. For more details on access and identity for AKS, see [Access and identity options for Azure Kubernetes Service (AKS)](../concepts-identity.md). -- If you have multiple Azure subscriptions, select the appropriate subscription ID in which the resources should be billed using the [az account set](/cli/azure/account#az-account-set) command. For more information, see [How to manage Azure subscriptions – Azure CLI](/cli/azure/manage-azure-subscriptions-azure-cli?tabs=bash#change-the-active-subscription). +> [!NOTE] +> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure/TestRecord.md). +> +> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) + +1. Create a directory in which to test the sample Terraform code and make it the current directory. + +1. Create a file named `providers.tf` and insert the following code: + + ```terraform + terraform { + required_version = ">=0.12" + + required_providers { + azapi = { + source = "azure/azapi" + version = "~>1.5" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } + } + + provider "azurerm" { + features {} + } + ``` -## Define environment variables +1. Create a file named `ssh.tf` and insert the following code: + + ```terraform + resource "random_pet" "ssh_key_name" { + prefix = "ssh" + separator = "" + } + + resource "azapi_resource_action" "ssh_public_key_gen" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + resource_id = azapi_resource.ssh_public_key.id + action = "generateKeyPair" + method = "POST" + + response_export_values = ["publicKey", "privateKey"] + } + + resource "azapi_resource" "ssh_public_key" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + name = random_pet.ssh_key_name.id + location = azurerm_resource_group.rg.location + parent_id = azurerm_resource_group.rg.id + } + + output "key_data" { + value = azapi_resource_action.ssh_public_key_gen.output.publicKey + } + ``` -Define the following environment variables for use throughout this quickstart: +1. Create a file named `main.tf` and insert the following code: + + ```terraform + resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix + } + + resource "azurerm_resource_group" "rg" { + location = var.resource_group_location + name = random_pet.rg_name.id + } + + # Create virtual network + resource "azurerm_virtual_network" "my_terraform_network" { + name = "myVnet" + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + } + + # Create subnet + resource "azurerm_subnet" "my_terraform_subnet" { + name = "mySubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.my_terraform_network.name + address_prefixes = ["10.0.1.0/24"] + } + + # Create public IPs + resource "azurerm_public_ip" "my_terraform_public_ip" { + name = "myPublicIP" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + allocation_method = "Dynamic" + } + + # Create Network Security Group and rule + resource "azurerm_network_security_group" "my_terraform_nsg" { + name = "myNetworkSecurityGroup" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + security_rule { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } + } + + # Create network interface + resource "azurerm_network_interface" "my_terraform_nic" { + name = "myNIC" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + ip_configuration { + name = "my_nic_configuration" + subnet_id = azurerm_subnet.my_terraform_subnet.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.my_terraform_public_ip.id + } + } + + # Connect the security group to the network interface + resource "azurerm_network_interface_security_group_association" "example" { + network_interface_id = azurerm_network_interface.my_terraform_nic.id + network_security_group_id = azurerm_network_security_group.my_terraform_nsg.id + } + + # Generate random text for a unique storage account name + resource "random_id" "random_id" { + keepers = { + # Generate a new ID only when a new resource group is defined + resource_group = azurerm_resource_group.rg.name + } + + byte_length = 8 + } + + # Create storage account for boot diagnostics + resource "azurerm_storage_account" "my_storage_account" { + name = "diag${random_id.random_id.hex}" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + account_tier = "Standard" + account_replication_type = "LRS" + } + + # Create virtual machine + resource "azurerm_linux_virtual_machine" "my_terraform_vm" { + name = "myVM" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + network_interface_ids = [azurerm_network_interface.my_terraform_nic.id] + size = "Standard_DS1_v2" + + os_disk { + name = "myOsDisk" + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-gen2" + version = "latest" + } + + computer_name = "hostname" + admin_username = var.username + + admin_ssh_key { + username = var.username + public_key = azapi_resource_action.ssh_public_key_gen.output.publicKey + } + + boot_diagnostics { + storage_account_uri = azurerm_storage_account.my_storage_account.primary_blob_endpoint + } + } + ``` -```azurecli-interactive -export RANDOM_ID="$(openssl rand -hex 3)" -export MY_RESOURCE_GROUP_NAME="myAKSResourceGroup$RANDOM_ID" -export REGION="westeurope" -export MY_AKS_CLUSTER_NAME="myAKSCluster$RANDOM_ID" -export MY_DNS_LABEL="mydnslabel$RANDOM_ID" -``` +1. Create a file named `variables.tf` and insert the following code: + + ```terraform + variable "resource_group_location" { + type = string + default = "eastus" + description = "Location of the resource group." + } + + variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." + } + + variable "username" { + type = string + description = "The username for the local account that will be created on the new VM." + default = "azureadmin" + } + ``` -## Create a resource group +1. Create a file named `outputs.tf` and insert the following code: -An [Azure resource group][azure-resource-group] is a logical group in which Azure resources are deployed and managed. When you create a resource group, you're prompted to specify a location. This location is the storage location of your resource group metadata and where your resources run in Azure if you don't specify another region during resource creation. + ```terraform + output "resource_group_name" { + value = azurerm_resource_group.rg.name + } -Create a resource group using the [`az group create`][az-group-create] command. + output "public_ip_address" { + value = azurerm_linux_virtual_machine.my_terraform_vm.public_ip_address + } + ``` -```azurecli-interactive -az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION -``` +## Initialize Terraform -Results: - -```JSON -{ - "id": "/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroups/myAKSResourceGroupxxxxxx", - "location": "eastus", - "managedBy": null, - "name": "testResourceGroup", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" -} +Run terraform init to initialize the Terraform deployment. This command downloads the Azure provider required to manage your Azure resources. + +```bash +terraform init -upgrade ``` -## Create an AKS cluster +Key points: + +- The -upgrade parameter upgrades the necessary provider plugins to the newest version that complies with the configuration's version constraints. -Create an AKS cluster using the [`az aks create`][az-aks-create] command. The following example creates a cluster with one node and enables a system-assigned managed identity. +## Create a Terraform execution plan -```azurecli-interactive -az aks create \ - --resource-group $MY_RESOURCE_GROUP_NAME \ - --name $MY_AKS_CLUSTER_NAME \ - --node-count 1 \ - --generate-ssh-keys +Run terraform plan to create an execution plan. + +```bash +terraform plan -out main.tfplan ``` -> [!NOTE] -> When you create a new cluster, AKS automatically creates a second resource group to store the AKS resources. For more information, see [Why are two resource groups created with AKS?](../faq.yml) +Key points: -## Connect to the cluster +- The terraform plan command creates an execution plan, but doesn't execute it. Instead, it determines what actions are necessary to create the configuration specified in your configuration files. This pattern allows you to verify whether the execution plan matches your expectations before making any changes to actual resources. +- The optional -out parameter allows you to specify an output file for the plan. Using the -out parameter ensures that the plan you reviewed is exactly what is applied. -To manage a Kubernetes cluster, use the Kubernetes command-line client, [kubectl][kubectl]. `kubectl` is already installed if you use Azure Cloud Shell. To install `kubectl` locally, use the [`az aks install-cli`][az-aks-install-cli] command. +## Apply a Terraform execution plan -1. Configure `kubectl` to connect to your Kubernetes cluster using the [az aks get-credentials][az-aks-get-credentials] command. This command downloads credentials and configures the Kubernetes CLI to use them. +Run terraform apply to apply the execution plan to your cloud infrastructure. - ```azurecli-interactive - az aks get-credentials --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_AKS_CLUSTER_NAME - ``` +```bash +terraform apply main.tfplan +``` -1. Verify the connection to your cluster using the [kubectl get][kubectl-get] command. This command returns a list of the cluster nodes. +Key points: - ```azurecli-interactive - kubectl get nodes - ``` +- The example terraform apply command assumes you previously ran terraform plan -out main.tfplan. +- If you specified a different filename for the -out parameter, use that same filename in the call to terraform apply. +- If you didn't use the -out parameter, call terraform apply without any parameters. -## Deploy the application +Cost information isn't presented during the virtual machine creation process for Terraform like it is for the [Azure portal](quick-create-portal.md). If you want to learn more about how cost works for virtual machines, see the [Cost optimization Overview page](../plan-to-manage-costs.md). -To deploy the application, you use a manifest file to create all the objects required to run the [AKS Store application](https://github.com/Azure-Samples/aks-store-demo). A [Kubernetes manifest file][kubernetes-deployment] defines a cluster's desired state, such as which container images to run. The manifest includes the following Kubernetes deployments and services: +## Verify the results -:::image type="content" source="media/quick-kubernetes-deploy-portal/aks-store-architecture.png" alt-text="Screenshot of Azure Store sample architecture." lightbox="media/quick-kubernetes-deploy-portal/aks-store-architecture.png"::: +#### [Azure CLI](#tab/azure-cli) -- **Store front**: Web application for customers to view products and place orders. -- **Product service**: Shows product information. -- **Order service**: Places orders. -- **Rabbit MQ**: Message queue for an order queue. +1. Get the Azure resource group name. -> [!NOTE] -> We don't recommend running stateful containers, such as Rabbit MQ, without persistent storage for production. These are used here for simplicity, but we recommend using managed services, such as Azure CosmosDB or Azure Service Bus. - -1. Create a file named `aks-store-quickstart.yaml` and copy in the following manifest: - - ```yaml - apiVersion: apps/v1 - kind: StatefulSet - metadata: - name: rabbitmq - spec: - serviceName: rabbitmq - replicas: 1 - selector: - matchLabels: - app: rabbitmq - template: - metadata: - labels: - app: rabbitmq - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: rabbitmq - image: mcr.microsoft.com/mirror/docker/library/rabbitmq:3.10-management-alpine - ports: - - containerPort: 5672 - name: rabbitmq-amqp - - containerPort: 15672 - name: rabbitmq-http - env: - - name: RABBITMQ_DEFAULT_USER - value: "username" - - name: RABBITMQ_DEFAULT_PASS - value: "password" - resources: - requests: - cpu: 10m - memory: 128Mi - limits: - cpu: 250m - memory: 256Mi - volumeMounts: - - name: rabbitmq-enabled-plugins - mountPath: /etc/rabbitmq/enabled_plugins - subPath: enabled_plugins - volumes: - - name: rabbitmq-enabled-plugins - configMap: - name: rabbitmq-enabled-plugins - items: - - key: rabbitmq_enabled_plugins - path: enabled_plugins - --- - apiVersion: v1 - data: - rabbitmq_enabled_plugins: | - [rabbitmq_management,rabbitmq_prometheus,rabbitmq_amqp1_0]. - kind: ConfigMap - metadata: - name: rabbitmq-enabled-plugins - --- - apiVersion: v1 - kind: Service - metadata: - name: rabbitmq - spec: - selector: - app: rabbitmq - ports: - - name: rabbitmq-amqp - port: 5672 - targetPort: 5672 - - name: rabbitmq-http - port: 15672 - targetPort: 15672 - type: ClusterIP - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: order-service - spec: - replicas: 1 - selector: - matchLabels: - app: order-service - template: - metadata: - labels: - app: order-service - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: order-service - image: ghcr.io/azure-samples/aks-store-demo/order-service:latest - ports: - - containerPort: 3000 - env: - - name: ORDER_QUEUE_HOSTNAME - value: "rabbitmq" - - name: ORDER_QUEUE_PORT - value: "5672" - - name: ORDER_QUEUE_USERNAME - value: "username" - - name: ORDER_QUEUE_PASSWORD - value: "password" - - name: ORDER_QUEUE_NAME - value: "orders" - - name: FASTIFY_ADDRESS - value: "0.0.0.0" - resources: - requests: - cpu: 1m - memory: 50Mi - limits: - cpu: 75m - memory: 128Mi - startupProbe: - httpGet: - path: /health - port: 3000 - failureThreshold: 5 - initialDelaySeconds: 20 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /health - port: 3000 - failureThreshold: 3 - initialDelaySeconds: 3 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /health - port: 3000 - failureThreshold: 5 - initialDelaySeconds: 3 - periodSeconds: 3 - initContainers: - - name: wait-for-rabbitmq - image: busybox - command: ['sh', '-c', 'until nc -zv rabbitmq 5672; do echo waiting for rabbitmq; sleep 2; done;'] - resources: - requests: - cpu: 1m - memory: 50Mi - limits: - cpu: 75m - memory: 128Mi - --- - apiVersion: v1 - kind: Service - metadata: - name: order-service - spec: - type: ClusterIP - ports: - - name: http - port: 3000 - targetPort: 3000 - selector: - app: order-service - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: product-service - spec: - replicas: 1 - selector: - matchLabels: - app: product-service - template: - metadata: - labels: - app: product-service - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: product-service - image: ghcr.io/azure-samples/aks-store-demo/product-service:latest - ports: - - containerPort: 3002 - env: - - name: AI_SERVICE_URL - value: "http://ai-service:5001/" - resources: - requests: - cpu: 1m - memory: 1Mi - limits: - cpu: 2m - memory: 20Mi - readinessProbe: - httpGet: - path: /health - port: 3002 - failureThreshold: 3 - initialDelaySeconds: 3 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /health - port: 3002 - failureThreshold: 5 - initialDelaySeconds: 3 - periodSeconds: 3 - --- - apiVersion: v1 - kind: Service - metadata: - name: product-service - spec: - type: ClusterIP - ports: - - name: http - port: 3002 - targetPort: 3002 - selector: - app: product-service - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: store-front - spec: - replicas: 1 - selector: - matchLabels: - app: store-front - template: - metadata: - labels: - app: store-front - spec: - nodeSelector: - "kubernetes.io/os": linux - containers: - - name: store-front - image: ghcr.io/azure-samples/aks-store-demo/store-front:latest - ports: - - containerPort: 8080 - name: store-front - env: - - name: VUE_APP_ORDER_SERVICE_URL - value: "http://order-service:3000/" - - name: VUE_APP_PRODUCT_SERVICE_URL - value: "http://product-service:3002/" - resources: - requests: - cpu: 1m - memory: 200Mi - limits: - cpu: 1000m - memory: 512Mi - startupProbe: - httpGet: - path: /health - port: 8080 - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 5 - readinessProbe: - httpGet: - path: /health - port: 8080 - failureThreshold: 3 - initialDelaySeconds: 3 - periodSeconds: 3 - livenessProbe: - httpGet: - path: /health - port: 8080 - failureThreshold: 5 - initialDelaySeconds: 3 - periodSeconds: 3 - --- - apiVersion: v1 - kind: Service - metadata: - name: store-front - spec: - ports: - - port: 80 - targetPort: 8080 - selector: - app: store-front - type: LoadBalancer + ```bash + resource_group_name=$(terraform output -raw resource_group_name) ``` - For a breakdown of YAML manifest files, see [Deployments and YAML manifests](../concepts-clusters-workloads.md#deployments-and-yaml-manifests). +1. Run [az vm list](/cli/azure/vm#az-vm-list) with a [JMESPath](/cli/azure/query-azure-cli) query to display the names of the virtual machines created in the resource group. - If you create and save the YAML file locally, then you can upload the manifest file to your default directory in CloudShell by selecting the **Upload/Download files** button and selecting the file from your local file system. + ```azurecli + az vm list \ + --resource-group $resource_group_name \ + --query "[].{\"VM Name\":name}" -o table + ``` -1. Deploy the application using the [`kubectl apply`][kubectl-apply] command and specify the name of your YAML manifest. +#### [Azure PowerShell](#tab/azure-powershell) - ```azurecli-interactive - kubectl apply -f aks-store-quickstart.yaml - ``` +1. Get the Azure resource group name. -## Test the application - -You can validate that the application is running by visiting the public IP address or the application URL. - -Get the application URL using the following commands: - -```azurecli-interactive -runtime="5 minutes" -endtime=$(date -ud "$runtime" +%s) -while [[ $(date -u +%s) -le $endtime ]] -do - STATUS=$(kubectl get pods -l app=store-front -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') - echo $STATUS - if [ "$STATUS" == 'True' ] - then - export IP_ADDRESS=$(kubectl get service store-front --output 'jsonpath={..status.loadBalancer.ingress[0].ip}') - echo "Service IP Address: $IP_ADDRESS" - break - else - sleep 10 - fi -done -``` + ```console + $resource_group_name=$(terraform output -raw resource_group_name) + ``` -```azurecli-interactive -curl $IP_ADDRESS -``` +1. Run [Get-AzVm](/powershell/module/az.compute/get-azvm) to display the names of all the virtual machines in the resource group. -Results: - -```HTML - - - - - - - - store-front - - - - - -
- - -``` + ```azurepowershell + Get-AzVm -ResourceGroupName $resource_group_name + ``` -```OUTPUT -echo "You can now visit your web server at $IP_ADDRESS" -``` +--- -:::image type="content" source="media/quick-kubernetes-deploy-cli/aks-store-application.png" alt-text="Screenshot of AKS Store sample application." lightbox="media/quick-kubernetes-deploy-cli/aks-store-application.png"::: +## Clean up resources -## Delete the cluster +[!INCLUDE [terraform-plan-destroy.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan-destroy.md)] -If you don't plan on going through the [AKS tutorial][aks-tutorial], clean up unnecessary resources to avoid Azure charges. You can remove the resource group, container service, and all related resources using the [`az group delete`][az-group-delete] command. +## Troubleshoot Terraform on Azure -> [!NOTE] -> The AKS cluster was created with a system-assigned managed identity, which is the default identity option used in this quickstart. The platform manages this identity so you don't need to manually remove it. +[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) ## Next steps -In this quickstart, you deployed a Kubernetes cluster and then deployed a simple multi-container application to it. This sample application is for demo purposes only and doesn't represent all the best practices for Kubernetes applications. For guidance on creating full solutions with AKS for production, see [AKS solution guidance][aks-solution-guidance]. - -To learn more about AKS and walk through a complete code-to-deployment example, continue to the Kubernetes cluster tutorial. +In this quickstart, you deployed a simple virtual machine using Terraform. To learn more about Azure virtual machines, continue to the tutorial for Linux VMs. > [!div class="nextstepaction"] -> [AKS tutorial][aks-tutorial] - - -[kubectl]: https://kubernetes.io/docs/reference/kubectl/ -[kubectl-apply]: https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply -[kubectl-get]: https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#get - - -[kubernetes-concepts]: ../concepts-clusters-workloads.md -[aks-tutorial]: ../tutorial-kubernetes-prepare-app.md -[azure-resource-group]: /azure/azure-resource-manager/management/overview -[az-aks-create]: /cli/azure/aks#az-aks-create -[az-aks-get-credentials]: /cli/azure/aks#az-aks-get-credentials -[az-aks-install-cli]: /cli/azure/aks#az-aks-install-cli -[az-group-create]: /cli/azure/group#az-group-create -[az-group-delete]: /cli/azure/group#az-group-delete -[kubernetes-deployment]: ../concepts-clusters-workloads.md#deployments-and-yaml-manifests -[aks-solution-guidance]: /azure/architecture/reference-architectures/containers/aks-start-here?toc=/azure/aks/toc.json&bc=/azure/aks/breadcrumb/toc.json -[baseline-reference-architecture]: /azure/architecture/reference-architectures/containers/aks/baseline-aks?toc=/azure/aks/toc.json&bc=/azure/aks/breadcrumb/toc.json \ No newline at end of file +> [Azure Linux virtual machine tutorials](./tutorial-manage-vm.md) \ No newline at end of file diff --git a/tools/execution_log.csv b/tools/execution_log.csv index b3e251a79..2bb0c316f 100644 --- a/tools/execution_log.csv +++ b/tools/execution_log.csv @@ -226,3 +226,998 @@ See vm create -h for more information on specifying an image. ' StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. See vm create -h for more information on specifying an image.",153.62026572227478,Success +2025-03-05 11:22:50,file,doc.md,doc_converted.md,11,"time=2025-03-05T11:10:31-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. +Error: invalid character '\x1b' looking for beginning of value +StdErr: + + time=2025-03-05T11:11:10-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. +Error: invalid character 'I' looking for beginning of value +StdErr: + + time=2025-03-05T11:13:50-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. +Error: command exited with 'exit status 1' and the message ' +Error: creating Linux Virtual Machine (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Virtual Machine Name: ""myVM""): performing CreateOrUpdate: unexpected status 409 (409 Conflict) with error: SkuNotAvailable: The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details. + + with azurerm_linux_virtual_machine.my_terraform_vm, + on main.tf line 93, in resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"": + 93: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + +' +StdErr: +Error: creating Linux Virtual Machine (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Virtual Machine Name: ""myVM""): performing CreateOrUpdate: unexpected status 409 (409 Conflict) with error: SkuNotAvailable: The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details. + + with azurerm_linux_virtual_machine.my_terraform_vm, + on main.tf line 93, in resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"": + 93: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + + time=2025-03-05T11:15:42-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. +Error: command exited with 'exit status 1' and the message ' +Error: deleting Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Network Interface Name: ""myNIC""): performing Delete: unexpected status 400 (400 Bad Request) with error: NicReservedForAnotherVm: Nic(s) in request is reserved for another Virtual Machine for 180 seconds. Please provide another nic(s) or retry after 180 seconds. Reserved VM: /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/virtualMachines/myVM + +' +StdErr: +Error: deleting Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Network Interface Name: ""myNIC""): performing Delete: unexpected status 400 (400 Bad Request) with error: NicReservedForAnotherVm: Nic(s) in request is reserved for another Virtual Machine for 180 seconds. Please provide another nic(s) or retry after 180 seconds. Reserved VM: /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/virtualMachines/myVM + + time=2025-03-05T11:16:24-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:5,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_virtual_network"" configuration + + on network.tf line 11: + 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { + +A azurerm_virtual_network resource named ""my_terraform_network"" was already +declared at main.tf:11,1-58. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_subnet"" configuration + + on network.tf line 19: + 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { + +A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at +main.tf:19,1-48. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_public_ip"" configuration + + on network.tf line 27: + 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { + +A azurerm_public_ip resource named ""my_terraform_public_ip"" was already +declared at main.tf:27,1-54. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_security_group"" configuration + + on network.tf line 35: + 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { + +A azurerm_network_security_group resource named ""my_terraform_nsg"" was +already declared at main.tf:35,1-61. Resource names must be unique per type +in each module. + + +Error: Duplicate resource ""azurerm_network_interface"" configuration + + on network.tf line 54: + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + +A azurerm_network_interface resource named ""my_terraform_nic"" was already +declared at main.tf:54,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { + +A azurerm_network_interface_security_group_association resource named +""example"" was already declared at main.tf:68,1-74. Resource names must be +unique per type in each module. + + +Error: Duplicate resource ""random_id"" configuration + + on network.tf line 74: + 74: resource ""random_id"" ""random_id"" { + +A random_id resource named ""random_id"" was already declared at +main.tf:74,1-33. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_storage_account"" configuration + + on network.tf line 82: + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + +A azurerm_storage_account resource named ""my_storage_account"" was already +declared at main.tf:84,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration + + on vm.tf line 1: + 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + +A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already +declared at main.tf:93,1-59. Resource names must be unique per type in each +module. + +' +StdErr: There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:5,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_virtual_network"" configuration + + on network.tf line 11: + 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { + +A azurerm_virtual_network resource named ""my_terraform_network"" was already +declared at main.tf:11,1-58. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_subnet"" configuration + + on network.tf line 19: + 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { + +A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at +main.tf:19,1-48. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_public_ip"" configuration + + on network.tf line 27: + 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { + +A azurerm_public_ip resource named ""my_terraform_public_ip"" was already +declared at main.tf:27,1-54. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_security_group"" configuration + + on network.tf line 35: + 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { + +A azurerm_network_security_group resource named ""my_terraform_nsg"" was +already declared at main.tf:35,1-61. Resource names must be unique per type +in each module. + + +Error: Duplicate resource ""azurerm_network_interface"" configuration + + on network.tf line 54: + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + +A azurerm_network_interface resource named ""my_terraform_nic"" was already +declared at main.tf:54,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { + +A azurerm_network_interface_security_group_association resource named +""example"" was already declared at main.tf:68,1-74. Resource names must be +unique per type in each module. + + +Error: Duplicate resource ""random_id"" configuration + + on network.tf line 74: + 74: resource ""random_id"" ""random_id"" { + +A random_id resource named ""random_id"" was already declared at +main.tf:74,1-33. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_storage_account"" configuration + + on network.tf line 82: + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + +A azurerm_storage_account resource named ""my_storage_account"" was already +declared at main.tf:84,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration + + on vm.tf line 1: + 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + +A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already +declared at main.tf:93,1-59. Resource names must be unique per type in each +module. + + time=2025-03-05T11:16:58-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:5,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_virtual_network"" configuration + + on network.tf line 11: + 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { + +A azurerm_virtual_network resource named ""my_terraform_network"" was already +declared at main.tf:11,1-58. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_subnet"" configuration + + on network.tf line 19: + 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { + +A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at +main.tf:19,1-48. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_public_ip"" configuration + + on network.tf line 27: + 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { + +A azurerm_public_ip resource named ""my_terraform_public_ip"" was already +declared at main.tf:27,1-54. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_security_group"" configuration + + on network.tf line 35: + 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { + +A azurerm_network_security_group resource named ""my_terraform_nsg"" was +already declared at main.tf:35,1-61. Resource names must be unique per type +in each module. + + +Error: Duplicate resource ""azurerm_network_interface"" configuration + + on network.tf line 54: + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + +A azurerm_network_interface resource named ""my_terraform_nic"" was already +declared at main.tf:54,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { + +A azurerm_network_interface_security_group_association resource named +""example"" was already declared at main.tf:68,1-74. Resource names must be +unique per type in each module. + + +Error: Duplicate resource ""random_id"" configuration + + on network.tf line 74: + 74: resource ""random_id"" ""random_id"" { + +A random_id resource named ""random_id"" was already declared at +main.tf:74,1-33. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_storage_account"" configuration + + on network.tf line 82: + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + +A azurerm_storage_account resource named ""my_storage_account"" was already +declared at main.tf:84,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration + + on vm.tf line 1: + 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + +A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already +declared at main.tf:93,1-59. Resource names must be unique per type in each +module. + +' +StdErr: There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:5,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_virtual_network"" configuration + + on network.tf line 11: + 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { + +A azurerm_virtual_network resource named ""my_terraform_network"" was already +declared at main.tf:11,1-58. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_subnet"" configuration + + on network.tf line 19: + 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { + +A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at +main.tf:19,1-48. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_public_ip"" configuration + + on network.tf line 27: + 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { + +A azurerm_public_ip resource named ""my_terraform_public_ip"" was already +declared at main.tf:27,1-54. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_security_group"" configuration + + on network.tf line 35: + 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { + +A azurerm_network_security_group resource named ""my_terraform_nsg"" was +already declared at main.tf:35,1-61. Resource names must be unique per type +in each module. + + +Error: Duplicate resource ""azurerm_network_interface"" configuration + + on network.tf line 54: + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + +A azurerm_network_interface resource named ""my_terraform_nic"" was already +declared at main.tf:54,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { + +A azurerm_network_interface_security_group_association resource named +""example"" was already declared at main.tf:68,1-74. Resource names must be +unique per type in each module. + + +Error: Duplicate resource ""random_id"" configuration + + on network.tf line 74: + 74: resource ""random_id"" ""random_id"" { + +A random_id resource named ""random_id"" was already declared at +main.tf:74,1-33. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_storage_account"" configuration + + on network.tf line 82: + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + +A azurerm_storage_account resource named ""my_storage_account"" was already +declared at main.tf:84,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration + + on vm.tf line 1: + 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + +A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already +declared at main.tf:93,1-59. Resource names must be unique per type in each +module. + + time=2025-03-05T11:17:48-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:5,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_virtual_network"" configuration + + on network.tf line 11: + 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { + +A azurerm_virtual_network resource named ""my_terraform_network"" was already +declared at main.tf:11,1-58. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_subnet"" configuration + + on network.tf line 19: + 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { + +A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at +main.tf:19,1-48. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_public_ip"" configuration + + on network.tf line 27: + 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { + +A azurerm_public_ip resource named ""my_terraform_public_ip"" was already +declared at main.tf:27,1-54. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_security_group"" configuration + + on network.tf line 35: + 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { + +A azurerm_network_security_group resource named ""my_terraform_nsg"" was +already declared at main.tf:35,1-61. Resource names must be unique per type +in each module. + + +Error: Duplicate resource ""azurerm_network_interface"" configuration + + on network.tf line 54: + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + +A azurerm_network_interface resource named ""my_terraform_nic"" was already +declared at main.tf:54,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { + +A azurerm_network_interface_security_group_association resource named +""example"" was already declared at main.tf:68,1-74. Resource names must be +unique per type in each module. + + +Error: Duplicate resource ""random_id"" configuration + + on network.tf line 74: + 74: resource ""random_id"" ""random_id"" { + +A random_id resource named ""random_id"" was already declared at +main.tf:74,1-33. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_storage_account"" configuration + + on network.tf line 82: + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + +A azurerm_storage_account resource named ""my_storage_account"" was already +declared at main.tf:84,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration + + on vm.tf line 1: + 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + +A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already +declared at main.tf:93,1-59. Resource names must be unique per type in each +module. + +' +StdErr: There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:5,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_virtual_network"" configuration + + on network.tf line 11: + 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { + +A azurerm_virtual_network resource named ""my_terraform_network"" was already +declared at main.tf:11,1-58. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_subnet"" configuration + + on network.tf line 19: + 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { + +A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at +main.tf:19,1-48. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_public_ip"" configuration + + on network.tf line 27: + 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { + +A azurerm_public_ip resource named ""my_terraform_public_ip"" was already +declared at main.tf:27,1-54. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_security_group"" configuration + + on network.tf line 35: + 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { + +A azurerm_network_security_group resource named ""my_terraform_nsg"" was +already declared at main.tf:35,1-61. Resource names must be unique per type +in each module. + + +Error: Duplicate resource ""azurerm_network_interface"" configuration + + on network.tf line 54: + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + +A azurerm_network_interface resource named ""my_terraform_nic"" was already +declared at main.tf:54,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { + +A azurerm_network_interface_security_group_association resource named +""example"" was already declared at main.tf:68,1-74. Resource names must be +unique per type in each module. + + +Error: Duplicate resource ""random_id"" configuration + + on network.tf line 74: + 74: resource ""random_id"" ""random_id"" { + +A random_id resource named ""random_id"" was already declared at +main.tf:74,1-33. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_storage_account"" configuration + + on network.tf line 82: + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + +A azurerm_storage_account resource named ""my_storage_account"" was already +declared at main.tf:84,1-56. Resource names must be unique per type in each +module. + + +Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration + + on vm.tf line 1: + 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { + +A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already +declared at main.tf:93,1-59. Resource names must be unique per type in each +module. + + time=2025-03-05T11:20:29-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. +Error: command exited with 'exit status 1' and the message ' +Error: Provider produced inconsistent result after apply + +When applying changes to azurerm_network_security_group.my_terraform_nsg, +provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an +unexpected new value: Root resource was present, but now absent. + +This is a bug in the provider, which should be reported in the provider's own +issue tracker. + +Error: Provider produced inconsistent result after apply + +When applying changes to azurerm_virtual_network.my_terraform_network, +provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an +unexpected new value: Root resource was present, but now absent. + +This is a bug in the provider, which should be reported in the provider's own +issue tracker. + +Error: creating Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Network Interface Name: ""myNIC""): performing CreateOrUpdate: unexpected status 400 (400 Bad Request) with error: InvalidResourceReference: Resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/virtualNetworks/myVnet/subnets/mySubnet referenced by resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/networkInterfaces/myNIC was not found. Please make sure that the referenced resource exists, and that both resources are in the same region. + + with azurerm_network_interface.my_terraform_nic, + on network.tf line 54, in resource ""azurerm_network_interface"" ""my_terraform_nic"": + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + + +Error: retrieving Storage Account (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Storage Account Name: ""diag0bdcb34b14495a71""): unexpected status 404 (404 Not Found) with error: ResourceNotFound: The Resource 'Microsoft.Storage/storageAccounts/diag0bdcb34b14495a71' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix + + with azurerm_storage_account.my_storage_account, + on network.tf line 82, in resource ""azurerm_storage_account"" ""my_storage_account"": + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + + +Error: Failed to create/update resource + + with azapi_resource.ssh_public_key, + on ssh.tf line 15, in resource ""azapi_resource"" ""ssh_public_key"": + 15: resource ""azapi_resource"" ""ssh_public_key"" { + +creating/updating Resource: (ResourceId +""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar"" +/ Api Version ""2022-11-01""): GET +https://management.azure.com/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar +-------------------------------------------------------------------------------- +RESPONSE 404: 404 Not Found +ERROR CODE: ResourceNotFound +-------------------------------------------------------------------------------- +{ + ""error"": { + ""code"": ""ResourceNotFound"", + ""message"": ""The Resource 'Microsoft.Compute/sshPublicKeys/sshevidentjaguar' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix"" + } +} +-------------------------------------------------------------------------------- + +' +StdErr: +Error: Provider produced inconsistent result after apply + +When applying changes to azurerm_network_security_group.my_terraform_nsg, +provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an +unexpected new value: Root resource was present, but now absent. + +This is a bug in the provider, which should be reported in the provider's own +issue tracker. + +Error: Provider produced inconsistent result after apply + +When applying changes to azurerm_virtual_network.my_terraform_network, +provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an +unexpected new value: Root resource was present, but now absent. + +This is a bug in the provider, which should be reported in the provider's own +issue tracker. + +Error: creating Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Network Interface Name: ""myNIC""): performing CreateOrUpdate: unexpected status 400 (400 Bad Request) with error: InvalidResourceReference: Resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/virtualNetworks/myVnet/subnets/mySubnet referenced by resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/networkInterfaces/myNIC was not found. Please make sure that the referenced resource exists, and that both resources are in the same region. + + with azurerm_network_interface.my_terraform_nic, + on network.tf line 54, in resource ""azurerm_network_interface"" ""my_terraform_nic"": + 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { + + +Error: retrieving Storage Account (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" +Resource Group Name: ""rg-bold-caiman"" +Storage Account Name: ""diag0bdcb34b14495a71""): unexpected status 404 (404 Not Found) with error: ResourceNotFound: The Resource 'Microsoft.Storage/storageAccounts/diag0bdcb34b14495a71' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix + + with azurerm_storage_account.my_storage_account, + on network.tf line 82, in resource ""azurerm_storage_account"" ""my_storage_account"": + 82: resource ""azurerm_storage_account"" ""my_storage_account"" { + + +Error: Failed to create/update resource + + with azapi_resource.ssh_public_key, + on ssh.tf line 15, in resource ""azapi_resource"" ""ssh_public_key"": + 15: resource ""azapi_resource"" ""ssh_public_key"" { + +creating/updating Resource: (ResourceId +""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar"" +/ Api Version ""2022-11-01""): GET +https://management.azure.com/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar +-------------------------------------------------------------------------------- +RESPONSE 404: 404 Not Found +ERROR CODE: ResourceNotFound +-------------------------------------------------------------------------------- +{ + ""error"": { + ""code"": ""ResourceNotFound"", + ""message"": ""The Resource 'Microsoft.Compute/sshPublicKeys/sshevidentjaguar' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix"" + } +} +-------------------------------------------------------------------------------- + + time=2025-03-05T11:21:26-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message ' +Error: Failed to query available provider packages + +Could not retrieve the list of available versions for provider +hashicorp/azapi: provider registry registry.terraform.io does not have a +provider named registry.terraform.io/hashicorp/azapi + +Did you intend to use azure/azapi? If so, you must specify that source +address in each module which requires that provider. To see which modules are +currently depending on hashicorp/azapi, run the following command: + terraform providers + +' +StdErr: +Error: Failed to query available provider packages + +Could not retrieve the list of available versions for provider +hashicorp/azapi: provider registry registry.terraform.io does not have a +provider named registry.terraform.io/hashicorp/azapi + +Did you intend to use azure/azapi? If so, you must specify that source +address in each module which requires that provider. To see which modules are +currently depending on hashicorp/azapi, run the following command: + terraform providers + + time=2025-03-05T11:22:09-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. +Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:12,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { + +A azurerm_network_interface_security_group_association resource named +""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be +unique per type in each module. + +' +StdErr: There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:12,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { + +A azurerm_network_interface_security_group_association resource named +""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be +unique per type in each module. + + time=2025-03-05T11:22:50-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. +Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:12,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { + +A azurerm_network_interface_security_group_association resource named +""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be +unique per type in each module. + +' +StdErr: There are some problems with the configuration, described below. + +The Terraform configuration must be valid before initialization so that +Terraform can determine which modules and providers need to be installed. + +Error: Duplicate resource ""random_pet"" configuration + + on network.tf line 1: + 1: resource ""random_pet"" ""rg_name"" { + +A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. +Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_resource_group"" configuration + + on network.tf line 5: + 5: resource ""azurerm_resource_group"" ""rg"" { + +A azurerm_resource_group resource named ""rg"" was already declared at +main.tf:12,1-39. Resource names must be unique per type in each module. + + +Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration + + on network.tf line 68: + 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { + +A azurerm_network_interface_security_group_association resource named +""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be +unique per type in each module.",873.9581248760223,Failure +2025-03-05 12:00:05,workload_description,create a linux vm and ssh into it,Deploy a Linux VM and Connect via SSH_ai_generated.md,1,"time=2025-03-05T11:58:41-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. +' +StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image.",109.34437084197998,Success +2025-03-05 15:01:21,workload_description,create a linux vm and ssh into it ,Create Linux VM with SSH Access on Azure_ai_generated.md,1,"time=2025-03-05T15:00:22-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. +Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. +' +StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image.",87.87348413467407,Success From 8fa1a14046d28126cf40a2b32a721029e570c82c Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 6 Mar 2025 00:25:58 -0800 Subject: [PATCH 12/87] updated tool --- .../linux/quick-create-terraform.md | 375 ++++++++++++++++++ scenarios/metadata.json | 22 + tools/ada.py | 241 +++++++++-- tools/execution_log.csv | 15 + 4 files changed, 627 insertions(+), 26 deletions(-) create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md new file mode 100644 index 000000000..9ef176aa1 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md @@ -0,0 +1,375 @@ +--- +title: 'Quickstart: Use Terraform to create a Linux VM' +description: In this quickstart, you learn how to use Terraform to create a Linux virtual machine +author: tomarchermsft +ms.service: azure-virtual-machines +ms.collection: linux +ms.topic: quickstart +ms.date: 07/24/2023 +ms.author: tarcher +ms.custom: devx-track-terraform, linux-related-content, innovation-engine +ai-usage: ai-assisted +--- + +# Quickstart: Use Terraform to create a Linux VM + +**Applies to:** :heavy_check_mark: Linux VMs + +Article tested with the following Terraform and Terraform provider versions: + +This article shows you how to create a complete Linux environment and supporting resources with Terraform. Those resources include a virtual network, subnet, public IP address, and more. + +[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] + +In this article, you learn how to: +> [!div class="checklist"] +> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet). +> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group). +> * Create a virtual network (VNET) using [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network). +> * Create a subnet using [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet). +> * Create a public IP using [azurerm_public_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip). +> * Create a network security group using [azurerm_network_security_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_group). +> * Create a network interface using [azurerm_network_interface](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface). +> * Create an association between the network security group and the network interface using [azurerm_network_interface_security_group_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface_security_group_association). +> * Generate a random value for a unique storage account name using [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id). +> * Create a storage account for boot diagnostics using [azurerm_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account). +> * Create a Linux VM using [azurerm_linux_virtual_machine](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine). +> * Create an AzAPI resource using [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource). +> * Create an AzAPI resource to generate an SSH key pair using [azapi_resource_action](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource_action). + +## Prerequisites + +- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) + +## Implement the Terraform code + +> [!NOTE] +> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure/TestRecord.md). +> +> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) + +1. Create a directory in which to test the sample Terraform code and make it the current directory. + +1. Create a file named providers.tf and insert the following code: + +```bash +cat <<'EOF' > providers.tf +terraform { + required_version = ">=0.12" + + required_providers { + azapi = { + source = "azure/azapi" + version = "~>1.5" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} + +provider "azurerm" { + features {} +} +EOF +``` + +1. Create a file named ssh.tf and insert the following code: + +```bash +cat <<'EOF' > ssh.tf +resource "random_pet" "ssh_key_name" { + prefix = "ssh" + separator = "" +} + +resource "azapi_resource_action" "ssh_public_key_gen" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + resource_id = azapi_resource.ssh_public_key.id + action = "generateKeyPair" + method = "POST" + + response_export_values = ["publicKey", "privateKey"] +} + +resource "azapi_resource" "ssh_public_key" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + name = random_pet.ssh_key_name.id + location = azurerm_resource_group.rg.location + parent_id = azurerm_resource_group.rg.id +} + +output "key_data" { + value = azapi_resource_action.ssh_public_key_gen.output.publicKey +} +EOF +``` + +1. Create a file named main.tf and insert the following code: + +```bash +cat <<'EOF' > main.tf +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +resource "azurerm_resource_group" "rg" { + location = var.resource_group_location + name = random_pet.rg_name.id +} + +# Create virtual network +resource "azurerm_virtual_network" "my_terraform_network" { + name = "myVnet" + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name +} + +# Create subnet +resource "azurerm_subnet" "my_terraform_subnet" { + name = "mySubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.my_terraform_network.name + address_prefixes = ["10.0.1.0/24"] +} + +# Create public IPs +resource "azurerm_public_ip" "my_terraform_public_ip" { + name = "myPublicIP" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + allocation_method = "Dynamic" +} + +# Create Network Security Group and rule +resource "azurerm_network_security_group" "my_terraform_nsg" { + name = "myNetworkSecurityGroup" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + security_rule { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } +} + +# Create network interface +resource "azurerm_network_interface" "my_terraform_nic" { + name = "myNIC" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + ip_configuration { + name = "my_nic_configuration" + subnet_id = azurerm_subnet.my_terraform_subnet.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.my_terraform_public_ip.id + } +} + +# Connect the security group to the network interface +resource "azurerm_network_interface_security_group_association" "example" { + network_interface_id = azurerm_network_interface.my_terraform_nic.id + network_security_group_id = azurerm_network_security_group.my_terraform_nsg.id +} + +# Generate random text for a unique storage account name +resource "random_id" "random_id" { + keepers = { + # Generate a new ID only when a new resource group is defined + resource_group = azurerm_resource_group.rg.name + } + + byte_length = 8 +} + +# Create storage account for boot diagnostics +resource "azurerm_storage_account" "my_storage_account" { + name = "diag${random_id.random_id.hex}" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + account_tier = "Standard" + account_replication_type = "LRS" +} + +# Create virtual machine +resource "azurerm_linux_virtual_machine" "my_terraform_vm" { + name = "myVM" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + network_interface_ids = [azurerm_network_interface.my_terraform_nic.id] + size = "Standard_DS1_v2" + + os_disk { + name = "myOsDisk" + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-gen2" + version = "latest" + } + + computer_name = "hostname" + admin_username = var.username + + admin_ssh_key { + username = var.username + public_key = azapi_resource_action.ssh_public_key_gen.output.publicKey + } + + boot_diagnostics { + storage_account_uri = azurerm_storage_account.my_storage_account.primary_blob_endpoint + } +} +EOF +``` + +1. Create a file named variables.tf and insert the following code: + +```bash +cat <<'EOF' > variables.tf +variable "resource_group_location" { + type = string + default = "eastus" + description = "Location of the resource group." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." +} + +variable "username" { + type = string + description = "The username for the local account that will be created on the new VM." + default = "azureadmin" +} +EOF +``` + +1. Create a file named outputs.tf and insert the following code: + +```bash +cat <<'EOF' > outputs.tf +output "resource_group_name" { + value = azurerm_resource_group.rg.name +} + +output "public_ip_address" { + value = azurerm_linux_virtual_machine.my_terraform_vm.public_ip_address +} +EOF +``` + +## Initialize Terraform + +In this section, Terraform is initialized; this command downloads the Azure provider required to manage your Azure resources. Before running the command, ensure you are in the directory where you created the Terraform files. You can set any necessary environment variables here. + +```bash +export TERRAFORM_DIR=$(pwd) +terraform init -upgrade +``` + +Key points: + +- The -upgrade parameter upgrades the necessary provider plugins to the newest version that complies with the configuration's version constraints. + +## Create a Terraform execution plan + +This step creates an execution plan but does not execute it. It shows what actions are necessary to create the configuration specified in your files. + +```bash +terraform plan -out main.tfplan +``` + +Key points: + +- The terraform plan command creates an execution plan, allowing you to verify whether it matches your expectations before applying any changes. +- The optional -out parameter writes the plan to a file so that the exact plan can be applied later. + +## Apply a Terraform execution plan + +Apply the previously created execution plan to deploy the infrastructure to your cloud. + +```bash +terraform apply main.tfplan +``` + +Key points: + +- This command applies the plan created with terraform plan -out main.tfplan. +- If you used a different filename for the -out parameter, use that same filename with terraform apply. +- If the -out parameter wasn’t used, run terraform apply without any parameters. + +Cost information isn't presented during the virtual machine creation process for Terraform like it is for the [Azure portal](quick-create-portal.md). If you want to learn more about how cost works for virtual machines, see the [Cost optimization Overview page](../plan-to-manage-costs.md). + +## Verify the results + +#### [Azure CLI](#tab/azure-cli) + +1. Get the Azure resource group name. + +```bash +export RESOURCE_GROUP_NAME=$(terraform output -raw resource_group_name) +``` + +1. Run az vm list with a JMESPath query to display the names of the virtual machines created in the resource group. + +```azurecli +az vm list \ + --resource-group $RESOURCE_GROUP_NAME \ + --query "[].{\"VM Name\":name}" -o table +``` + +Results: + + + +```console +VM Name +----------- +myVM +``` + +#### [Azure PowerShell](#tab/azure-powershell) + +1. Get the Azure resource group name. + +```console +$resource_group_name=$(terraform output -raw resource_group_name) +``` + +1. Run Get-AzVm to display the names of all the virtual machines in the resource group. + +```azurepowershell +Get-AzVm -ResourceGroupName $resource_group_name +``` + +## Troubleshoot Terraform on Azure + +[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) + +## Next steps + +In this quickstart, you deployed a simple virtual machine using Terraform. To learn more about Azure virtual machines, continue to the tutorial for Linux VMs. + +> [!div class="nextstepaction"] +> [Azure Linux virtual machine tutorials](./tutorial-manage-vm.md) \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 64d3ef194..5b3920199 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1131,5 +1131,27 @@ ], "configurations": { } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md", + "title": "Quickstart: Use Terraform to create a Linux VM", + "description": "In this quickstart, you learn how to use Terraform to create a Linux virtual machine", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/quick-create-terraform?tabs=azure-cli", + "nextSteps": [ + { + "title": "Troubleshoot common problems when using Terraform on Azure", + "url": "https://learn.microsoft.com/en-us/azure/developer/terraform/troubleshoot" + }, + { + "title": "Azure Linux Virtual Machine Tutorials", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-manage-vm" + } + ], + "configurations": { + } } ] diff --git a/tools/ada.py b/tools/ada.py index ce6a8fd7b..2ba497894 100644 --- a/tools/ada.py +++ b/tools/ada.py @@ -424,14 +424,14 @@ def generate_dependency_files(doc_path): """Extract and generate dependency files referenced in an Exec Doc.""" if not os.path.isfile(doc_path): print(f"\nError: The file {doc_path} does not exist.") - return False + return False, [] try: with open(doc_path, "r") as f: doc_content = f.read() except Exception as e: print(f"\nError reading document: {e}") - return False + return False, [] # Directory where the doc is located doc_dir = os.path.dirname(doc_path) or "." @@ -447,10 +447,12 @@ def generate_dependency_files(doc_path): 3. YAML files (configuration, templates, manifests) 4. JSON files (configuration, templates, API payloads) 5. Shell scripts (.sh files) - 6. Any other files where content is provided and meant to be saved separately + 6. Terraform files (.tf or .tfvars) + 7. Any other files where content is provided and meant to be saved separately IMPORTANT: Include files even if their full content is provided in the document! If the doc instructs the user to create a file and provides its content, this IS a dependency file. + Look for patterns like "create the following file" or "save this content to filename.xyz". For each file you identify: 1. Extract the exact filename with its extension @@ -466,6 +468,8 @@ def generate_dependency_files(doc_path): ] ) + created_dep_files = [] + try: # Extract the JSON part from the response with improved robustness response_text = response.choices[0].message.content @@ -501,7 +505,7 @@ def generate_dependency_files(doc_path): if not dependency_list: print("\nNo dependency files identified.") - return True + return True, [] # Create each dependency file with type-specific handling created_files = [] @@ -518,8 +522,25 @@ def generate_dependency_files(doc_path): # Check if file already exists if os.path.exists(file_path): print(f"\nFile already exists: {filename} - Skipping") + # Load content from existing file + try: + with open(file_path, "r") as f: + existing_content = f.read() + created_dep_files.append({ + "filename": filename, + "path": file_path, + "type": file_type, + "content": existing_content # Include content + }) + except Exception as e: + print(f"\nWarning: Could not read content from {filename}: {e}") + created_dep_files.append({ + "filename": filename, + "path": file_path, + "type": file_type + }) continue - + # Validate and format content based on file type try: if filename.endswith('.json') or file_type == 'json': @@ -538,6 +559,10 @@ def generate_dependency_files(doc_path): except yaml.YAMLError: print(f"\nWarning: Content for {filename} is not valid YAML. Saving as plain text.") + elif filename.endswith('.tf') or filename.endswith('.tfvars') or file_type == 'terraform': + # Just store terraform files as-is + pass + elif filename.endswith('.sh') or file_type == 'shell': # Ensure shell scripts are executable is_executable = True @@ -547,10 +572,16 @@ def generate_dependency_files(doc_path): f.write(content) # Make shell scripts executable if needed - if (filename.endswith('.sh') or file_type == 'shell') and is_executable: + if (filename.endswith('.sh') or file_type == 'shell') and 'is_executable' in locals() and is_executable: os.chmod(file_path, os.stat(file_path).st_mode | 0o111) # Add executable bit created_files.append(filename) + created_dep_files.append({ + "filename": filename, + "path": file_path, + "type": file_type, + "content": content + }) except Exception as e: print(f"\nError creating {filename}: {e}") @@ -559,13 +590,126 @@ def generate_dependency_files(doc_path): else: print("\nNo new dependency files were created.") - return True + return True, created_dep_files except Exception as e: print(f"\nError generating dependency files: {e}") print("\nResponse from model was not valid JSON. Raw response:") - # print(response.choices[0].message.content[:500] + "..." if len(response.choices[0].message.content) > 500 else response.choices[0].message.content) + return False, [] + +def update_dependency_file(file_info, error_message, main_doc_path): + """Update a dependency file based on error message.""" + filename = file_info["filename"] + file_path = file_info["path"] + file_type = file_info["type"] + + print(f"\nUpdating dependency file: {filename} based on error...") + + try: + with open(file_path, "r") as f: + file_content = f.read() + + with open(main_doc_path, "r") as f: + doc_content = f.read() + + # Prompt for fixing the dependency file + fix_prompt = f"""The following dependency file related to the Exec Doc is causing errors: + + File: {filename} + Type: {file_type} + Error: {error_message} + + Here is the current content of the file: + + {file_content} + + Here is the main Exec Doc for context: + + {doc_content} + + Please fix the issue in the dependency file. Return ONLY the corrected file content, nothing else. + """ + + response = client.chat.completions.create( + model=deployment_name, + messages=[ + {"role": "system", "content": "You are an AI specialized in fixing technical issues in configuration and code files."}, + {"role": "user", "content": fix_prompt} + ] + ) + + updated_content = response.choices[0].message.content + + # Remove any markdown formatting that might have been added + updated_content = re.sub(r'^```.*$', '', updated_content, flags=re.MULTILINE) + updated_content = re.sub(r'^```$', '', updated_content, flags=re.MULTILINE) + updated_content = updated_content.strip() + + # Validate the updated content based on file type + if filename.endswith('.json') or file_type == 'json': + try: + parsed = json.loads(updated_content) + updated_content = json.dumps(parsed, indent=2) # Pretty-print JSON + except json.JSONDecodeError: + print(f"\nWarning: Updated content for {filename} is not valid JSON.") + + elif filename.endswith('.yaml') or filename.endswith('.yml') or file_type == 'yaml': + try: + parsed = yaml.safe_load(updated_content) + updated_content = yaml.dump(parsed, default_flow_style=False) # Pretty-print YAML + except yaml.YAMLError: + print(f"\nWarning: Updated content for {filename} is not valid YAML.") + + # Write the updated content to the file + with open(file_path, "w") as f: + f.write(updated_content) + + print(f"\nUpdated dependency file: {filename}") + return True + except Exception as e: + print(f"\nError updating dependency file {filename}: {e}") return False + +def analyze_error(error_log, dependency_files=[]): + """Analyze error log to determine if issue is in main doc or dependency files.""" + if not dependency_files: + return {"type": "main_doc", "file": None} + + for dep_file in dependency_files: + filename = dep_file["filename"] + # Check if error mentions the dependency file name + if filename in error_log: + return { + "type": "dependency_file", + "file": dep_file, + "message": error_log + } + + # If no specific dependency file is mentioned, check for patterns + error_patterns = [ + r"Error: open (.*?): no such file or directory", + r"couldn't find file (.*?)( |$|\n)", + r"failed to read (.*?):( |$|\n)", + r"file (.*?) not found", + r"YAML|yaml parsing error", + r"JSON|json parsing error", + r"invalid format in (.*?)( |$|\n)" + ] + + for pattern in error_patterns: + matches = re.search(pattern, error_log, re.IGNORECASE) + if matches and len(matches.groups()) > 0: + file_mentioned = matches.group(1) + for dep_file in dependency_files: + if dep_file["filename"] in file_mentioned: + return { + "type": "dependency_file", + "file": dep_file, + "message": error_log + } + # Default to main doc if no specific dependency file issues found + return {"type": "main_doc", "file": None} + def remove_backticks_from_file(file_path): with open(file_path, "r") as f: lines = f.readlines() @@ -725,8 +869,9 @@ def main(): with open(user_input, "r") as f: input_content = f.read() input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS:\n\n{input_content}" - if input("\nMake new files referenced in the doc for its execution? (y/n): ").lower() == 'y': - generate_dependency_files(user_input) + # We'll generate dependency files later in the process + dependency_files = [] + generate_deps = input("\nMake new files referenced in the doc for its execution? (y/n): ").lower() == 'y' elif choice == "2": user_input = input("\nDescribe your workload for the new Exec Doc: ") if not user_input: @@ -734,6 +879,8 @@ def main(): sys.exit(1) input_type = 'workload_description' input_content = user_input + dependency_files = [] + generate_deps = True elif choice == "3": user_input = input("\nEnter the path to your shell script: ") context = input("\nProvide additional context for the script (optional): ") @@ -778,8 +925,13 @@ def main(): start_time = time.time() errors_encountered = [] + errors_text = "" # Initialize errors_text here + success = False + dependency_files_generated = False + additional_instruction = "" while attempt <= max_attempts: + made_dependency_change = False if attempt == 1: print(f"\n{'='*40}\nAttempt {attempt}: Generating Exec Doc...\n{'='*40}") response = client.chat.completions.create( @@ -792,20 +944,47 @@ def main(): output_file_content = response.choices[0].message.content with open(output_file, "w") as f: f.write(output_file_content) + + # Generate dependency files after first creation + if generate_deps and not dependency_files_generated: + _, dependency_files = generate_dependency_files(output_file) + dependency_files_generated = True else: print(f"\n{'='*40}\nAttempt {attempt}: Generating corrections based on error...\n{'='*40}") - response = client.chat.completions.create( - model=deployment_name, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": input_content}, - {"role": "assistant", "content": output_file_content}, - {"role": "user", "content": f"The following error(s) have occurred during testing:\n{errors_text}\n{additional_instruction}\n\nPlease carefully analyze these errors and make necessary corrections to the document to prevent them from happening again. Try to find different solutions if the same errors keep occurring. \nGiven that context, please think hard and don't hurry. I want you to correct the converted document in ALL instances where this error has been or can be found. Then, correct ALL other errors apart from this that you see in the doc. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"} - ] - ) - output_file_content = response.choices[0].message.content - with open(output_file, "w") as f: - f.write(output_file_content) + + # Use a flag to track if we made a dependency change + # made_dependency_change = False + + # Analyze if the error is in the main doc or in dependency files + error_analysis = analyze_error(errors_text, dependency_files) + + if error_analysis["type"] == "dependency_file" and error_analysis["file"]: + # If error is in a dependency file, try to fix it + dep_file = error_analysis["file"] + print(f"\nDetected issue in dependency file: {dep_file['filename']}") + update_dependency_file(dep_file, error_analysis["message"], output_file) + made_dependency_change = True # Set the flag + else: + # If error is in main doc or unknown, update the main doc + response = client.chat.completions.create( + model=deployment_name, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": input_content}, + {"role": "assistant", "content": output_file_content}, + {"role": "user", "content": f"The following error(s) have occurred during testing:\n{errors_text}\n{additional_instruction}\n\nPlease carefully analyze these errors and make necessary corrections to the document to prevent them from happening again. Try to find different solutions if the same errors keep occurring. \nGiven that context, please think hard and don't hurry. I want you to correct the converted document in ALL instances where this error has been or can be found. Then, correct ALL other errors apart from this that you see in the doc. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"} + ] + ) + output_file_content = response.choices[0].message.content + with open(output_file, "w") as f: + f.write(output_file_content) + + # Check if we need to regenerate dependency files after updating main doc + if generate_deps and dependency_files_generated: + # Regenerate dependency files if major changes were made to the main doc + _, updated_dependency_files = generate_dependency_files(output_file) + if updated_dependency_files: + dependency_files = updated_dependency_files remove_backticks_from_file(output_file) @@ -817,6 +996,7 @@ def main(): errors_encountered.append("The 'ie test' command timed out after 11 minutes.") attempt += 1 continue # Proceed to the next attempt + if result.returncode == 0: print(f"\n{'*'*40}\nAll tests passed successfully.\n{'*'*40}") success = True @@ -831,9 +1011,15 @@ def main(): {"role": "user", "content": f"Take the working converted Exec Doc and merge it with the original source document provided for conversion as needed. Ensure that every piece of information outside of code blocks – such as metadata, descriptions, comments, instructions, and any other narrative content – is preserved. The final output should be a comprehensive document that retains all correct code blocks as well as the rich contextual and descriptive details from the source doc, creating the best of both worlds. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"} ] ) - output_file_content = response.choices[0].message.content - with open(output_file, "w") as f: - f.write(output_file_content) + output_file_content = response.choices[0].message.content + with open(output_file, "w") as f: + f.write(output_file_content) + + # Generate dependency files for successful docs if not already done + if (input_type == 'file' or input_type == 'workload_description') and not dependency_files_generated and generate_deps: + print("\nGenerating dependency files for the successful document...") + _, dependency_files = generate_dependency_files(output_file) + remove_backticks_from_file(output_file) break else: @@ -889,7 +1075,10 @@ def main(): print(f"\nError: {error_log.strip()}") print(f"\n{'!'*40}\nApplying an error troubleshooting strategy...\n{'!'*40}") - attempt += 1 + + # Only increment attempt if we didn't make a dependency change + if not made_dependency_change: + attempt += 1 success = False if attempt > max_attempts: diff --git a/tools/execution_log.csv b/tools/execution_log.csv index 2bb0c316f..c16086ab1 100644 --- a/tools/execution_log.csv +++ b/tools/execution_log.csv @@ -1221,3 +1221,18 @@ See vm create -h for more information on specifying an image. ' StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. See vm create -h for more information on specifying an image.",87.87348413467407,Success +2025-03-05 22:12:36,workload_description,create a linux vm and ssh into it,Deploy Linux VM with SSH Access in Azure_ai_generated.md,1,"time=2025-03-05T22:10:23-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. +' +StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image.",160.46278858184814,Success +2025-03-06 00:09:24,file,doc.md,doc_converted.md,2,"time=2025-03-06T00:04:08-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 5. +Error: command exited with 'exit status 127' and the message 'bash: line 2: Get-AzVm: command not found +' +StdErr: bash: line 2: Get-AzVm: command not found + + time=2025-03-06T00:06:37-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 5. +Error: command exited with 'exit status 127' and the message 'bash: line 2: pwsh: command not found +' +StdErr: bash: line 2: pwsh: command not found",578.4860949516296,Success From 14f103b02f75d520a1dcda1a9383f819ea72c844 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 6 Mar 2025 01:49:38 -0800 Subject: [PATCH 13/87] updated readme around tool --- tools/README.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tools/README.md b/tools/README.md index ddbaa4404..ab42a9901 100644 --- a/tools/README.md +++ b/tools/README.md @@ -10,7 +10,6 @@ Welcome to ADA! This tool helps you convert documents and troubleshoot errors ef - Redacts Personally Identifiable Information (PII) from Exec Doc result blocks. - Automatically identifies and generates dependency files referenced in documents. - Performs comprehensive security vulnerability analysis on Exec Docs. -- Runs tests on the converted document using the Innovation Engine. - Logs execution data to a CSV file for analytics. ## Prerequisites @@ -150,6 +149,33 @@ Welcome to ADA! This tool helps you convert documents and troubleshoot errors ef 8. **Logging**: Logs execution data to `execution_log.csv`. +## Advanced Features + +### Dependency File Management +ADA can identify, generate, and manage auxiliary files referenced in your Exec Docs: +- Automatically detects files referenced in the document +- Creates dependency files with proper formatting based on file type +- Tracks existing files to prevent overwriting user modifications +- Intelligently updates dependency files when errors are detected +- Regenerates dependencies when major document changes occur + +### Error Resolution System +When errors occur during testing, ADA employs a sophisticated resolution system: +- Analyzes errors to determine if they originate in main document or dependency files +- Uses progressive troubleshooting strategies for persistent errors +- Only counts attempts against the maximum when fixing the main document +- Provides specific strategies for different error patterns +- Remembers previous errors to avoid repetitive solutions + +### Progressive Error Strategies +ADA uses increasingly more aggressive strategies when encountering repeated errors: +1. Target specific issues identified in error messages +2. Simplify complex code blocks into smaller, manageable steps +3. Remove problematic result blocks that may be causing validation issues +4. Try alternative commands or approaches to achieve the same result +5. Completely redesign problematic sections with simpler implementations +6. Remove and rebuild problematic sections from scratch + ## Logging The script logs the following data to `execution_log.csv`: From d7dd7a8d600f1166aebe20119de2c0b3f03d2619 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 6 Mar 2025 13:33:33 -0800 Subject: [PATCH 14/87] updated doc and tool --- ...y Linux VM with SSH Access_ai_generated.md | 108 +++ tools/ada.py | 106 ++- tools/aks.sh | 45 + tools/aks_documented.md | 144 ++++ tools/convert.md | 141 +++ tools/convert_converted.md | 194 +++++ tools/doc.md | 362 -------- tools/execution_log.csv | 30 + tools/mongodb.md | 816 ++++++++++++++++++ tools/mongodb_redacted.md | 815 +++++++++++++++++ tools/mongodb_security_report.md | 90 ++ 11 files changed, 2487 insertions(+), 364 deletions(-) create mode 100644 tools/Deploy Linux VM with SSH Access_ai_generated.md create mode 100644 tools/aks.sh create mode 100644 tools/aks_documented.md create mode 100644 tools/convert.md create mode 100644 tools/convert_converted.md delete mode 100644 tools/doc.md create mode 100644 tools/mongodb.md create mode 100644 tools/mongodb_redacted.md create mode 100644 tools/mongodb_security_report.md diff --git a/tools/Deploy Linux VM with SSH Access_ai_generated.md b/tools/Deploy Linux VM with SSH Access_ai_generated.md new file mode 100644 index 000000000..7994ce6a6 --- /dev/null +++ b/tools/Deploy Linux VM with SSH Access_ai_generated.md @@ -0,0 +1,108 @@ +--- +title: 'Quickstart: Create a Linux VM and SSH into it' +description: Learn how to create a Linux virtual machine in Azure using Azure CLI and then SSH into it. +ms.topic: quickstart +ms.date: 10/12/2023 +author: yourgithubusername +ms.author: yourgithubusername +ms.custom: innovation-engine, azurecli, linux-related-content +--- + +# Quickstart: Create a Linux VM and SSH into it + +This Exec Doc demonstrates how to create a resource group, deploy a Linux VM using a supported Ubuntu image, retrieve its public IP address, and then SSH into the VM. The process uses environment variables to manage configuration details and appends a random suffix to resource names to ensure uniqueness. + +The following sections walk through each step with code blocks. Remember that you must already be logged in to Azure and have your subscription set. + +## Step 1: Create a Resource Group + +In this section, we declare environment variables necessary for the deployment and create a resource group in the "centralindia" region. A random suffix is appended to the resource group name to guarantee uniqueness. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="centralindia" +export RG_NAME="LinuxRG$RANDOM_SUFFIX" +az group create --name $RG_NAME --location $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/LinuxRGabc123", + "location": "centralindia", + "managedBy": null, + "name": "LinuxRGabc123", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Step 2: Create a Linux Virtual Machine + +Now we create a Linux VM using a supported Ubuntu image ('Ubuntu2204'). In this example, we use a Standard_B1s VM size. We also set an administrator username and let Azure generate SSH key pairs automatically. A random suffix is used in the VM name for uniqueness. + +```bash +export VM_NAME="LinuxVM$RANDOM_SUFFIX" +export ADMIN_USERNAME="azureuser" +az vm create \ + --resource-group $RG_NAME \ + --name $VM_NAME \ + --image Ubuntu2204 \ + --size Standard_B1s \ + --admin-username $ADMIN_USERNAME \ + --generate-ssh-keys +``` + +Results: + + +```JSON +{ + "fqdns": "", + "id": "/subscriptions/xxxxx/resourceGroups/LinuxRGabc123/providers/Microsoft.Compute/virtualMachines/LinuxVMabc123", + "location": "centralindia", + "macAddress": "00-0X-0X-0X-0X-0X", + "machineId": "xxxxx", + "name": "LinuxVMabc123", + "powerState": "VM running", + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "13.92.xxx.xxx", + "resourceGroup": "LinuxRGabc123", + "zones": "" +} +``` + +## Step 3: Retrieve the VM Public IP Address + +This step retrieves the public IP address of the newly created VM. The public IP is stored in an environment variable to be used in the SSH step. + +```bash +export VM_PUBLIC_IP=$(az vm list-ip-addresses --resource-group $RG_NAME --name $VM_NAME --query "[].virtualMachine.network.publicIpAddresses[0].ipAddress" --output tsv) +echo "The public IP address of the VM is: $VM_PUBLIC_IP" +``` + +Results: + + +```text +The public IP address of the VM is: 13.92.xxx.xxx +``` + +## Step 4: SSH into the Linux VM + +Finally, once you have retrieved the public IP address, you can SSH into your Linux VM using the generated SSH key pair. This command establishes an SSH connection without prompting for host key verification. + +```bash +ssh -o StrictHostKeyChecking=no $ADMIN_USERNAME@$VM_PUBLIC_IP +``` + +When executed, this command initiates an SSH session with your Linux VM. After connecting, you will have full access to manage and configure the VM as needed. + +--- + +This Exec Doc has successfully deployed a Linux VM in Azure using a supported Ubuntu image and shown how to connect to it using SSH, all accomplished with a series of Azure CLI commands executed via the Innovation Engine. \ No newline at end of file diff --git a/tools/ada.py b/tools/ada.py index 2ba497894..b4116edc7 100644 --- a/tools/ada.py +++ b/tools/ada.py @@ -438,6 +438,31 @@ def generate_dependency_files(doc_path): print("\nAnalyzing document for dependencies...") + # First, detect file creation patterns in the document to avoid conflicts + file_creation_patterns = [ + # Cat heredoc to a file + (r'cat\s*<<\s*[\'"]?(EOF|END)[\'"]?\s*>\s*([^\s;]+)', 1), + # Echo content to a file + (r'echo\s+.*?>\s*([^\s;]+)', 0), + # Tee command + (r'tee\s+([^\s;]+)', 0) + ] + + doc_created_files = [] + for pattern, group_idx in file_creation_patterns: + matches = re.findall(pattern, doc_content, re.DOTALL) + for match in matches: + if isinstance(match, tuple): + filename = match[group_idx] + else: + filename = match + doc_created_files.append(filename) + + if doc_created_files: + print("\nDetected file creation commands in document:") + for file in doc_created_files: + print(f" - {file}") + # Enhanced prompt for better dependency file identification dependency_prompt = """Analyze this Exec Doc and identify ANY files that the user is instructed to create. @@ -507,9 +532,23 @@ def generate_dependency_files(doc_path): print("\nNo dependency files identified.") return True, [] + # Filter out dependency files that have inline creation commands in the document + filtered_deps = [] + for dep in dependency_list: + filename = dep.get("filename") + if not filename: + continue + + if filename in doc_created_files: + print(f"\nWARNING: File '{filename}' is both created in document and identified as a dependency.") + print(f" - Skipping dependency management for this file to avoid conflicts.") + continue + + filtered_deps.append(dep) + # Create each dependency file with type-specific handling created_files = [] - for dep in dependency_list: + for dep in filtered_deps: filename = dep.get("filename") content = dep.get("content") file_type = dep.get("type", "").lower() @@ -596,6 +635,60 @@ def generate_dependency_files(doc_path): print("\nResponse from model was not valid JSON. Raw response:") return False, [] +# Add this function after generate_dependency_files function (approximately line 609) + +def transform_document_for_dependencies(doc_path, dependency_files): + """Remove file creation commands from document when using dependency files.""" + if not dependency_files: + return False + + try: + with open(doc_path, "r") as f: + doc_content = f.read() + + original_content = doc_content + modified = False + + for dep_file in dependency_files: + filename = dep_file["filename"] + + # Pattern to match cat/EOF blocks for file creation + cat_pattern = re.compile( + r'```(?:bash|azurecli|azure-cli-interactive|azurecli-interactive)\s*\n' + r'(.*?cat\s*<<\s*[\'"]?(EOF|END)[\'"]?\s*>\s*' + re.escape(filename) + r'.*?EOF.*?)' + r'\n```', + re.DOTALL + ) + + # Replace with a reference to the external file + if cat_pattern.search(doc_content): + replacement = f"```bash\n# Using external file: {filename}\n```\n\n" + doc_content = cat_pattern.sub(replacement, doc_content) + modified = True + print(f"\nTransformed document to use external {filename} instead of inline creation") + + # Handle other file creation patterns (echo, tee) + echo_pattern = re.compile( + r'```(?:bash|azurecli|azure-cli-interactive|azurecli-interactive)\s*\n' + r'(.*?echo\s+.*?>\s*' + re.escape(filename) + r'.*?)' + r'\n```', + re.DOTALL + ) + if echo_pattern.search(doc_content): + replacement = f"```bash\n# Using external file: {filename}\n```\n\n" + doc_content = echo_pattern.sub(replacement, doc_content) + modified = True + + if modified: + with open(doc_path, "w") as f: + f.write(doc_content) + print("\nDocument transformed to use external dependency files") + return True + return False + except Exception as e: + print(f"\nError transforming document: {e}") + return False + def update_dependency_file(file_info, error_message, main_doc_path): """Update a dependency file based on error message.""" filename = file_info["filename"] @@ -857,7 +950,7 @@ def main(): print(" 2. Describe workload to generate a new Exec Doc") print(" 3. Add descriptions to a shell script as an Exec Doc") print(" 4. Redact PII from an existing Exec Doc") - print(" 5. Perform security vulnerability check on an Exec Doc") + print(" 5. Generate a security analysis report for an Exec Doc") choice = input("\nEnter the number corresponding to your choice: ") if choice == "1": @@ -949,6 +1042,15 @@ def main(): if generate_deps and not dependency_files_generated: _, dependency_files = generate_dependency_files(output_file) dependency_files_generated = True + + # Generate dependency files after first creation + if generate_deps and not dependency_files_generated: + _, dependency_files = generate_dependency_files(output_file) + dependency_files_generated = True + + # Add this new line to transform the document after dependency generation + if dependency_files: + transform_document_for_dependencies(output_file, dependency_files) else: print(f"\n{'='*40}\nAttempt {attempt}: Generating corrections based on error...\n{'='*40}") diff --git a/tools/aks.sh b/tools/aks.sh new file mode 100644 index 000000000..48719c724 --- /dev/null +++ b/tools/aks.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# This script creates an AKS cluster using Azure CLI + +# Exit on error +set -e + +# Configuration variables +RESOURCE_GROUP="myAKSResourceGroup" +LOCATION="eastus" +CLUSTER_NAME="myAKSCluster" +NODE_COUNT=3 +NODE_VM_SIZE="Standard_DS2_v2" +KUBERNETES_VERSION="1.26.3" # Check available versions with: az aks get-versions --location $LOCATION --output table + +# Login to Azure (uncomment if not already logged in) +# az login + +# Create resource group +echo "Creating resource group $RESOURCE_GROUP in $LOCATION..." +az group create --name $RESOURCE_GROUP --location $LOCATION + +# Create AKS cluster +echo "Creating AKS cluster $CLUSTER_NAME..." +az aks create \ + --resource-group $RESOURCE_GROUP \ + --name $CLUSTER_NAME \ + --node-count $NODE_COUNT \ + --node-vm-size $NODE_VM_SIZE \ + --kubernetes-version $KUBERNETES_VERSION \ + --generate-ssh-keys \ + --enable-managed-identity \ + --enable-cluster-autoscaler \ + --min-count 1 \ + --max-count 5 + +# Get credentials for the Kubernetes cluster +echo "Getting credentials for cluster $CLUSTER_NAME..." +az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME + +echo "AKS cluster $CLUSTER_NAME has been created successfully!" +echo "You can now use kubectl to manage your cluster" + +# Verify connection to the cluster +echo "Verifying connection to the cluster..." +kubectl get nodes \ No newline at end of file diff --git a/tools/aks_documented.md b/tools/aks_documented.md new file mode 100644 index 000000000..74fb71d55 --- /dev/null +++ b/tools/aks_documented.md @@ -0,0 +1,144 @@ +--- +title: Explanation: AKS Cluster Creation Script +description: This Exec Doc explains a shell script that creates an AKS cluster using Azure CLI. The document walks you through each functional block to help you understand the purpose of the script and how each section contributes to the overall process. +ms.topic: article +ms.date: 2023-10-12 +author: chatgpt +ms.author: chatgpt +ms.custom: innovation-engine, ms-learn, azure, cluster-creation +--- + +# Explanation: AKS Cluster Creation Script + +In this Exec Doc, we examine a shell script that automates the process of creating an Azure Kubernetes Service (AKS) cluster. The script covers several key tasks: setting safe execution options, defining configuration variables, creating a resource group, deploying the AKS cluster, retrieving credentials, and finally verifying the cluster connectivity. Read on to understand the purpose and function of each block. + +--- + +## Script Header and Safety Settings + +Below the shebang line, the script uses `set -e` to ensure that the script exits immediately upon encountering any error. This helps prevent cascading failures during the deployment process. + +```bash +#!/bin/bash +# This script creates an AKS cluster using Azure CLI + +# Exit on error +set -e +``` + +The above code ensures that any failure in subsequent commands stops the script, thereby protecting against unintended side effects. + +--- + +## Configuration Variables + +This section defines the necessary configuration variables for the deployment. These variables include the resource group name, location, cluster name, node count, node VM size, and the Kubernetes version. The comments also guide you on how to check for available Kubernetes versions using the Azure CLI. + +```bash +# Configuration variables +RESOURCE_GROUP="myAKSResourceGroup" +LOCATION="eastus" +CLUSTER_NAME="myAKSCluster" +NODE_COUNT=3 +NODE_VM_SIZE="Standard_DS2_v2" +KUBERNETES_VERSION="1.26.3" # Check available versions with: az aks get-versions --location $LOCATION --output table +``` + +Each variable is critical for the subsequent commands that create and configure the AKS cluster. Note that these values are hardcoded; changing them will adjust the deployment accordingly. + +--- + +## (Optional) Azure Login Comment + +The script includes a commented-out Azure login command. This serves as a reminder to log in if you aren’t already authenticated. Since the Exec Doc guidelines do not allow login commands, the line remains commented out. + +```bash +# Login to Azure (uncomment if not already logged in) +# az login +``` + +This block is informational and does not affect the execution when the script is run in a pre-authenticated session. + +--- + +## Creating the Resource Group + +Before deploying the AKS cluster, the script creates a resource group in the specified location. This resource group will contain all the resources associated with the AKS cluster. + +```bash +# Create resource group +echo "Creating resource group $RESOURCE_GROUP in $LOCATION..." +az group create --name $RESOURCE_GROUP --location $LOCATION +``` + +The echo statement provides user feedback, while the `az group create` command creates the resource group if it does not already exist. + +--- + +## Deploying the AKS Cluster + +The next functional block involves the creation of the AKS cluster. The script uses several parameters to customize the deployment, such as node count, VM size, Kubernetes version, SSH key generation, managed identity, and autoscaling settings. + +```bash +# Create AKS cluster +echo "Creating AKS cluster $CLUSTER_NAME..." +az aks create \ + --resource-group $RESOURCE_GROUP \ + --name $CLUSTER_NAME \ + --node-count $NODE_COUNT \ + --node-vm-size $NODE_VM_SIZE \ + --kubernetes-version $KUBERNETES_VERSION \ + --generate-ssh-keys \ + --enable-managed-identity \ + --enable-cluster-autoscaler \ + --min-count 1 \ + --max-count 5 +``` + +This block deploys the AKS cluster with the defined specifications. It also enables cluster autoscaling between 1 and 5 nodes to adapt to workload demands. + +--- + +## Retrieving Cluster Credentials + +Once the AKS cluster is deployed, the script retrieves the cluster's credentials. This allows you to manage the Kubernetes cluster using the `kubectl` command-line tool. + +```bash +# Get credentials for the Kubernetes cluster +echo "Getting credentials for cluster $CLUSTER_NAME..." +az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME +``` + +The credentials command updates your local kubeconfig file, enabling seamless interaction with your cluster. + +--- + +## Final Confirmation and Cluster Verification + +After the credentials are fetched, the script prints success messages and then verifies the cluster connection by listing the cluster nodes using `kubectl`. + +```bash +echo "AKS cluster $CLUSTER_NAME has been created successfully!" +echo "You can now use kubectl to manage your cluster" + +# Verify connection to the cluster +echo "Verifying connection to the cluster..." +kubectl get nodes +``` + +This verification confirms that the cluster is operational and that the kubectl context is correctly set up. + +Results: + + + +```console +NAME STATUS ROLES AGE VERSION +aks-nodepool1-abcdef12-vmss000000 Ready agent 5m v1.26.3 +``` + +The above result block illustrates a typical output from `kubectl get nodes`, indicating that at least one node in the AKS cluster is ready and connected. + +--- + +This Exec Doc provides a short and sweet explanation of every major functional block in the AKS cluster creation script. By following the annotated steps, you gain a clearer understanding of how cloud resources are provisioned in a streamlined, automated manner. \ No newline at end of file diff --git a/tools/convert.md b/tools/convert.md new file mode 100644 index 000000000..e050b2055 --- /dev/null +++ b/tools/convert.md @@ -0,0 +1,141 @@ +--- +title: 'How-to: Create and deploy an Azure OpenAI Service resource' +titleSuffix: Azure OpenAI +description: Learn how to get started with Azure OpenAI Service and create your first resource and deploy your first model in the Azure CLI or the Azure portal. +#services: cognitive-services +manager: nitinme +ms.service: azure-ai-openai +ms.custom: devx-track-azurecli, build-2023, build-2023-dataai, devx-track-azurepowershell +ms.topic: how-to +ms.date: 01/31/2025 +zone_pivot_groups: openai-create-resource +author: mrbullwinkle +ms.author: mbullwin +recommendations: false +--- + +# Create and deploy an Azure OpenAI Service resource + +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://go.microsoft.com/fwlink/?linkid=2303211) + +This article describes how to get started with Azure OpenAI Service and provides step-by-step instructions to create a resource and deploy a model. You can create resources in Azure in several different ways: + +- The [Azure portal](https://portal.azure.com/?microsoft_azure_marketplace_ItemHideKey=microsoft_openai_tip#create/Microsoft.CognitiveServicesOpenAI) +- The REST APIs, the Azure CLI, PowerShell, or client libraries +- Azure Resource Manager (ARM) templates + +In this article, you review examples for creating and deploying resources in the Azure portal and with the Azure CLI. + +## Prerequisites + +- An Azure subscription. Create one for free. +- Access permissions to [create Azure OpenAI resources and to deploy models](../how-to/role-based-access-control.md). +- The Azure CLI. For more information, see [How to install the Azure CLI](/cli/azure/install-azure-cli). + +## Create an Azure resource group + +To create an Azure OpenAI resource, you need an Azure resource group. When you create a new resource through the Azure CLI, you can also create a new resource group or instruct Azure to use an existing group. The following example shows how to create a new resource group named _OAIResourceGroup_ with the [az group create](/cli/azure/group?view=azure-cli-latest&preserve-view=true#az-group-create) command. The resource group is created in the East US location. + +```azurecli-interactive +az group create \ +--name OAIResourceGroup \ +--location eastus +``` + +## Create a resource + +Use the [az cognitiveservices account create](/cli/azure/cognitiveservices/account?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-create) command to create an Azure OpenAI resource in the resource group. In the following example, you create a resource named _MyOpenAIResource_ in the _OAIResourceGroup_ resource group. When you try the example, update the code to use your desired values for the resource group and resource name, along with your Azure subscription ID _\_. + +```azurecli +az cognitiveservices account create \ +--name MyOpenAIResource \ +--resource-group OAIResourceGroup \ +--location eastus \ +--kind OpenAI \ +--sku s0 \ +--subscription +``` + +## Retrieve information about the resource + +After you create the resource, you can use different commands to find useful information about your Azure OpenAI Service instance. The following examples demonstrate how to retrieve the REST API endpoint base URL and the access keys for the new resource. + +### Get the endpoint URL + +Use the [az cognitiveservices account show](/cli/azure/cognitiveservices/account?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-show) command to retrieve the REST API endpoint base URL for the resource. In this example, we direct the command output through the [jq](https://jqlang.github.io/jq/) JSON processor to locate the `.properties.endpoint` value. + +When you try the example, update the code to use your values for the resource group _\_ and resource _\_. + +```azurecli +az cognitiveservices account show \ +--name \ +--resource-group \ +| jq -r .properties.endpoint +``` + +### Get the primary API key + +To retrieve the access keys for the resource, use the [az cognitiveservices account keys list](/cli/azure/cognitiveservices/account?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-keys-list) command. In this example, we direct the command output through the [jq](https://jqlang.github.io/jq/) JSON processor to locate the `.key1` value. + +When you try the example, update the code to use your values for the resource group and resource. + +```azurecli +az cognitiveservices account keys list \ +--name \ +--resource-group \ +| jq -r .key1 +``` + +## Deploy a model + +To deploy a model, use the [az cognitiveservices account deployment create](/cli/azure/cognitiveservices/account/deployment?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-deployment-create) command. In the following example, you deploy an instance of the `text-embedding-ada-002` model and give it the name _MyModel_. When you try the example, update the code to use your values for the resource group and resource. You don't need to change the `model-version`, `model-format` or `sku-capacity`, and `sku-name` values. + +```azurecli +az cognitiveservices account deployment create \ +--name \ +--resource-group \ +--deployment-name MyModel \ +--model-name text-embedding-ada-002 \ +--model-version "1" \ +--model-format OpenAI \ +--sku-capacity "1" \ +--sku-name "Standard" +``` + +`--sku-name` accepts the following deployment types: `Standard`, `GlobalBatch`, `GlobalStandard`, and `ProvisionedManaged`. Learn more about [deployment type options](../how-to/deployment-types.md). + + +> [!IMPORTANT] +> When you access the model via the API, you need to refer to the deployment name rather than the underlying model name in API calls, which is one of the [key differences](../how-to/switching-endpoints.yml) between OpenAI and Azure OpenAI. OpenAI only requires the model name. Azure OpenAI always requires deployment name, even when using the model parameter. In our docs, we often have examples where deployment names are represented as identical to model names to help indicate which model works with a particular API endpoint. Ultimately your deployment names can follow whatever naming convention is best for your use case. + +## Delete a model from your resource + +You can delete any model deployed from your resource with the [az cognitiveservices account deployment delete](/cli/azure/cognitiveservices/account/deployment?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-deployment-delete) command. In the following example, you delete a model named _MyModel_. When you try the example, update the code to use your values for the resource group, resource, and deployed model. + +```azurecli +az cognitiveservices account deployment delete \ +--name \ +--resource-group \ +--deployment-name MyModel +``` + +## Delete a resource + +If you want to clean up after these exercises, you can remove your Azure OpenAI resource by deleting the resource through the Azure CLI. You can also delete the resource group. If you choose to delete the resource group, all resources contained in the group are also deleted. + +To remove the resource group and its associated resources, use the [az cognitiveservices account delete](/cli/azure/cognitiveservices/account/deployment?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-delete) command. + +If you're not going to continue to use the resources created in these exercises, run the following command to delete your resource group. Be sure to update the example code to use your values for the resource group and resource. + +```azurecli +az cognitiveservices account delete \ +--name \ +--resource-group +``` + +## Next steps + +- [Get started with the Azure OpenAI security building block](/azure/developer/ai/get-started-securing-your-ai-app?tabs=github-codespaces&pivots=python) +- Make API calls and generate text with [Azure OpenAI Service quickstarts](../quickstart.md). +- Learn more about the [Azure OpenAI Service models](../concepts/models.md). +- For information on pricing visit the [Azure OpenAI pricing page](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/) \ No newline at end of file diff --git a/tools/convert_converted.md b/tools/convert_converted.md new file mode 100644 index 000000000..3544ff150 --- /dev/null +++ b/tools/convert_converted.md @@ -0,0 +1,194 @@ +--- +title: 'How-to: Create and deploy an Azure OpenAI Service resource' +titleSuffix: Azure OpenAI +description: Learn how to get started with Azure OpenAI Service and create your first resource and deploy your first model in the Azure CLI or the Azure portal. +#services: cognitive-services +manager: nitinme +ms.service: azure-ai-openai +ms.custom: devx-track-azurecli, build-2023, build-2023-dataai, devx-track-azurepowershell, innovation-engine +ms.topic: how-to +ms.date: 01/31/2025 +zone_pivot_groups: openai-create-resource +author: mrbullwinkle +ms.author: mbullwin +recommendations: false +--- + +# Create and deploy an Azure OpenAI Service resource + +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://go.microsoft.com/fwlink/?linkid=2303211) + +This article describes how to get started with Azure OpenAI Service and provides step-by-step instructions to create a resource and deploy a model. You can create resources in Azure in several different ways: + +- The [Azure portal](https://portal.azure.com/?microsoft_azure_marketplace_ItemHideKey=microsoft_openai_tip#create/Microsoft.CognitiveServicesOpenAI) +- The REST APIs, the Azure CLI, PowerShell, or client libraries +- Azure Resource Manager (ARM) templates + +In this article, you review examples for creating and deploying resources in the Azure portal and with the Azure CLI. + +## Prerequisites + +- An Azure subscription. Create one for free. +- Access permissions to [create Azure OpenAI resources and to deploy models](../how-to/role-based-access-control.md). +- The Azure CLI. For more information, see [How to install the Azure CLI](/cli/azure/install-azure-cli). + +## Create an Azure resource group + +To create an Azure OpenAI resource, you need an Azure resource group. When you create a new resource through the Azure CLI, you can also create a new resource group or instruct Azure to use an existing group. The following example shows how to create a new resource group named OAIResourceGroup with the az group create command. The resource group is created in the East US location. + +```azurecli +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="eastus" +export OAI_RESOURCE_GROUP="OAIResourceGroup$RANDOM_SUFFIX" +az group create --name $OAI_RESOURCE_GROUP --location $REGION +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/OAIResourceGroupxxxxx", + "location": "eastus", + "managedBy": null, + "name": "OAIResourceGroupxxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create a resource + +Use the az cognitiveservices account create command to create an Azure OpenAI resource in the resource group. In the following example, you create a resource named MyOpenAIResource in the OAI_RESOURCE_GROUP resource group. When you try the example, update the code to use your desired values for the resource group and resource name. + +```azurecli +export OPENAI_RESOURCE_NAME="MyOpenAIResource$RANDOM_SUFFIX" +az cognitiveservices account create \ +--name $OPENAI_RESOURCE_NAME \ +--resource-group $OAI_RESOURCE_GROUP \ +--location $REGION \ +--kind OpenAI \ +--sku s0 +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/OAIResourceGroupxxxxx/providers/Microsoft.CognitiveServices/accounts/MyOpenAIResourcexxxxx", + "kind": "OpenAI", + "location": "eastus", + "name": "MyOpenAIResourcexxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "sku": { + "name": "s0" + }, + "type": "Microsoft.CognitiveServices/accounts" +} +``` + +## Retrieve information about the resource + +After you create the resource, you can use different commands to find useful information about your Azure OpenAI Service instance. The following examples demonstrate how to retrieve the REST API endpoint base URL and the access keys for the new resource. + +### Get the endpoint URL + +Use the az cognitiveservices account show command to retrieve the REST API endpoint base URL for the resource. In this example, we direct the command output through the jq JSON processor to locate the .properties.endpoint value. + +When you try the example, update the code to use your values for the resource group and resource. + +```azurecli +az cognitiveservices account show \ +--name $OPENAI_RESOURCE_NAME \ +--resource-group $OAI_RESOURCE_GROUP \ +| jq -r .properties.endpoint +``` + +Results: + + + +```text +https://openaiendpointxxxxx.cognitiveservices.azure.com/ +``` + +### Get the primary API key + +To retrieve the access keys for the resource, use the az cognitiveservices account keys list command. In this example, we direct the command output through the jq JSON processor to locate the .key1 value. + +When you try the example, update the code to use your values for the resource group and resource. + +```azurecli +az cognitiveservices account keys list \ +--name $OPENAI_RESOURCE_NAME \ +--resource-group $OAI_RESOURCE_GROUP \ +| jq -r .key1 +``` + +Results: + + + +```text +xxxxxxxxxxxxxxxxxxxxxx +``` + +## Deploy a model + +To deploy a model, use the az cognitiveservices account deployment create command. In the following example, you deploy an instance of the text-embedding-ada-002 model and give it the name MyModel. When you try the example, update the code to use your values for the resource group and resource. You don't need to change the model-version, model-format, sku-capacity, or sku-name values. + +```azurecli +export MODEL_DEPLOYMENT_NAME="MyModel" +az cognitiveservices account deployment create \ +--name $OPENAI_RESOURCE_NAME \ +--resource-group $OAI_RESOURCE_GROUP \ +--deployment-name $MODEL_DEPLOYMENT_NAME \ +--model-name text-embedding-ada-002 \ +--model-version "1" \ +--model-format OpenAI \ +--sku-capacity "1" \ +--sku-name "Standard" +``` + +Results: + + + +```JSON +{ + "deploymentName": "MyModel", + "provisioningState": "Succeeded" +} +``` + +> [!IMPORTANT] +> When you access the model via the API, you need to refer to the deployment name rather than the underlying model name in API calls, which is one of the [key differences](../how-to/switching-endpoints.yml) between OpenAI and Azure OpenAI. OpenAI only requires the model name. Azure OpenAI always requires deployment name, even when using the model parameter. In our docs, we often have examples where deployment names are represented as identical to model names to help indicate which model works with a particular API endpoint. Ultimately your deployment names can follow whatever naming convention is best for your use case. + +## Delete a model from your resource + +You can delete any model deployed from your resource with the az cognitiveservices account deployment delete command. In the following example, the original document provided instructions to delete a model named MyModel. When you try the example, update the code to use your values for the resource group, resource, and deployed model. + +(Note: The deletion code block has been removed from this Exec Doc as deletion commands are not executed automatically in Exec Docs.) + +## Delete a resource + +If you want to clean up after these exercises, you can remove your Azure OpenAI resource by deleting the resource through the Azure CLI. You can also delete the resource group. If you choose to delete the resource group, all resources contained in the group are also deleted. + +To remove the resource group and its associated resources, the original document provided a command example. Be sure to update the example code to use your values for the resource group and resource. + +(Note: The deletion code block has been removed from this Exec Doc as deletion commands are not executed automatically in Exec Docs.) + +## Next steps + +- [Get started with the Azure OpenAI security building block](/azure/developer/ai/get-started-securing-your-ai-app?tabs=github-codespaces&pivots=python) +- Make API calls and generate text with [Azure OpenAI Service quickstarts](../quickstart.md). +- Learn more about the [Azure OpenAI Service models](../concepts/models.md). +- For information on pricing visit the [Azure OpenAI pricing page](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/) \ No newline at end of file diff --git a/tools/doc.md b/tools/doc.md deleted file mode 100644 index 63f5bf371..000000000 --- a/tools/doc.md +++ /dev/null @@ -1,362 +0,0 @@ ---- -title: 'Quickstart: Use Terraform to create a Linux VM' -description: In this quickstart, you learn how to use Terraform to create a Linux virtual machine -author: tomarchermsft -ms.service: azure-virtual-machines -ms.collection: linux -ms.topic: quickstart -ms.date: 07/24/2023 -ms.author: tarcher -ms.custom: devx-track-terraform, linux-related-content -content_well_notification: - - AI-contribution -ai-usage: ai-assisted ---- - -# Quickstart: Use Terraform to create a Linux VM - -**Applies to:** :heavy_check_mark: Linux VMs - -Article tested with the following Terraform and Terraform provider versions: - -This article shows you how to create a complete Linux environment and supporting resources with Terraform. Those resources include a virtual network, subnet, public IP address, and more. - -[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] - -In this article, you learn how to: -> [!div class="checklist"] -> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet). -> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group). -> * Create a virtual network (VNET) using [azurerm_virtual_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network). -> * Create a subnet using [azurerm_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet). -> * Create a public IP using [azurerm_public_ip](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/public_ip). -> * Create a network security group using [azurerm_network_security_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_group). -> * Create a network interface using [azurerm_network_interface](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface). -> * Create an association between the network security group and the network interface using [azurerm_network_interface_security_group_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface_security_group_association). -> * Generate a random value for a unique storage account name using [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id). -> * Create a storage account for boot diagnostics using [azurerm_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/storage_account). -> * Create a Linux VM using [azurerm_linux_virtual_machine](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_virtual_machine) -> * Create an AzAPI resource [azapi_resource](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource). -> * Create an AzAPI resource to generate an SSH key pair using [azapi_resource_action](https://registry.terraform.io/providers/Azure/azapi/latest/docs/resources/azapi_resource_action). - -## Prerequisites - -- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) - -## Implement the Terraform code - -> [!NOTE] -> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-vm-with-infrastructure/TestRecord.md). -> -> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) - -1. Create a directory in which to test the sample Terraform code and make it the current directory. - -1. Create a file named `providers.tf` and insert the following code: - - ```terraform - terraform { - required_version = ">=0.12" - - required_providers { - azapi = { - source = "azure/azapi" - version = "~>1.5" - } - azurerm = { - source = "hashicorp/azurerm" - version = "~>3.0" - } - random = { - source = "hashicorp/random" - version = "~>3.0" - } - } - } - - provider "azurerm" { - features {} - } - ``` - -1. Create a file named `ssh.tf` and insert the following code: - - ```terraform - resource "random_pet" "ssh_key_name" { - prefix = "ssh" - separator = "" - } - - resource "azapi_resource_action" "ssh_public_key_gen" { - type = "Microsoft.Compute/sshPublicKeys@2022-11-01" - resource_id = azapi_resource.ssh_public_key.id - action = "generateKeyPair" - method = "POST" - - response_export_values = ["publicKey", "privateKey"] - } - - resource "azapi_resource" "ssh_public_key" { - type = "Microsoft.Compute/sshPublicKeys@2022-11-01" - name = random_pet.ssh_key_name.id - location = azurerm_resource_group.rg.location - parent_id = azurerm_resource_group.rg.id - } - - output "key_data" { - value = azapi_resource_action.ssh_public_key_gen.output.publicKey - } - ``` - -1. Create a file named `main.tf` and insert the following code: - - ```terraform - resource "random_pet" "rg_name" { - prefix = var.resource_group_name_prefix - } - - resource "azurerm_resource_group" "rg" { - location = var.resource_group_location - name = random_pet.rg_name.id - } - - # Create virtual network - resource "azurerm_virtual_network" "my_terraform_network" { - name = "myVnet" - address_space = ["10.0.0.0/16"] - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - } - - # Create subnet - resource "azurerm_subnet" "my_terraform_subnet" { - name = "mySubnet" - resource_group_name = azurerm_resource_group.rg.name - virtual_network_name = azurerm_virtual_network.my_terraform_network.name - address_prefixes = ["10.0.1.0/24"] - } - - # Create public IPs - resource "azurerm_public_ip" "my_terraform_public_ip" { - name = "myPublicIP" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - allocation_method = "Dynamic" - } - - # Create Network Security Group and rule - resource "azurerm_network_security_group" "my_terraform_nsg" { - name = "myNetworkSecurityGroup" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - - security_rule { - name = "SSH" - priority = 1001 - direction = "Inbound" - access = "Allow" - protocol = "Tcp" - source_port_range = "*" - destination_port_range = "22" - source_address_prefix = "*" - destination_address_prefix = "*" - } - } - - # Create network interface - resource "azurerm_network_interface" "my_terraform_nic" { - name = "myNIC" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - - ip_configuration { - name = "my_nic_configuration" - subnet_id = azurerm_subnet.my_terraform_subnet.id - private_ip_address_allocation = "Dynamic" - public_ip_address_id = azurerm_public_ip.my_terraform_public_ip.id - } - } - - # Connect the security group to the network interface - resource "azurerm_network_interface_security_group_association" "example" { - network_interface_id = azurerm_network_interface.my_terraform_nic.id - network_security_group_id = azurerm_network_security_group.my_terraform_nsg.id - } - - # Generate random text for a unique storage account name - resource "random_id" "random_id" { - keepers = { - # Generate a new ID only when a new resource group is defined - resource_group = azurerm_resource_group.rg.name - } - - byte_length = 8 - } - - # Create storage account for boot diagnostics - resource "azurerm_storage_account" "my_storage_account" { - name = "diag${random_id.random_id.hex}" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - account_tier = "Standard" - account_replication_type = "LRS" - } - - # Create virtual machine - resource "azurerm_linux_virtual_machine" "my_terraform_vm" { - name = "myVM" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - network_interface_ids = [azurerm_network_interface.my_terraform_nic.id] - size = "Standard_DS1_v2" - - os_disk { - name = "myOsDisk" - caching = "ReadWrite" - storage_account_type = "Premium_LRS" - } - - source_image_reference { - publisher = "Canonical" - offer = "0001-com-ubuntu-server-jammy" - sku = "22_04-lts-gen2" - version = "latest" - } - - computer_name = "hostname" - admin_username = var.username - - admin_ssh_key { - username = var.username - public_key = azapi_resource_action.ssh_public_key_gen.output.publicKey - } - - boot_diagnostics { - storage_account_uri = azurerm_storage_account.my_storage_account.primary_blob_endpoint - } - } - ``` - -1. Create a file named `variables.tf` and insert the following code: - - ```terraform - variable "resource_group_location" { - type = string - default = "eastus" - description = "Location of the resource group." - } - - variable "resource_group_name_prefix" { - type = string - default = "rg" - description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." - } - - variable "username" { - type = string - description = "The username for the local account that will be created on the new VM." - default = "azureadmin" - } - ``` - -1. Create a file named `outputs.tf` and insert the following code: - - ```terraform - output "resource_group_name" { - value = azurerm_resource_group.rg.name - } - - output "public_ip_address" { - value = azurerm_linux_virtual_machine.my_terraform_vm.public_ip_address - } - ``` - -## Initialize Terraform - -Run terraform init to initialize the Terraform deployment. This command downloads the Azure provider required to manage your Azure resources. - -```bash -terraform init -upgrade -``` - -Key points: - -- The -upgrade parameter upgrades the necessary provider plugins to the newest version that complies with the configuration's version constraints. - -## Create a Terraform execution plan - -Run terraform plan to create an execution plan. - -```bash -terraform plan -out main.tfplan -``` - -Key points: - -- The terraform plan command creates an execution plan, but doesn't execute it. Instead, it determines what actions are necessary to create the configuration specified in your configuration files. This pattern allows you to verify whether the execution plan matches your expectations before making any changes to actual resources. -- The optional -out parameter allows you to specify an output file for the plan. Using the -out parameter ensures that the plan you reviewed is exactly what is applied. - -## Apply a Terraform execution plan - -Run terraform apply to apply the execution plan to your cloud infrastructure. - -```bash -terraform apply main.tfplan -``` - -Key points: - -- The example terraform apply command assumes you previously ran terraform plan -out main.tfplan. -- If you specified a different filename for the -out parameter, use that same filename in the call to terraform apply. -- If you didn't use the -out parameter, call terraform apply without any parameters. - -Cost information isn't presented during the virtual machine creation process for Terraform like it is for the [Azure portal](quick-create-portal.md). If you want to learn more about how cost works for virtual machines, see the [Cost optimization Overview page](../plan-to-manage-costs.md). - -## Verify the results - -#### [Azure CLI](#tab/azure-cli) - -1. Get the Azure resource group name. - - ```bash - resource_group_name=$(terraform output -raw resource_group_name) - ``` - -1. Run [az vm list](/cli/azure/vm#az-vm-list) with a [JMESPath](/cli/azure/query-azure-cli) query to display the names of the virtual machines created in the resource group. - - ```azurecli - az vm list \ - --resource-group $resource_group_name \ - --query "[].{\"VM Name\":name}" -o table - ``` - -#### [Azure PowerShell](#tab/azure-powershell) - -1. Get the Azure resource group name. - - ```console - $resource_group_name=$(terraform output -raw resource_group_name) - ``` - -1. Run [Get-AzVm](/powershell/module/az.compute/get-azvm) to display the names of all the virtual machines in the resource group. - - ```azurepowershell - Get-AzVm -ResourceGroupName $resource_group_name - ``` - ---- - -## Clean up resources - -[!INCLUDE [terraform-plan-destroy.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan-destroy.md)] - -## Troubleshoot Terraform on Azure - -[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) - -## Next steps - -In this quickstart, you deployed a simple virtual machine using Terraform. To learn more about Azure virtual machines, continue to the tutorial for Linux VMs. - -> [!div class="nextstepaction"] -> [Azure Linux virtual machine tutorials](./tutorial-manage-vm.md) \ No newline at end of file diff --git a/tools/execution_log.csv b/tools/execution_log.csv index c16086ab1..7de510738 100644 --- a/tools/execution_log.csv +++ b/tools/execution_log.csv @@ -1236,3 +1236,33 @@ StdErr: bash: line 2: Get-AzVm: command not found Error: command exited with 'exit status 127' and the message 'bash: line 2: pwsh: command not found ' StdErr: bash: line 2: pwsh: command not found",578.4860949516296,Success +2025-03-06 12:42:03,workload_description,create a linux vm and ssh into it using terraform,Deploy Linux VM and SSH using Terraform_ai_generated.md,2,"time=2025-03-06T12:36:09-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. +Error: Expected output does not match actual output. +Got: +"""" + +Expected: +x.x.x.x + +Expected Score:0.300000 +Actual Score:0.000000 +StdErr: + + time=2025-03-06T12:38:45-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 5. +Error: command exited with 'exit status 255' and the message 'Pseudo-terminal will not be allocated because stdin is not a terminal. +ssh: Could not resolve hostname x.x.x.x: Name or service not known +' +StdErr: Pseudo-terminal will not be allocated because stdin is not a terminal. +ssh: Could not resolve hostname x.x.x.x: Name or service not known",490.6871666908264,Success +2025-03-06 13:11:31,file,convert.md,convert_converted.md,0,,97.25097727775574,Success +2025-03-06 13:17:04,workload_description,create a linux vm and ssh into it,Deploy Linux VM with SSH Access_ai_generated.md,2,"time=2025-03-06T13:14:15-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. +Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. +' +StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. +See vm create -h for more information on specifying an image. + + time=2025-03-06T13:14:46-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. +Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_sx95BEHUMfLmMWthesw8MpVq7FOIx45d' is not valid according to the validation procedure. The tracking id is '44b14b15-e2ea-4ac8-b5db-a9415338882f'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} +' +StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_sx95BEHUMfLmMWthesw8MpVq7FOIx45d' is not valid according to the validation procedure. The tracking id is '44b14b15-e2ea-4ac8-b5db-a9415338882f'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}}",245.48310685157776,Success diff --git a/tools/mongodb.md b/tools/mongodb.md new file mode 100644 index 000000000..a61019c32 --- /dev/null +++ b/tools/mongodb.md @@ -0,0 +1,816 @@ +--- +title: 'Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS)' +description: In this article, you configure and deploy a MongoDB cluster on AKS. +ms.topic: how-to +ms.date: 01/07/2025 +author: fossygirl +ms.author: carols +ms.custom: aks-related-content +--- + +# Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS) + +In this article, you configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS). + +## Configure a workload identity + +1. Create a namespace for the MongoDB cluster using the `kubectl create namespace` command. + + ```bash + kubectl create namespace ${AKS_MONGODB_NAMESPACE} --dry-run=client --output yaml | kubectl apply -f - + ``` + + Example output: + + + ```output + namespace/mongodb created + ``` + +2. Create a service account and configure a workload identity using the `kubectl apply` command. + + ```bash + export TENANT_ID=$(az account show --query tenantId -o tsv) + cat < + ```output + serviceaccount/mongodb created + ``` + +## Install External Secrets Operator + +In this section, you use Helm to install External Secrets Operator. External Secrets Operator is a Kubernetes operator that manages the life cycle of external secrets stored in external secret stores like Azure Key Vault. + +1. Add the External Secrets Helm repository and update the repository using the `helm repo add` and `helm repo update` commands. + + ```bash + helm repo add external-secrets https://charts.external-secrets.io + helm repo update + ``` + + Example output: + + + ```output + Hang tight while we grab the latest from your chart repositories... + ...Successfully got an update from the "external-secrets" chart repository + ``` + +2. Install External Secrets Operator using the `helm install` command. + + ```bash + helm install external-secrets \ + external-secrets/external-secrets \ + --namespace ${AKS_MONGODB_NAMESPACE} \ + --create-namespace \ + --set installCRDs=true \ + --wait \ + --set nodeSelector."kubernetes\.azure\.com/agentpool"=userpool + ``` + + Example output: + + + ```output + NAME: external-secrets + LAST DEPLOYED: Tue Jun 11 11:55:32 2024 + NAMESPACE: mongodb + STATUS: deployed + REVISION: 1 + TEST SUITE: None + NOTES: + external-secrets has been deployed successfully in namespace mongodb! + + In order to begin using ExternalSecrets, you will need to set up a SecretStore + or ClusterSecretStore resource (for example, by creating a 'vault' SecretStore). + + More information on the different types of SecretStores and how to configure them + can be found in our Github: https://github.com/external-secrets/external-secrets + ``` + +3. Generate a random password for the MongoDB cluster and store it in Azure Key Vault using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + #MongoDB connection strings can contain special characters in the password, which need to be URL encoded. + #This is because the connection string is a URI, and special characters can interfere with the URI structure. + #This function generates secrets of 32 characters using only alphanumeric characters. + + generateRandomPasswordString() { + cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 + } + ``` + +## Create MongoDB secrets + +1. Create a MongoDB [backup user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#backup-and-restoration-roles) secret to use for any backup and restore operations using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-USER --value MONGODB_BACKUP_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +2. Create a MongoDB [database admin user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#all-database-roles) secret for database administration using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-USER --value MONGODB_DATABASE_ADMIN_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +3. Create a MongoDB [cluster administration user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterAdmin) secret for a cluster administration role that provides administration for more than one database using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-USER --value MONGODB_CLUSTER_ADMIN_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +4. Create a MongoDB [cluster monitoring user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterMonitor) secret for cluster monitoring using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-USER --value MONGODB_CLUSTER_MONITOR_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +5. Create a user and password secret for [user administration](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-userAdminAnyDatabase) using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-USER --value MONGODB_USER_ADMIN_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +6. Create a secret for the API key used to access the Percona Monitoring and Management (PMM) server using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. You update the value of this secret later when you deploy the PMM server. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name PMM-SERVER-API-KEY --value $(openssl rand -base64 32) --output table + ``` + +7. Add `AZURE-STORAGE-ACCOUNT-NAME` to use later for backups using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name AZURE-STORAGE-ACCOUNT-NAME --value $AKS_MONGODB_BACKUP_STORAGE_ACCOUNT_NAME --output table + ``` + +## Create secrets resources + +1. Create a `SecretStore` resource to access the MongoDB passwords stored in your key vault using the `kubectl apply` command. + + ```bash + kubectl apply -f - < + ```output + secretstore.external-secrets.io/azure-store created + ``` + +2. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace with the MongoDB secrets stored in your key vault. + + ```bash + kubectl apply -f - < + ```output + externalsecret.external-secrets.io/cluster-aks-mongodb-secrets created + ``` + +3. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace for Azure Blob Storage secrets stored in your key vault. + + ```bash + kubectl apply -f - < + ```output + externalsecret.external-secrets.io/cluster-aks-azure-secrets created + ``` + +4. Create a federated credential using the [`az identity federated-credential create`](/cli/azure/identity/federated-credential#az-identity-federated-credential-create) command. + + ```azurecli-interactive + az identity federated-credential create \ + --name external-secret-operator \ + --identity-name ${MY_IDENTITY_NAME} \ + --resource-group ${MY_RESOURCE_GROUP_NAME} \ + --issuer ${OIDC_URL} \ + --subject system:serviceaccount:${AKS_MONGODB_NAMESPACE}:${SERVICE_ACCOUNT_NAME} \ + --output table + ``` + + Example output: + + + ```output + Issuer Name ResourceGroup Subject + ----------------------------------------------------------------------------------------------------------------------- ------------------------ -------------------------------- ------------------------------------- + https://australiaeast.oic.prod-aks.azure.com/aaaa0a0a-bb1b-cc2c-dd3d-eeeeee4e4e4e/a0a0a0a0-bbbb-cccc-dddd-e1e1e1e1e1e1/ external-secret-operator myResourceGroup-rg-australiaeast system:serviceaccount:mongodb:mongodb + ``` + +5. Give permission to the user-assigned identity to access the secret using the [`az keyvault set-policy`](/cli/azure/keyvault#az-keyvault-set-policy) command. + + ```azurecli-interactive + az keyvault set-policy --name $MY_KEYVAULT_NAME --object-id $MY_IDENTITY_NAME_PRINCIPAL_ID --secret-permissions get --output table + ``` + + Example output: + + + ```output + Location Name ResourceGroup + ------------- -------------- -------------------------------- + australiaeast vault-cjcfc-kv myResourceGroup-rg-australiaeast + ``` + +## Install the Percona Operator and CRDs + +The Percona Operator is typically distributed as a Kubernetes `Deployment` or `Operator`. You can deploy it by using a `kubectl apply -f` command with a manifest file. You can find the latest manifests in the [Percona GitHub repository](https://github.com/percona/percona-server-mongodb-operator) or the [official documentation](https://docs.percona.com/percona-operator-for-mongodb/aks.html). + +* Deploy the Percona Operator and custom resource definitions (CRDs) using the `kubectl apply` command. + + ```bash + kubectl apply --server-side -f https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/v1.16.0/deploy/bundle.yaml -n "${AKS_MONGODB_NAMESPACE}" + ``` + + Example output: + + + ```output + customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied + customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied + customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + role.rbac.authorization.k8s.io/percona-server-mongodb-operator serverside-applied + serviceaccount/percona-server-mongodb-operator serverside-applied + rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator serverside-applied + deployment.apps/percona-server-mongodb-operator serverside-applied + ``` + +## Deploy the MongoDB cluster + +1. Deploy a MongoDB cluster with the Percona Operator using the `kubectl apply` command. To help ensure high availability, you deploy the MongoDB cluster with a replica set, with sharding enabled, in multiple availability zones, and with a backup solution that stores the backups in an Azure Blob Storage account. + + ```bash + kubectl apply -f - < + ```output + perconaservermongodb.psmdb.percona.com/cluster-aks-mongodb created + ``` + +2. Finish the MongoDB cluster deployment process using the following script: + + ```bash + while [ "$(kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for MongoDB cluster to be ready"; sleep 10; done + ``` + +3. When the process finishes, your cluster shows the `Ready` status. You can view the status using the `kubectl get` command. + + ```bash + kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME ENDPOINT STATUS AGE + cluster-aks-mongodb cluster-aks-mongodb-mongos.mongodb.svc.cluster.local ready 3m1s + ``` + +4. View the availability zones of the nodes in your cluster using the `kubectl get` command. + + ```bash + kubectl get node -o custom-columns=Name:.metadata.name,Zone:".metadata.labels.topology\.kubernetes\.io/zone" + ``` + + Example output: + + + ```output + Name Zone + aks-systempool-30094695-vmss000000 australiaeast-1 + aks-nodepool1-28994785-vmss000000 australiaeast-1 + aks-nodepool1-28994785-vmss000001 australiaeast-2 + aks-nodepool1-28994785-vmss000002 australiaeast-3 + ``` + +## Connect to the Percona Server + +To connect to Percona Server for MongoDB, you need to construct the MongoDB connection URI string. It includes the credentials of the admin user, which are stored in the `Secrets` object. + +1. List the `Secrets` objects using the `kubectl get` command. + + ```bash + kubectl get secrets -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME TYPE DATA AGE + cluster-aks-azure-secrets Opaque 2 2m56s + cluster-aks-mongodb-mongodb-keyfile Opaque 1 2m54s + cluster-aks-mongodb-secrets Opaque 11 2m56s + cluster-aks-mongodb-secrets-mongodb-encryption-key Opaque 1 2m54s + cluster-aks-mongodb-ssl kubernetes.io/tls 3 2m55s + cluster-aks-mongodb-ssl-internal kubernetes.io/tls 3 2m54s + external-secrets-webhook Opaque 4 3m49s + internal-cluster-aks-mongodb-users Opaque 11 2m56s + sh.helm.release.v1.external-secrets.v1 helm.sh/release.v1 1 3m49s + ``` + +2. View the `Secrets` contents to retrieve the admin user credentials using the `kubectl get` command. + + ```bash + kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -o yaml -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + apiVersion: v1 + data: + MONGODB_BACKUP_PASSWORD: aB1cD2eF-3gH... + MONGODB_BACKUP_USER: cD2eF3gH4iJ... + MONGODB_CLUSTER_ADMIN_PASSWORD: eF3gH4iJ5kL6mN7oP... + MONGODB_CLUSTER_ADMIN_USER: gH4iJ5kL6mN7oP8... + MONGODB_CLUSTER_MONITOR_PASSWORD: iJ5kL6mN7oP8qR9sT0-u... + MONGODB_CLUSTER_MONITOR_USER: kL6mN7oP8qR9sT0... + MONGODB_DATABASE_ADMIN_PASSWORD: mN7oP8qR9sT0uV1... + MONGODB_DATABASE_ADMIN_USER: A1bC2dE3fH4iJ5kL... + MONGODB_USER_ADMIN_PASSWORD: C2dE3fH4iJ5kL6mN7oP... + MONGODB_USER_ADMIN_USER: E3fH4iJ5kL6mN7... + immutable: false + kind: Secret + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"external-secrets.io/v1beta1","kind":"ExternalSecret","metadata":{"annotations":{},"name":"cluster-aks-mongodb-secrets","namespace":"mongodb"},"spec":{"data":[{"remoteRef":{"key":"MONGODB-BACKUP-USER"},"secretKey":"MONGODB_BACKUP_USER"},{"remoteRef":{"key":"MONGODB-BACKUP-PASSWORD"},"secretKey":"MONGODB_BACKUP_PASSWORD"},{"remoteRef":{"key":"MONGODB-DATABASE-ADMIN-USER"},"secretKey":"MONGODB_DATABASE_ADMIN_USER"},{"remoteRef":{"key":"MONGODB-DATABASE-ADMIN-PASSWORD"},"secretKey":"MONGODB_DATABASE_ADMIN_PASSWORD"},{"remoteRef":{"key":"MONGODB-CLUSTER-ADMIN-USER"},"secretKey":"MONGODB_CLUSTER_ADMIN_USER"},{"remoteRef":{"key":"MONGODB-CLUSTER-ADMIN-PASSWORD"},"secretKey":"MONGODB_CLUSTER_ADMIN_PASSWORD"},{"remoteRef":{"key":"MONGODB-CLUSTER-MONITOR-USER"},"secretKey":"MONGODB_CLUSTER_MONITOR_USER"},{"remoteRef":{"key":"MONGODB-CLUSTER-MONITOR-PASSWORD"},"secretKey":"MONGODB_CLUSTER_MONITOR_PASSWORD"},{"remoteRef":{"key":"MONGODB-USER-ADMIN-USER"},"secretKey":"MONGODB_USER_ADMIN_USER"},{"remoteRef":{"key":"MONGODB-USER-ADMIN-PASSWORD"},"secretKey":"MONGODB_USER_ADMIN_PASSWORD"}],"refreshInterval":"1h","secretStoreRef":{"kind":"SecretStore","name":"azure-store"},"target":{"creationPolicy":"Owner","name":"cluster-aks-mongodb-secrets"}}} + reconcile.external-secrets.io/data-hash: aB1cD2eF-3gH4iJ5kL6-mN7oP8qR= + creationTimestamp: "2024-07-01T12:24:38Z" + labels: + reconcile.external-secrets.io/created-by: N7oP8qR9sT0uV1wX2yZ3aB4cD5eF6g + name: cluster-aks-mongodb-secrets + namespace: mongodb + ownerReferences: + - apiVersion: external-secrets.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: ExternalSecret + name: cluster-aks-mongodb-secrets + uid: aaaaaaaa-0b0b-1c1c-2d2d-333333333333 + resourceVersion: "1872" + uid: bbbbbbbb-1c1c-2d2d-3e3e-444444444444 + type: Opaque + ``` + +3. Decode the Base64-encoded login name and password from the output using the following commands: + + ```bash + #Decode login name and password on the output, which are Base64-encoded + export databaseAdmin=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_USER}" | base64 --decode) + export databaseAdminPassword=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_PASSWORD}" | base64 --decode) + + echo $databaseAdmin + echo $databaseAdminPassword + echo $AKS_MONGODB_CLUSTER_NAME + ``` + + Example output: + + + ```output + MONGODB_DATABASE_ADMIN_USER + gH4iJ5kL6mN7oP8... + cluster-aks-mongodb + ``` + +## Verify the MongoDB cluster + +In this section, you verify your MongoDB cluster by running a container with a MongoDB client and connect its console output to your terminal. + +1. Create a pod named `percona-client` under the `${AKS_MONGODB_NAMESPACE}` namespace in your cluster using the `kubectl run` command. + + ```bash + kubectl -n "${AKS_MONGODB_NAMESPACE}" run -i --rm --tty percona-client --image=${MY_ACR_REGISTRY}.azurecr.io/percona-server-mongodb:7.0.8-5 --restart=Never -- bash -il + ``` + +2. In a different terminal window, verify the pod was successfully created using the `kubectl get` command. + + ```bash + kubectl get pod percona-client -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME READY STATUS RESTARTS AGE + percona-client 1/1 Running 0 39s + ``` + +3. Connect to the MongoDB cluster using the admin user credentials from the previous section in the terminal window that you used to create the `percona-client` pod. + + ```bash + # Note: Replace variables `databaseAdmin` , `databaseAdminPassword` and `AKS_MONGODB_CLUSTER_NAME` with actual values printed in step 3. + + mongosh "mongodb://${databaseAdmin}:${databaseAdminPassword}@${AKS_MONGODB_CLUSTER_NAME}-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=false&directConnection=true" + ``` + + Example output: + + + ```output + Current Mongosh Log ID: L6mN7oP8qR9sT0uV1wX2yZ3a + Connecting to: mongodb://@cluster-aks-mongodb-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=false&directConnection=true&appName=mongosh+2.1.5 + Using MongoDB: 7.0.8-5 + Using Mongosh: 2.1.5 + + For mongosh info see: https://docs.mongodb.com/mongodb-shell/ + ... + ``` + +4. List the databases in your cluster using the `show dbs` command. + + ```bash + show dbs + ``` + + Example output: + + + ```output + rs0 [direct: mongos] admin> show dbs + admin 960.00 KiB + config 3.45 MiB + rs0 [direct: mongos] admin> + ``` + +## Create a MongoDB backup + +You can back up your data to Azure using one of the following methods: + +* **Manual**: Manually back up your data at any time. +* **Scheduled**: Configure backups and their schedules in the CRD YAML. The Percona Operator makes the backups automatically according to the specified schedule. + +The Percona Operator can perform either of the following backup types: + +* **Logical backup**: Query Percona Server for MongoDB for the database data, and then write the retrieved data to the remote backup storage. +* **Physical backup**: Copy physical files from the Percona Server for MongoDB `dbPath` data directory to the remote backup storage. + +Logical backups use less storage but are slower than physical backups. + +To store backups on Azure Blob Storage using Percona, you need to create a secret. You completed this step in an earlier command. For detailed instructions, follow the steps in the [Percona documentation about Azure Blob Storage](https://docs.percona.com/percona-operator-for-mongodb/backups-storage.html#microsoft-azure-blob-storage). + +### Configure scheduled backups + +You can define the backup schedule in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: + +* Set the `backup.enabled` key to `true`. +* Ensure that the `backup.storages` subsection contains at least one configured storage resource. +* Ensure that the `backup.tasks` subsection enables backup scheduling. + +For more information, see [Making scheduled backups](https://docs.percona.com/percona-operator-for-mongodb/backups-scheduled.html). + +### Perform manual backups + +You can make a manual, on-demand backup in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: + +* Set the `backup.enabled` key to `true`. +* Ensure that the `backup.storages` subsection contains at least one configured storage resource. + +For more information, see [Making on-demand backups](https://docs.percona.com/percona-operator-for-mongodb/backups-ondemand.html). + +## Deploy a MongoDB backup + +1. Deploy your MongoDB backup using the `kubectl apply` command. + + ```bash + kubectl apply -f - < + ```output + perconaservermongodbbackup.psmdb.percona.com/az-backup1 created + ``` + +2. Finish the MongoDB backup deployment process using the following script: + + ```bash + while [ "$(kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for the backup to be ready"; sleep 10; done + ``` + + Example output: + + + ```output + waiting for the backup to be ready + ``` + +3. When the process finishes, the backup should return the `Ready` status. Verify the backup deployment was successful using the `kubectl get` command. + + ```bash + kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE + az-backup1 cluster-aks-mongodb azure-blob https://mongodbsacjcfc.blob.core.windows.net/backups/psmdb/2024-07-01T12:27:57Z logical ready 3h3m 3h3m + ``` + +4. If you have any problems with the backup, you can view logs from the `backup-agent` container of the appropriate pod using the `kubectl logs` command. + + ```bash + kubectl logs pod/${AKS_MONGODB_CLUSTER_NAME}-rs0-0 -c backup-agent -n ${AKS_MONGODB_NAMESPACE} + ``` + +## Next step + +> [!div class="nextstepaction"] +> [Deploy a client application (Mongo Express)][validate-mongodb-cluster] + + +[validate-mongodb-cluster]: ./validate-mongodb-cluster.md \ No newline at end of file diff --git a/tools/mongodb_redacted.md b/tools/mongodb_redacted.md new file mode 100644 index 000000000..291975a60 --- /dev/null +++ b/tools/mongodb_redacted.md @@ -0,0 +1,815 @@ +--- +title: 'Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS)' +description: In this article, you configure and deploy a MongoDB cluster on AKS. +ms.topic: how-to +ms.date: 01/07/2025 +author: xxxxx +ms.author: xxxxx +ms.custom: aks-related-content +--- + +# Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS) + +In this article, you configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS). + +## Configure a workload identity + +1. Create a namespace for the MongoDB cluster using the `kubectl create namespace` command. + + ```bash + kubectl create namespace ${AKS_MONGODB_NAMESPACE} --dry-run=client --output yaml | kubectl apply -f - + ``` + + Example output: + + + ```output + namespace/xxxxx created + ``` + +2. Create a service account and configure a workload identity using the `kubectl apply` command. + + ```bash + export TENANT_ID=$(az account show --query tenantId -o tsv) + cat < + ```output + serviceaccount/xxxxx created + ``` + +## Install External Secrets Operator + +In this section, you use Helm to install External Secrets Operator. External Secrets Operator is a Kubernetes operator that manages the life cycle of external secrets stored in external secret stores like Azure Key Vault. + +1. Add the External Secrets Helm repository and update the repository using the `helm repo add` and `helm repo update` commands. + + ```bash + helm repo add external-secrets https://charts.external-secrets.io + helm repo update + ``` + + Example output: + + + ```output + Hang tight while we grab the latest from your chart repositories... + ...Successfully got an update from the "external-secrets" chart repository + ``` + +2. Install External Secrets Operator using the `helm install` command. + + ```bash + helm install external-secrets \ + external-secrets/external-secrets \ + --namespace ${AKS_MONGODB_NAMESPACE} \ + --create-namespace \ + --set installCRDs=true \ + --wait \ + --set nodeSelector."kubernetes\.azure\.com/agentpool"=userpool + ``` + + Example output: + + + ```output + NAME: external-secrets + LAST DEPLOYED: Tue Jun 11 11:55:32 2024 + NAMESPACE: xxxxx + STATUS: deployed + REVISION: 1 + TEST SUITE: None + NOTES: + external-secrets has been deployed successfully in namespace xxxxx! + + In order to begin using ExternalSecrets, you will need to set up a SecretStore + or ClusterSecretStore resource (for example, by creating a 'vault' SecretStore). + + More information on the different types of SecretStores and how to configure them + can be found in our Github: https://github.com/external-secrets/external-secrets + ``` + +3. Generate a random password for the MongoDB cluster and store it in Azure Key Vault using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + #MongoDB connection strings can contain special characters in the password, which need to be URL encoded. + #This is because the connection string is a URI, and special characters can interfere with the URI structure. + #This function generates secrets of 32 characters using only alphanumeric characters. + + generateRandomPasswordString() { + cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 + } + ``` + +## Create MongoDB secrets + +1. Create a MongoDB [backup user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#backup-and-restoration-roles) secret to use for any backup and restore operations using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-USER --value MONGODB_BACKUP_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +2. Create a MongoDB [database admin user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#all-database-roles) secret for database administration using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-USER --value MONGODB_DATABASE_ADMIN_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +3. Create a MongoDB [cluster administration user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterAdmin) secret for a cluster administration role that provides administration for more than one database using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-USER --value MONGODB_CLUSTER_ADMIN_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +4. Create a MongoDB [cluster monitoring user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterMonitor) secret for cluster monitoring using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-USER --value MONGODB_CLUSTER_MONITOR_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +5. Create a user and password secret for [user administration](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-userAdminAnyDatabase) using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-USER --value MONGODB_USER_ADMIN_USER --output table + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table + ``` + +6. Create a secret for the API key used to access the Percona Monitoring and Management (PMM) server using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. You update the value of this secret later when you deploy the PMM server. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name PMM-SERVER-API-KEY --value $(openssl rand -base64 32) --output table + ``` + +7. Add `AZURE-STORAGE-ACCOUNT-NAME` to use later for backups using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. + + ```azurecli-interactive + az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name AZURE-STORAGE-ACCOUNT-NAME --value $AKS_MONGODB_BACKUP_STORAGE_ACCOUNT_NAME --output table + ``` + +## Create secrets resources + +1. Create a `SecretStore` resource to access the MongoDB passwords stored in your key vault using the `kubectl apply` command. + + ```bash + kubectl apply -f - < + ```output + secretstore.external-secrets.io/xxxxx created + ``` + +2. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace with the MongoDB secrets stored in your key vault. + + ```bash + kubectl apply -f - < + ```output + externalsecret.external-secrets.io/xxxxx created + ``` + +3. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace for Azure Blob Storage secrets stored in your key vault. + + ```bash + kubectl apply -f - < + ```output + externalsecret.external-secrets.io/xxxxx created + ``` + +4. Create a federated credential using the [`az identity federated-credential create`](/cli/azure/identity/federated-credential#az-identity-federated-credential-create) command. + + ```azurecli-interactive + az identity federated-credential create \ + --name external-secret-operator \ + --identity-name ${MY_IDENTITY_NAME} \ + --resource-group ${MY_RESOURCE_GROUP_NAME} \ + --issuer ${OIDC_URL} \ + --subject system:serviceaccount:${AKS_MONGODB_NAMESPACE}:${SERVICE_ACCOUNT_NAME} \ + --output table + ``` + + Example output: + + + ```output + Issuer Name ResourceGroup Subject + ----------------------------------------------------------------------------------------------------------------------- ------------------------ -------------------------------- ------------------------------------- + https://australiaeast.oic.prod-aks.azure.com/xxxxx/xxxxx/ xxxxx xxxxx system:serviceaccount:xxxxx:xxxxx + ``` + +5. Give permission to the user-assigned identity to access the secret using the [`az keyvault set-policy`](/cli/azure/keyvault#az-keyvault-set-policy) command. + + ```azurecli-interactive + az keyvault set-policy --name $MY_KEYVAULT_NAME --object-id $MY_IDENTITY_NAME_PRINCIPAL_ID --secret-permissions get --output table + ``` + + Example output: + + + ```output + Location Name ResourceGroup + ------------- -------------- -------------------------------- + australiaeast xxxxx xxxxx + ``` + +## Install the Percona Operator and CRDs + +The Percona Operator is typically distributed as a Kubernetes `Deployment` or `Operator`. You can deploy it by using a `kubectl apply -f` command with a manifest file. You can find the latest manifests in the [Percona GitHub repository](https://github.com/percona/percona-server-mongodb-operator) or the [official documentation](https://docs.percona.com/percona-operator-for-mongodb/aks.html). + +* Deploy the Percona Operator and custom resource definitions (CRDs) using the `kubectl apply` command. + + ```bash + kubectl apply --server-side -f https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/v1.16.0/deploy/bundle.yaml -n "${AKS_MONGODB_NAMESPACE}" + ``` + + Example output: + + + ```output + customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied + customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied + customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + role.rbac.authorization.k8s.io/percona-server-mongodb-operator serverside-applied + serviceaccount/percona-server-mongodb-operator serverside-applied + rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator serverside-applied + deployment.apps/percona-server-mongodb-operator serverside-applied + ``` + +## Deploy the MongoDB cluster + +1. Deploy a MongoDB cluster with the Percona Operator using the `kubectl apply` command. To help ensure high availability, you deploy the MongoDB cluster with a replica set, with sharding enabled, in multiple availability zones, and with a backup solution that stores the backups in an Azure Blob Storage account. + + ```bash + kubectl apply -f - < + ```output + perconaservermongodb.psmdb.percona.com/xxxxx created + ``` + +2. Finish the MongoDB cluster deployment process using the following script: + + ```bash + while [ "$(kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for MongoDB cluster to be ready"; sleep 10; done + ``` + +3. When the process finishes, your cluster shows the `Ready` status. You can view the status using the `kubectl get` command. + + ```bash + kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME ENDPOINT STATUS AGE + xxxxx xxxxx ready 3m1s + ``` + +4. View the availability zones of the nodes in your cluster using the `kubectl get` command. + + ```bash + kubectl get node -o custom-columns=Name:.metadata.name,Zone:".metadata.labels.topology\.kubernetes\.io/zone" + ``` + + Example output: + + + ```output + Name Zone + xxxxx australiaeast-1 + xxxxx australiaeast-1 + xxxxx australiaeast-2 + xxxxx australiaeast-3 + ``` + +## Connect to the Percona Server + +To connect to Percona Server for MongoDB, you need to construct the MongoDB connection URI string. It includes the credentials of the admin user, which are stored in the `Secrets` object. + +1. List the `Secrets` objects using the `kubectl get` command. + + ```bash + kubectl get secrets -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME TYPE DATA AGE + xxxxx Opaque 2 2m56s + xxxxx Opaque 1 2m54s + xxxxx Opaque 11 2m56s + xxxxx Opaque 1 2m54s + xxxxx kubernetes.io/tls 3 2m55s + xxxxx kubernetes.io/tls 3 2m54s + external-secrets-webhook Opaque 4 3m49s + xxxxx Opaque 11 2m56s + xxxxx helm.sh/release.v1 1 3m49s + ``` + +2. View the `Secrets` contents to retrieve the admin user credentials using the `kubectl get` command. + + ```bash + kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -o yaml -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + apiVersion: v1 + data: + MONGODB_BACKUP_PASSWORD: xxxxx + MONGODB_BACKUP_USER: xxxxx + MONGODB_CLUSTER_ADMIN_PASSWORD: xxxxx + MONGODB_CLUSTER_ADMIN_USER: xxxxx + MONGODB_CLUSTER_MONITOR_PASSWORD: xxxxx + MONGODB_CLUSTER_MONITOR_USER: xxxxx + MONGODB_DATABASE_ADMIN_PASSWORD: xxxxx + MONGODB_DATABASE_ADMIN_USER: xxxxx + MONGODB_USER_ADMIN_PASSWORD: xxxxx + MONGODB_USER_ADMIN_USER: xxxxx + immutable: false + kind: Secret + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: xxxxx + reconcile.external-secrets.io/data-hash: xxxxx + creationTimestamp: "xxxxx" + labels: + reconcile.external-secrets.io/created-by: xxxxx + name: xxxxx + namespace: mongodb + ownerReferences: + - apiVersion: external-secrets.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: ExternalSecret + name: xxxxx + uid: xxxxx + resourceVersion: "xxxxx" + uid: xxxxx + type: Opaque + ``` + +3. Decode the Base64-encoded login name and password from the output using the following commands: + + ```bash + #Decode login name and password on the output, which are Base64-encoded + export databaseAdmin=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_USER}" | base64 --decode) + export databaseAdminPassword=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_PASSWORD}" | base64 --decode) + + echo $databaseAdmin + echo $databaseAdminPassword + echo $AKS_MONGODB_CLUSTER_NAME + ``` + + Example output: + + + ```output + xxxxx + xxxxx + xxxxx + ``` + +## Verify the MongoDB cluster + +In this section, you verify your MongoDB cluster by running a container with a MongoDB client and connect its console output to your terminal. + +1. Create a pod named `percona-client` under the `${AKS_MONGODB_NAMESPACE}` namespace in your cluster using the `kubectl run` command. + + ```bash + kubectl -n "${AKS_MONGODB_NAMESPACE}" run -i --rm --tty percona-client --image=${MY_ACR_REGISTRY}.azurecr.io/percona-server-mongodb:7.0.8-5 --restart=Never -- bash -il + ``` + +2. In a different terminal window, verify the pod was successfully created using the `kubectl get` command. + + ```bash + kubectl get pod percona-client -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME READY STATUS RESTARTS AGE + xxxxx 1/1 Running 0 39s + ``` + +3. Connect to the MongoDB cluster using the admin user credentials from the previous section in the terminal window that you used to create the `percona-client` pod. + + ```bash + # Note: Replace variables `databaseAdmin` , `databaseAdminPassword` and `AKS_MONGODB_CLUSTER_NAME` with actual values printed in step 3. + + mongosh "mongodb://${databaseAdmin}:${databaseAdminPassword}@${AKS_MONGODB_CLUSTER_NAME}-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=false&directConnection=true" + ``` + + Example output: + + + ```output + Current Mongosh Log ID: xxxxx + Connecting to: mongodb://@xxxxx/admin?replicaSet=rs0&ssl=false&directConnection=true&appName=mongosh+2.1.5 + Using MongoDB: 7.0.8-5 + Using Mongosh: 2.1.5 + + For mongosh info see: https://docs.mongodb.com/mongodb-shell/ + ... + ``` + +4. List the databases in your cluster using the `show dbs` command. + + ```bash + show dbs + ``` + + Example output: + + + ```output + rs0 [direct: mongos] admin> show dbs + admin 960.00 KiB + config 3.45 MiB + rs0 [direct: mongos] admin> + ``` + +## Create a MongoDB backup + +You can back up your data to Azure using one of the following methods: + +* **Manual**: Manually back up your data at any time. +* **Scheduled**: Configure backups and their schedules in the CRD YAML. The Percona Operator makes the backups automatically according to the specified schedule. + +The Percona Operator can perform either of the following backup types: + +* **Logical backup**: Query Percona Server for MongoDB for the database data, and then write the retrieved data to the remote backup storage. +* **Physical backup**: Copy physical files from the Percona Server for MongoDB `dbPath` data directory to the remote backup storage. + +Logical backups use less storage but are slower than physical backups. + +To store backups on Azure Blob Storage using Percona, you need to create a secret. You completed this step in an earlier command. For detailed instructions, follow the steps in the [Percona documentation about Azure Blob Storage](https://docs.percona.com/percona-operator-for-mongodb/backups-storage.html#microsoft-azure-blob-storage). + +### Configure scheduled backups + +You can define the backup schedule in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: + +* Set the `backup.enabled` key to `true`. +* Ensure that the `backup.storages` subsection contains at least one configured storage resource. +* Ensure that the `backup.tasks` subsection enables backup scheduling. + +For more information, see [Making scheduled backups](https://docs.percona.com/percona-operator-for-mongodb/backups-scheduled.html). + +### Perform manual backups + +You can make a manual, on-demand backup in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: + +* Set the `backup.enabled` key to `true`. +* Ensure that the `backup.storages` subsection contains at least one configured storage resource. + +For more information, see [Making on-demand backups](https://docs.percona.com/percona-operator-for-mongodb/backups-ondemand.html). + +## Deploy a MongoDB backup + +1. Deploy your MongoDB backup using the `kubectl apply` command. + + ```bash + kubectl apply -f - < + ```output + perconaservermongodbbackup.psmdb.percona.com/xxxxx created + ``` + +2. Finish the MongoDB backup deployment process using the following script: + + ```bash + while [ "$(kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for the backup to be ready"; sleep 10; done + ``` + + Example output: + + + ```output + waiting for the backup to be ready + ``` + +3. When the process finishes, the backup should return the `Ready` status. Verify the backup deployment was successful using the `kubectl get` command. + + ```bash + kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} + ``` + + Example output: + + + ```output + NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE + xxxxx xxxxx xxxxx https://xxxxx.blob.core.windows.net/backups/psmdb/xxxxx logical ready 3h3m 3h3m + ``` + +4. If you have any problems with the backup, you can view logs from the `backup-agent` container of the appropriate pod using the `kubectl logs` command. + + ```bash + kubectl logs pod/${AKS_MONGODB_CLUSTER_NAME}-rs0-0 -c backup-agent -n ${AKS_MONGODB_NAMESPACE} + ``` + +## Next step + +> [!div class="nextstepaction"] +> [Deploy a client application (Mongo Express)][validate-mongodb-cluster] + + +[validate-mongodb-cluster]: ./validate-mongodb-cluster.md \ No newline at end of file diff --git a/tools/mongodb_security_report.md b/tools/mongodb_security_report.md new file mode 100644 index 000000000..937116796 --- /dev/null +++ b/tools/mongodb_security_report.md @@ -0,0 +1,90 @@ +Below is the complete security vulnerability analysis report for the provided Exec Doc. This analysis covers both static (code review) and dynamic (runtime environment) aspects using industry frameworks such as the OWASP Top 10 and cloud security best practices. + +------------------------------------------------------------ + +# Security Vulnerability Analysis Report + +## 1. Executive Summary + +This document outlines a comprehensive security review of the MongoDB cluster deployment instructions on Azure Kubernetes Service (AKS) using Percona Operator and External Secrets Operator. Overall, most risks are related to misconfigurations and reliance on external secret management. In particular, several areas require improvement regarding authentication and authorization settings, network security (e.g., non-enforced TLS), input validation, command injection risk in shell helpers, and secret management practices. While no immediate critical code-level injection was found, proper remediation and adherence to best practices are recommended to prevent potential privilege escalation, data leakage, and cloud exposure risks. + +## 2. Methodology + +The analysis was performed in two main phases: + +• Static Code Review: +– A manual review of the YAML manifests, shell scripts, Helm commands, and embedded Kubernetes objects. +– Assessment based on configuration best practices (namespace isolation, RBAC, workload identity annotations). +– Evaluation of inline scripts (e.g., password generation) for command injection and proper use of environment variable substitution. + +• Dynamic/Runtime Assessment: +– Consideration of how the deployment behaves (runtime secret handling, federated credential use, network communication). +– Review of cloud-specific operations such as creation of federated credentials, key vault secret policies, and external secret polling frequency. +– Evaluation of network configurations (unencrypted MongoDB connection string and cross-namespace secret accessibility). + +## 3. Findings + +The following table summarizes the identified vulnerabilities along with their severity, exact locations (where applicable), description, potential impact, and recommended fixes. + +| Severity | Location / Context | Description | Potential Impact | Recommended Fix / Code Example | +|----------|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Critical | MongoDB connection URI in client connection | The connection string uses “ssl=false”, disabling encrypted communication between clients and the MongoDB service. | Sensitive credentials and data transmissions are exposed to eavesdropping and man-in-the-middle attacks. | Enforce TLS/SSL by setting ssl=true and ensuring certificates are properly configured. Example:
``` +mongosh "mongodb://${databaseAdmin}:${databaseAdminPassword}@${AKS_MONGODB_CLUSTER_NAME}-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=true&directConnection=true" +``` | +| High | Workload Identity & Service Account Manifest | The ServiceAccount YAML includes annotations for workload identity (client-id, tenant-id) and creates federated credentials. If misconfigured (e.g., allowing overly broad access or not restricted to the intended namespace), it could allow unauthorized access or abuse of privileges in the cluster. | Potential privilege escalation and unauthorized access to resources in the AKS cluster and Azure Key Vault. | Limit the scope of the service account by using minimal RBAC privileges and enforce strict validation on annotations. Additionally, ensure the federated credential subject is tightly scoped. | +| High | Kubernetes RBAC and Secret Storage | Kubernetes Secrets are stored base64-encoded and referenced in multiple YAML files. Without proper encryption at rest (e.g., ETCD encryption) or strict RBAC restrictions, there is a risk that unauthorized users could read sensitive data. | Exposure of credentials (MongoDB admin, backup, cluster users) if an attacker gains read access to secrets. | Enable encryption at rest for Kubernetes secrets and restrict access via RBAC. Use tools such as Kubernetes Secret Encryption Providers and audit logs to monitor accesses. | +| Medium | Shell Function “generateRandomPasswordString” | The helper function uses /dev/urandom piped to tr and fold. Although the randomness is sufficient, interpolation of environment variables around this function (if uncontrolled) could allow local command injection in other contexts. | If an attacker controls input or environment variables, it could inject commands that compromise the system. | Validate or hard-code the allowed character set and ensure that environment variables used in the script (e.g., for names) are sanitized before use. | +| Medium | External Commands with Environment Variables | Many commands depend on environment variables (e.g., ${AKS_MONGODB_NAMESPACE}, ${MY_IDENTITY_NAME_CLIENT_ID}). Misconfiguration or injection in these variables (if not validated earlier) might lead to unintended command execution or resource exposure. | Unintended namespace creation, malicious resource targeting, or command injection if variables contain unsafe input.| Validate and sanitize environment variables prior to use. For example, using regex checks in your shell script before passing these values to kubectl or helm commands. | +| Medium | Federated Credential Creation (az identity) | The federation subject is constructed with a variable reference to the namespace and service account. If manipulated, attackers might elevate privileges by targeting the wrong subject, especially if OIDC endpoints are misconfigured. | Privilege escalation leading to unauthorized access to Azure resources. | Double-check the correctness of the issuer URL and subject field. Use strict identity policies and consider auditing the federated credential creation process for unusual modifications. | +| Low | Logging and Secret Disclosure in Shell Scripts | The documentation shows echoing of environment variables such as $databaseAdmin and $databaseAdminPassword directly on the console output. | Risk of leaking sensitive information to local logs or process history, especially in shared environments. | Remove unnecessary echo commands that print secret values. Use secure logging that obfuscates sensitive data. | +| Low | Backup and Cloud Storage Secrets | While backup operations and storage account access are configured via secrets, the lifecycle of these secrets is not discussed and could lead to outdated or leaked credentials if not rotated properly. | Persistent storage credentials might be exploited if not rotated; manual intervention needed for secret rotations. | Implement automated secret rotation and periodic audits of backup and storage credentials. Ensure that backups themselves are encrypted and access is strictly limited. | +| Low | Certificate and TLS Usage in Internal Communications | The YAML mostly does not enforce TLS for internal connections between pods (example: “ssl=false” in the MongoDB connection URI) and does not detail the use of mutual TLS between components such as the External Secrets Operator and Key Vault. | Risk of interception in a compromised cluster network or lateral movement if an attacker gains in-cluster access. | Enforce TLS between all cluster components (both intra-cluster and external communications). Configure mutual TLS (mTLS) for sensitive operations between operators and API servers where possible. | + +## 4. Recommendations + +Based on the findings above, the following steps are recommended: + +1. Secure Communication: + • Update the MongoDB connection string to enforce TLS (ssl=true). + • Configure certificates and enable mutual TLS for intra-cluster communications. + +2. Harden Identity and Access Management: + • Restrict ServiceAccount scopes using strict RBAC policies. + • Validate and lock down annotations used for workload identities. + • Review and minimize federated credential subject claims ensuring they match the intended namespace/service account. + +3. Protect Kubernetes Secrets: + • Enable encryption at rest for Kubernetes secrets. + • Tighten RBAC to limit secret read/write permissions only to required pods/users. + • Audit etcd and secret access logs for anomalous behavior. + +4. Sanitize Environment Variables and Shell Scripts: + • Validate all environment variables (namespaces, registry names, etc.) before use in commands. + • Refactor shell helpers to ensure they are protected against command injection by avoiding unsanitized interpolation. + • Remove or mask secret outputs in logs/echo commands. + +5. Improve Secret Management and Rotation: + • Ensure Azure Key Vault access policies are tightly controlled and secrets are rotated periodically. + • Monitor the use of External Secrets Operator and the secret sync frequency, ensuring timely updates and minimizing exposure if a secret is compromised. + +6. Monitor and Audit Cloud Configurations: + • Regularly audit federated credentials, backup policies, and Key Vault permissions. + • Enable logging and alerting on unusual configuration changes in the cloud environment. + +## 5. Best Practices + +To further improve the security posture of the deployment, consider the following general security best practices: + +• Adopt the Principle of Least Privilege (PoLP) for all identities and resources. +• Enable network segmentation and enforce security policies between namespaces. +• Implement regular vulnerability scans and penetration testing on both the Kubernetes infrastructure and deployed applications. +• Use automation for secret rotations and configuration audits. +• Integrate continuous monitoring and logging solutions (e.g., cloud-native SIEM) to detect abnormal behaviors quickly. +• Stay up-to-date with security patches for all deployed software components (Kubernetes, Operators, Helm charts). +• Educate users and administrators on secure configuration and incident response procedures. + +------------------------------------------------------------ + +By addressing the above recommendations and following best practices, the overall security posture of the MongoDB on AKS deployment can be significantly hardened against common vulnerabilities and misconfiguration risks. + +This concludes the security vulnerability analysis report. \ No newline at end of file From e10542b7a6ee3fec35b81c76c2b8c9c05930c660 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 6 Mar 2025 16:47:58 -0800 Subject: [PATCH 15/87] updated final tool --- scenarios/metadata.json | 2 +- ... on AKS with CloudNativePG_ai_generated.md | 156 ++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 5b3920199..fcce61321 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1136,7 +1136,7 @@ "status": "active", "key": "azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md", "title": "Quickstart: Use Terraform to create a Linux VM", - "description": "In this quickstart, you learn how to use Terraform to create a Linux virtual machine", + "description": "In this quickstart, you learn how to use Terraform to create a Linux virtual machine.", "stackDetails": [ ], "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md", diff --git a/tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md b/tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md new file mode 100644 index 000000000..0c15a1931 --- /dev/null +++ b/tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md @@ -0,0 +1,156 @@ +--- +title: Deploy a Highly Available PostgreSQL Database on AKS using CloudNativePG Operator +description: This Exec Doc demonstrates how to deploy a highly available PostgreSQL database on an Azure Kubernetes Service (AKS) cluster using the CloudNativePG operator. It covers creating the necessary Azure resources, installing the operator via Helm, and deploying a multi-instance PostgreSQL cluster. +ms.topic: quickstart +ms.date: 10/12/2023 +author: yourgithubusername +ms.author: youralias +ms.custom: innovation-engine, akshighavailability, cloudnativepg +--- + +# Deploy a Highly Available PostgreSQL Database on AKS using CloudNativePG Operator + +This document guides you through deploying a highly available PostgreSQL database on an AKS cluster using the CloudNativePG operator. You will create an Azure resource group and an AKS cluster with a random suffix for uniqueness, install the CloudNativePG operator using Helm, and then deploy a PostgreSQL cluster configured for high availability. + +The following steps include environment variable declarations, Azure CLI commands, and Kubernetes commands executed via bash code blocks. Each code block includes an accompanying result block to verify that the commands execute with the expected output. + +--- + +## Step 1: Create an Azure Resource Group + +In this section, we declare environment variables for the deployment. The resource group name will have a random suffix appended to ensure uniqueness. We then create the resource group in the designated region (WestUS2). + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="WestUS2" +export RESOURCE_GROUP="cnpg-rg$RANDOM_SUFFIX" +az group create --name $RESOURCE_GROUP --location $REGION +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/cnpg-rgxxxxxxxxx", + "location": "WestUS2", + "name": "cnpg-rgxxxxxxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": {} +} +``` + +--- + +## Step 2: Create an AKS Cluster + +Now we create an AKS cluster in the resource group. The cluster name is also appended with a random suffix. This cluster will have 3 nodes to support deployment of a highly available PostgreSQL database. + +```bash +export AKS_CLUSTER="cnpg-aks$RANDOM_SUFFIX" +az aks create --resource-group $RESOURCE_GROUP --name $AKS_CLUSTER --node-count 3 --enable-addons monitoring --generate-ssh-keys --location $REGION +``` + +Results: + + + +```JSON +{ + "fqdn": "cnpg-aksxxxxxxxxx.hcp.westus2.azmk8s.io", + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/cnpg-rgxxxxxxxxx/providers/Microsoft.ContainerService/managedClusters/cnpg-aksxxxxxxxxx", + "location": "WestUS2", + "name": "cnpg-aksxxxxxxxxx", + "provisioningState": "Succeeded", + "tags": {} +} +``` + +After creating the cluster, download its credentials so that kubectl can interact with it: + +```bash +az aks get-credentials --resource-group $RESOURCE_GROUP --name $AKS_CLUSTER +``` + +Results: + + + +```console +Merged "cnpg-aksxxxxxxxxx" as current context in /home/xxxxx/.kube/config +``` + +--- + +## Step 3: Install the CloudNativePG Operator + +The CloudNativePG operator is installed via Helm. This section adds the CloudNativePG Helm repository and deploys the operator into its own namespace (cnpg-system). + +```bash +helm repo add cloudnative-pg https://cloudnative-pg.io/charts +helm repo update +helm install cnpg cloudnative-pg/cnpg --namespace cnpg-system --create-namespace +``` + +Results: + + + +```console +NAME: cnpg +LAST DEPLOYED: Wed Oct 11 2023 12:34:56 PM +NAMESPACE: cnpg-system +STATUS: deployed +REVISION: 1 +``` + +--- + +## Step 4: Deploy a Highly Available PostgreSQL Cluster + +In this step, you'll deploy a PostgreSQL cluster using CloudNativePG. The configuration specifies three instances to achieve high availability, and a minimal storage allocation is used for demonstration purposes. + +First, create the PostgreSQL cluster manifest file named "ha-postgresql.yaml". This file should reside in the same folder as this Exec Doc. + +```bash +cat << 'EOF' > ha-postgresql.yaml +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ha-postgres +spec: + instances: 3 + storage: + size: 1Gi + postgresVersion: 14 +EOF +``` + +Results: + + + +```console +ha-postgresql.yaml created +``` + +Now, apply the YAML file to deploy the PostgreSQL cluster. + +```bash +kubectl apply -f ha-postgresql.yaml +``` + +Results: + + + +```console +cluster.postgresql.cnpg.io/ha-postgres created +``` + +--- + +In this Exec Doc, you've created an Azure resource group and an AKS cluster, installed the CloudNativePG operator using Helm, and deployed a highly available PostgreSQL database on the cluster using a custom YAML manifest. This automated, one-click deployment is repeatable and ensures that the resources are unique for every run. \ No newline at end of file From 7dfe87b13867362b376b00b9c04b3355447a6129 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 00:42:45 -0700 Subject: [PATCH 16/87] updated documentation with technical constraints --- README.md | 95 +++- tools/convert.md | 486 ++++++++++++++---- tools/convert_converted.md | 441 +++++++++++----- tools/execution_log.csv | 53 ++ tools/myazure_rm.yml | 4 + tools/myazure_rm.yml (initial version) | 4 + .../myazure_rm.yml (with conditional_groups) | 7 + tools/myazure_rm.yml (with keyed_groups) | 9 + tools/ping.yml | 9 + tools/win_ping.yml | 11 + 10 files changed, 884 insertions(+), 235 deletions(-) create mode 100644 tools/myazure_rm.yml create mode 100644 tools/myazure_rm.yml (initial version) create mode 100644 tools/myazure_rm.yml (with conditional_groups) create mode 100644 tools/myazure_rm.yml (with keyed_groups) create mode 100644 tools/ping.yml create mode 100644 tools/win_ping.yml diff --git a/README.md b/README.md index 303e9df81..6e2dc6a0b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ These experiences utilize [Innovation Engine](https://github.com/Azure/Innovatio ## Table of Contents +- [Selecting Documentation for Exec Docs](#selecting-documentation-for-exec-docs) - [How to Write an Exec Doc](#how-to-write-an-exec-doc) - [Training Resources (Optional)](#training-resources-optional) - [Setup](#setup) @@ -18,6 +19,79 @@ These experiences utilize [Innovation Engine](https://github.com/Azure/Innovatio - [Frequently Asked Questions (FAQs)](#frequently-asked-questions-faqs) - [Contact Information for Exec Docs](#contact-information-for-exec-docs) +## Selecting Documentation for Exec Docs + +Not all documentation is suitable for conversion to Exec Docs. Use these filters to determine if a document can be effectively converted: + +### Technical Constraints + +1. **Supported Code Block Types** + - The document must contain code blocks using at least one of these types: + - `bash` + - `azurecli` + - `azure-cli-interactive` + - `azurecli-interactive` + + **Example:** + ```markdown + ```bash + az group create --name myResourceGroup --location eastus + ``` + ``` + + >**Note:** This rule does not apply to output code blocks, which are used to display the results of commands, scripts, or other operations. These blocks help in illustrating what the expected output should look like. They include, but are not limited to, the following types: _output, json, yaml, console, text, and log._ + + >**Note:** While Innovation Engine can _parse_ a code block of any type, given its current features, it can only _execute_ code blocks of the types above. So, it is important to ensure that the code blocks in your Exec Doc are of the types above. + +2. **Command Execution Limitations** + - **Not supported for direct execution (unless executed via bash):** + - PowerShell scripts + - Python, Ruby, or Node.js code + - SQL commands + - GUI-based instructions + + - **Supported execution context:** + - Commands that run in a Linux/bash environment + - Azure CLI commands + - Terraform commands (with appropriate setup) + + **Example of supported command:** + ```markdown + ```bash + export VM_NAME="myVM" + az vm create --name $VM_NAME --resource-group myResourceGroup --image UbuntuLTS + ``` + ``` + +3. **Azure Portal Custom Cloud Shell Constraints** + - **Supported scenarios:** + - Standard Azure resource operations (create, read, update, delete) + - Commands running within the user's subscription scope + - Standard service deployments (VMs, storage, networking) + + - **Not supported without special configuration:** + - Commands requiring elevated Microsoft Graph API permissions + - Operations needing KeyVault special access + - Cross-subscription or tenant-level operations + - Commands requiring admin consent + + **Example of supported command:** + ```markdown + ```bash + az group create --name myResourceGroup --location eastus + ``` + ``` + + **Example of potentially unsupported command:** + ```markdown + ```bash + # This requires elevated Graph API permissions and would fail + az ad app create --display-name myApp --native-app + ``` + ``` + +This filter system ensures that you select documentation that can be effectively transformed into executable docs that provide value through automated deployment capabilities. + ## How to Write an Exec Doc Follow these steps in sequence to write an Exec Doc either by converting an existing Azure Doc i.e. building on top of the author's work or from scratch i.e. you are the author _(read the Notes in any step for more information)_: @@ -81,26 +155,7 @@ Check if all prerequisites below are met before writing the Exec Doc. ***If any │ └── my-script.yaml ``` -6. Code blocks are used to provide examples, commands, or other code snippets in Exec Docs. They are distinguished by a triple backtick (```) at the start and end of the block. - - Ensure that the Exec Doc contains at least 1 code block and every input code block's type in the Exec Doc is taken from this list: - - - bash - - azurecli - - azure-cli-interactive - - azurecli-interactive - - **Example:** - - ```bash - az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION - ``` - - >**Note:** This rule does not apply to output code blocks, which are used to display the results of commands, scripts, or other operations. These blocks help in illustrating what the expected output should look like. They include, but are not limited to, the following types: _output, json, yaml, console, text, and log._ - - >**Note:** While Innovation Engine can _parse_ a code block of any type, given its current features, it can only _execute_ code blocks of the types above. So, it is important to ensure that the code blocks in your Exec Doc are of the types above. - -7. Headings are used to organize content in a document. The number of hashes indicates the level of the heading. For example, a single hash (#) denotes an h1 heading, two hashes (##) denote an h2 heading, and so on. Innovation Engine uses headings to structure the content of an Exec Doc and to provide a clear outline of the document's contents. +6. Headings are used to organize content in a document. The number of hashes indicates the level of the heading. For example, a single hash (#) denotes an h1 heading, two hashes (##) denote an h2 heading, and so on. Innovation Engine uses headings to structure the content of an Exec Doc and to provide a clear outline of the document's contents. Ensure there is at least one h1 heading in the Exec Doc, denoted by a single hash (#) at the start of the line. diff --git a/tools/convert.md b/tools/convert.md index e050b2055..2b3bb6b17 100644 --- a/tools/convert.md +++ b/tools/convert.md @@ -1,141 +1,445 @@ --- -title: 'How-to: Create and deploy an Azure OpenAI Service resource' -titleSuffix: Azure OpenAI -description: Learn how to get started with Azure OpenAI Service and create your first resource and deploy your first model in the Azure CLI or the Azure portal. -#services: cognitive-services -manager: nitinme -ms.service: azure-ai-openai -ms.custom: devx-track-azurecli, build-2023, build-2023-dataai, devx-track-azurepowershell -ms.topic: how-to -ms.date: 01/31/2025 -zone_pivot_groups: openai-create-resource -author: mrbullwinkle -ms.author: mbullwin -recommendations: false +title: Tutorial - Configure dynamic inventories for Azure Virtual Machines using Ansible +description: Learn how to populate your Ansible inventory dynamically from information in Azure +keywords: ansible, azure, devops, bash, cloudshell, dynamic inventory +ms.topic: tutorial +ms.date: 08/14/2024 +ms.custom: devx-track-ansible, devx-track-azurecli, devx-track-azurepowershell, linux-related-content --- -# Create and deploy an Azure OpenAI Service resource +# Tutorial: Configure dynamic inventories of your Azure resources using Ansible -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://go.microsoft.com/fwlink/?linkid=2303211) +[!INCLUDE [ansible-28-note.md](includes/ansible-28-note.md)] -This article describes how to get started with Azure OpenAI Service and provides step-by-step instructions to create a resource and deploy a model. You can create resources in Azure in several different ways: +The [Ansible dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_dynamic_inventory.html) feature removes the burden of maintaining static inventory files. -- The [Azure portal](https://portal.azure.com/?microsoft_azure_marketplace_ItemHideKey=microsoft_openai_tip#create/Microsoft.CognitiveServicesOpenAI) -- The REST APIs, the Azure CLI, PowerShell, or client libraries -- Azure Resource Manager (ARM) templates +In this tutorial, you use Azure's dynamic-inventory plug-in to populate your Ansible inventory. -In this article, you review examples for creating and deploying resources in the Azure portal and with the Azure CLI. +In this article, you learn how to: + +> [!div class="checklist"] +> * Configure two test virtual machines. +> * Add tags to Azure virtual machines +> * Generate a dynamic inventory +> * Use conditional and keyed groups to populate group memberships +> * Run playbooks against groups within the dynamic inventory ## Prerequisites -- An Azure subscription. Create one for free. -- Access permissions to [create Azure OpenAI resources and to deploy models](../how-to/role-based-access-control.md). -- The Azure CLI. For more information, see [How to install the Azure CLI](/cli/azure/install-azure-cli). +[!INCLUDE [open-source-devops-prereqs-azure-subscription.md](../includes/open-source-devops-prereqs-azure-subscription.md)] +[!INCLUDE [open-source-devops-prereqs-create-service-principal.md](../includes/open-source-devops-prereqs-create-service-principal.md)] +[!INCLUDE [ansible-prereqs-cloudshell-use-or-vm-creation2.md](includes/ansible-prereqs-cloudshell-use-or-vm-creation2.md)] + +## Create Azure VMs + +1. Sign in to the [Azure portal](https://go.microsoft.com/fwlink/p/?LinkID=525040). + +1. Open [Cloud Shell](/azure/cloud-shell/overview). + +1. Create an Azure resource group to hold the virtual machines for this tutorial. + + > [!IMPORTANT] + > The Azure resource group you create in this step must have a name that is entirely lower-case. Otherwise, the generation of the dynamic inventory will fail. + + # [Azure CLI](#tab/azure-cli) + ```azurecli-interactive + az group create --resource-group ansible-inventory-test-rg --location eastus + ``` + # [Azure PowerShell](#tab/azure-powershell) + + ```azurepowershell + New-AzResourceGroup -Name ansible-inventory-test-rg -Location eastus + ``` + --- + +1. Create two Linux virtual machines on Azure using one of the following techniques: + + - **Ansible playbook** - The article, [Create a basic Linux virtual machine in Azure with Ansible](./vm-configure.md) and [Create a basic Windows virtual machine in Azure with Ansible](./vm-configure-windows.md) illustrates how to create a virtual machine from an Ansible playbook. + + - **Azure CLI** - Issue each of the following commands in the Cloud Shell to create the two virtual machines: -## Create an Azure resource group + # [Azure CLI](#tab/azure-cli) + ```azurecli-interactive + az vm create \ + --resource-group ansible-inventory-test-rg \ + --name win-vm \ + --image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest \ + --admin-username azureuser \ + --admin-password -To create an Azure OpenAI resource, you need an Azure resource group. When you create a new resource through the Azure CLI, you can also create a new resource group or instruct Azure to use an existing group. The following example shows how to create a new resource group named _OAIResourceGroup_ with the [az group create](/cli/azure/group?view=azure-cli-latest&preserve-view=true#az-group-create) command. The resource group is created in the East US location. + az vm create \ + --resource-group ansible-inventory-test-rg \ + --name linux-vm \ + --image Ubuntu2204 \ + --admin-username azureuser \ + --admin-password + ``` + + # [Azure PowerShell](#tab/azure-powershell) + + ```azurepowershell + $adminUsername = "azureuser" + $adminPassword = ConvertTo-SecureString -AsPlainText -Force + $credential = New-Object System.Management.Automation.PSCredential ($adminUsername, $adminPassword); + + New-AzVM ` + -ResourceGroupName ansible-inventory-test-rg ` + -Location eastus ` + -Image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest ` + -Name win-vm ` + -OpenPorts 3389 ` + -Credential $credential + + New-AzVM ` + -ResourceGroupName ansible-inventory-test-rg ` + -Location eastus ` + -Image Ubuntu2204 ` + -Name linux-vm ` + -OpenPorts 22 ` + -Credential $credential + ``` + --- + + Replace the `` your password. + +## Add application role tags + +Tags are used to organize and categorize Azure resources. Assigning the Azure VMs an application role allows you to use the tags as group names within the Azure dynamic inventory. + +Run the following commands to update the VM tags: + +# [Azure CLI](#tab/azure-cli) ```azurecli-interactive -az group create \ ---name OAIResourceGroup \ ---location eastus +az vm update \ +--resource-group ansible-inventory-test-rg \ +--name linux-vm \ +--set tags.applicationRole='message-broker' + +az vm update \ +--resource-group ansible-inventory-test-rg \ +--name win-vm \ +--set tags.applicationRole='web-server' ``` -## Create a resource +# [Azure PowerShell](#tab/azure-powershell) -Use the [az cognitiveservices account create](/cli/azure/cognitiveservices/account?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-create) command to create an Azure OpenAI resource in the resource group. In the following example, you create a resource named _MyOpenAIResource_ in the _OAIResourceGroup_ resource group. When you try the example, update the code to use your desired values for the resource group and resource name, along with your Azure subscription ID _\_. +```azurepowershell +Get-AzVM -Name win-vm -ResourceGroupName ansible-inventory-test-rg-pwsh | Update-AzVM -Tag @{"applicationRole"="web-server"} -```azurecli -az cognitiveservices account create \ ---name MyOpenAIResource \ ---resource-group OAIResourceGroup \ ---location eastus \ ---kind OpenAI \ ---sku s0 \ ---subscription +Get-AzVM -Name linux-vm -ResourceGroupName ansible-inventory-test-rg-pwsh | Update-AzVM -Tag @{"applicationRole"="message-broker"} ``` -## Retrieve information about the resource +--- + +Learn more about Azure tagging strategies at [Define your tagging strategy](/azure/cloud-adoption-framework/ready/azure-best-practices/resource-tagging). + +## Generate a dynamic inventory + +Ansible provides an [Azure dynamic-inventory plug-in](https://github.com/ansible/ansible/blob/stable-2.9/lib/ansible/plugins/inventory/azure_rm.py). + +The following steps walk you through using the plug-in: + +1. Create a dynamic inventory named `myazure_rm.yml` + + ```yml + plugin: azure_rm + include_vm_resource_groups: + - ansible-inventory-test-rg + auth_source: auto + ``` + + **Key point:** + * Ansible uses the inventory file name and extension to identify which inventory plug-in to use. To use the Azure dynamic inventory plug-in, the file must end with `azure_rm` and have an extension of either `yml` or `yaml`. + +1. Run the following command to query the VMs within the resource group: + + ```bash + ansible-inventory -i myazure_rm.yml --graph + ``` + +1. When you run the command, you see results similar to the following output: + + ```output + @all: + |--@ungrouped: + | |--linux-vm_cdb4 + | |--win-vm_3211 + ``` -After you create the resource, you can use different commands to find useful information about your Azure OpenAI Service instance. The following examples demonstrate how to retrieve the REST API endpoint base URL and the access keys for the new resource. +Both VMs belong to the `ungrouped` group, which is a child of the `all` group in the Ansible inventory. -### Get the endpoint URL +**Key point**: +* By default the Azure dynamic inventory plug-in returns globally unique names. For this reason, the VM names may contain extra characters. You can disable that behavior by adding `plain_host_names: yes` to the dynamic inventory. -Use the [az cognitiveservices account show](/cli/azure/cognitiveservices/account?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-show) command to retrieve the REST API endpoint base URL for the resource. In this example, we direct the command output through the [jq](https://jqlang.github.io/jq/) JSON processor to locate the `.properties.endpoint` value. +## Find Azure VM hostvars -When you try the example, update the code to use your values for the resource group _\_ and resource _\_. +Run the following command to view all the `hostvars`: -```azurecli -az cognitiveservices account show \ ---name \ ---resource-group \ -| jq -r .properties.endpoint +```bash +ansible-inventory -i myazure_rm.yml --list ``` -### Get the primary API key +```output +{ + "_meta": { + "hostvars": { + "linux-vm_cdb4": { + "ansible_host": "52.188.118.79", + "availability_zone": null, + "computer_name": "linux-vm", + "default_inventory_hostname": "linux-vm_cdb4", + "id": "/subscriptions//resourceGroups/ansible-inventory-test-rg/providers/Microsoft.Compute/virtualMachines/linux-vm", + "image": { + "offer": "0001-com-ubuntu-server-jammy", + "publisher": "Canonical", + "sku": "22_04-lts-gen2", + "version": "latest" + }, + ..., + "tags": { + "applicationRole": "message-broker" + }, + ... + }, + "win-vm_3211": { + "ansible_host": "52.188.112.110", + "availability_zone": null, + "computer_name": "win-vm", + "default_inventory_hostname": "win-vm_3211", + "id": "/subscriptions//resourceGroups/ansible-inventory-test-rg/providers/Microsoft.Compute/virtualMachines/win-vm", + "image": { + "offer": "WindowsServer", + "publisher": "MicrosoftWindowsServer", + "sku": "2019-Datacenter", + "version": "latest" + }, + ... + "tags": { + "applicationRole": "web-server" + }, + ... + } + } + }, + ... + } +} +``` + +By pulling information from Azure, the dynamic inventory populates the `hostvars` for each Azure VM. Those `hostvars` are then to determine the VM group memberships within the Ansible inventory. + +## Assign group membership with conditional_groups -To retrieve the access keys for the resource, use the [az cognitiveservices account keys list](/cli/azure/cognitiveservices/account?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-keys-list) command. In this example, we direct the command output through the [jq](https://jqlang.github.io/jq/) JSON processor to locate the `.key1` value. +Each conditional group is made of two parts. The name of the group and the condition for adding a member to the group. -When you try the example, update the code to use your values for the resource group and resource. +Use the property `image.offer` to create conditional group membership for the _linux-vm_. -```azurecli -az cognitiveservices account keys list \ ---name \ ---resource-group \ -| jq -r .key1 +Open the `myazure_rm.yml` dynamic inventory and add the following `conditional_group`: + +```yml +plugin: azure_rm +include_vm_resource_groups: + - ansible-inventory-test-rg +auth_source: auto +conditional_groups: + linux: "'ubuntu' in image.offer" + windows: "'WindowsServer' in image.offer" ``` -## Deploy a model +Run the `ansible-inventory` with the `--graph` option: -To deploy a model, use the [az cognitiveservices account deployment create](/cli/azure/cognitiveservices/account/deployment?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-deployment-create) command. In the following example, you deploy an instance of the `text-embedding-ada-002` model and give it the name _MyModel_. When you try the example, update the code to use your values for the resource group and resource. You don't need to change the `model-version`, `model-format` or `sku-capacity`, and `sku-name` values. +```bash +ansible-inventory -i myazure_rm.yml --graph +``` -```azurecli -az cognitiveservices account deployment create \ ---name \ ---resource-group \ ---deployment-name MyModel \ ---model-name text-embedding-ada-002 \ ---model-version "1" \ ---model-format OpenAI \ ---sku-capacity "1" \ ---sku-name "Standard" +```output +@all: + |--@linux: + | |--linux-vm_cdb4 + |--@ungrouped: + |--@windows: + | |--win-vm_3211 ``` -`--sku-name` accepts the following deployment types: `Standard`, `GlobalBatch`, `GlobalStandard`, and `ProvisionedManaged`. Learn more about [deployment type options](../how-to/deployment-types.md). +From the output, you can see the VMs are no longer associated with the `ungrouped` group. Instead, each VM is assigned to a new group created by the dynamic inventory. +**Key point**: +* Conditional groups allow you to name specific groups within your inventory and populate them using `hostvars`. -> [!IMPORTANT] -> When you access the model via the API, you need to refer to the deployment name rather than the underlying model name in API calls, which is one of the [key differences](../how-to/switching-endpoints.yml) between OpenAI and Azure OpenAI. OpenAI only requires the model name. Azure OpenAI always requires deployment name, even when using the model parameter. In our docs, we often have examples where deployment names are represented as identical to model names to help indicate which model works with a particular API endpoint. Ultimately your deployment names can follow whatever naming convention is best for your use case. +## Assign group membership with keyed_groups -## Delete a model from your resource +Keyed groups assign group membership the same way conditional groups do, but when using a keyed group the group name is also dynamically populated. -You can delete any model deployed from your resource with the [az cognitiveservices account deployment delete](/cli/azure/cognitiveservices/account/deployment?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-deployment-delete) command. In the following example, you delete a model named _MyModel_. When you try the example, update the code to use your values for the resource group, resource, and deployed model. +Add the following keyed_group to the `myazure_rm.yml` dynamic inventory: -```azurecli -az cognitiveservices account deployment delete \ ---name \ ---resource-group \ ---deployment-name MyModel +```yml +plugin: azure_rm +include_vm_resource_groups: + - ansible-inventory-test-rg +auth_source: auto +conditional_groups: + linux: "'ubuntu' in image.offer" + windows: "'WindowsServer' in image.offer" +keyed_groups: + - key: tags.applicationRole ``` -## Delete a resource +Run the `ansible-inventory` with the `--graph` option: -If you want to clean up after these exercises, you can remove your Azure OpenAI resource by deleting the resource through the Azure CLI. You can also delete the resource group. If you choose to delete the resource group, all resources contained in the group are also deleted. +```bash +ansible-inventory -i myazure_rm.yml --graph +``` -To remove the resource group and its associated resources, use the [az cognitiveservices account delete](/cli/azure/cognitiveservices/account/deployment?view=azure-cli-latest&preserve-view=true#az-cognitiveservices-account-delete) command. +```output +@all: + |--@_message_broker: + | |--linux-vm_cdb4 + |--@_web_server: + | |--win-vm_3211 + |--@linux: + | |--linux-vm_cdb4 + |--@ungrouped: + |--@windows: + | |--win-vm_3211 +``` -If you're not going to continue to use the resources created in these exercises, run the following command to delete your resource group. Be sure to update the example code to use your values for the resource group and resource. +From the output, you see two more groups `_message_broker` and `_web_server`. By using a keyed group, the `applicationRole` tag populates the group names and group memberships. + +**Key point**: +* By default, keyed groups include a separator. To remove the separator, add `separator: ""` under the key property. + +## Run playbooks with group name patterns + +Use the groups created by the dynamic inventory to target subgroups. + +1. Create a playbook called `win_ping.yml` with the following contents: + + ```yml + --- + - hosts: windows + gather_facts: false + + vars_prompt: + - name: username + prompt: "Enter local username" + private: false + - name: password + prompt: "Enter password" + + vars: + ansible_user: "{{ username }}" + ansible_password: "{{ password }}" + ansible_connection: winrm + ansible_winrm_transport: ntlm + ansible_winrm_server_cert_validation: ignore + + tasks: + - name: run win_ping + win_ping: + ``` + +1. Run the `win_ping.yml` playbook. + + ```bash + ansible-playbook win_ping.yml -i myazure_rm.yml + ``` + + When prompted, enter the `username` and `password` for the Azure Windows VM. + + ```output + Enter local username: azureuser + Enter password: + + PLAY [windows] ************************************************************************************************************************************** + + TASK [run win_ping] ********************************************************************************************************************************* + ok: [win-vm_3211] + + PLAY RECAP ****************************************************************************************************************************************** + win-vm_3211 : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 + ``` + + > [!IMPORTANT] + > If you get the error `winrm or requests is not installed: No module named 'winrm'`, install pywinrm with the following command: `pip install "pywinrm>=0.3.0"` + +1. Create a second playbook named `ping.yml` with the following contents: + + ```yml + --- + - hosts: all + gather_facts: false + + vars_prompt: + - name: username + prompt: "Enter ssh user" + - name: password + prompt: "Enter password for ssh user" + + vars: + ansible_user: "{{ username }}" + ansible_password: "{{ password }}" + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + + tasks: + - name: run ping + ping: + ``` + +1. Run the `ping.yml` playbook. + + ```bash + ansible-playbook ping.yml -i myazure_rm.yml + ``` + + When prompted, enter the `username` and `password` for the Azure Linux VM. + + ```output + Enter ssh username: azureuser + Enter password for ssh user: + + PLAY [linux] ******************************************************************************************************* + + TASK [run ping] **************************************************************************************************** + ok: [linux-vm_cdb4] + + PLAY RECAP ********************************************************************************************************* + linux-vm_cdb4 : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 + ``` + +## Clean up resources + +# [Azure CLI](#tab/azure-cli) + +1. Run [az group delete](/cli/azure/group#az-group-delete) to delete the resource group. All resources within the resource group are deleted. + + ```azurecli + az group delete --name + ``` + +1. Verify that the resource group was deleted by using [az group show](/cli/azure/group#az-group-show). + + ```azurecli + az group show --name + ``` + +# [Azure PowerShell](#tab/azure-powershell) + +1. Run [Remove-AzResourceGroup](/powershell/module/az.resources/Remove-AzResourceGroup) to delete the resource group. All resources within the resource group are deleted. + + ```azurepowershell + Remove-AzResourceGroup -Name + ``` + +1. Verify that the resource group was deleted by using [Get-AzResourceGroup](/powershell/module/az.resources/Get-AzResourceGroup). + + ```azurepowershell + Get-AzResourceGroup -Name + ``` -```azurecli -az cognitiveservices account delete \ ---name \ ---resource-group -``` +--- ## Next steps -- [Get started with the Azure OpenAI security building block](/azure/developer/ai/get-started-securing-your-ai-app?tabs=github-codespaces&pivots=python) -- Make API calls and generate text with [Azure OpenAI Service quickstarts](../quickstart.md). -- Learn more about the [Azure OpenAI Service models](../concepts/models.md). -- For information on pricing visit the [Azure OpenAI pricing page](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/) \ No newline at end of file +> [!div class="nextstepaction"] +> [Quickstart: Configure Linux virtual machines in Azure using Ansible](./vm-configure.md) \ No newline at end of file diff --git a/tools/convert_converted.md b/tools/convert_converted.md index 3544ff150..8637dec59 100644 --- a/tools/convert_converted.md +++ b/tools/convert_converted.md @@ -1,194 +1,387 @@ --- -title: 'How-to: Create and deploy an Azure OpenAI Service resource' -titleSuffix: Azure OpenAI -description: Learn how to get started with Azure OpenAI Service and create your first resource and deploy your first model in the Azure CLI or the Azure portal. -#services: cognitive-services -manager: nitinme -ms.service: azure-ai-openai -ms.custom: devx-track-azurecli, build-2023, build-2023-dataai, devx-track-azurepowershell, innovation-engine -ms.topic: how-to -ms.date: 01/31/2025 -zone_pivot_groups: openai-create-resource -author: mrbullwinkle -ms.author: mbullwin -recommendations: false +title: Tutorial - Configure dynamic inventories for Azure Virtual Machines using Ansible +description: Learn how to populate your Ansible inventory dynamically from information in Azure +keywords: ansible, azure, devops, bash, cloudshell, dynamic inventory +ms.topic: tutorial +ms.date: 08/14/2024 +ms.custom: devx-track-ansible, devx-track-azurecli, devx-track-azurepowershell, linux-related-content +author: ansibleexpert +ms.author: ansibleexpert --- -# Create and deploy an Azure OpenAI Service resource +# Tutorial: Configure dynamic inventories of your Azure resources using Ansible -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://go.microsoft.com/fwlink/?linkid=2303211) +[!INCLUDE [ansible-28-note.md](includes/ansible-28-note.md)] -This article describes how to get started with Azure OpenAI Service and provides step-by-step instructions to create a resource and deploy a model. You can create resources in Azure in several different ways: +Before you begin, ensure that your environment has Ansible installed. -- The [Azure portal](https://portal.azure.com/?microsoft_azure_marketplace_ItemHideKey=microsoft_openai_tip#create/Microsoft.CognitiveServicesOpenAI) -- The REST APIs, the Azure CLI, PowerShell, or client libraries -- Azure Resource Manager (ARM) templates +Set the following environment variables. These declarations ensure unique resource names and provide needed configuration so that the Exec Doc runs non-interactively. -In this article, you review examples for creating and deploying resources in the Azure portal and with the Azure CLI. +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP="ansibleinventorytestrg${RANDOM_SUFFIX}" +export REGION="centralindia" +export ADMIN_PASSWORD="P@ssw0rd123!" +``` + +The [Ansible dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_dynamic_inventory.html) feature removes the burden of maintaining static inventory files. + +In this tutorial, you use Azure's dynamic-inventory plug-in to populate your Ansible inventory. + +In this article, you learn how to: + +> [!div class="checklist"] +> * Configure two test virtual machines. +> * Add tags to Azure virtual machines. +> * Generate a dynamic inventory. +> * Use conditional and keyed groups to populate group memberships. +> * Run playbooks against groups within the dynamic inventory. ## Prerequisites -- An Azure subscription. Create one for free. -- Access permissions to [create Azure OpenAI resources and to deploy models](../how-to/role-based-access-control.md). -- The Azure CLI. For more information, see [How to install the Azure CLI](/cli/azure/install-azure-cli). +[!INCLUDE [open-source-devops-prereqs-azure-subscription.md](../includes/open-source-devops-prereqs-azure-subscription.md)] +[!INCLUDE [open-source-devops-prereqs-create-service-principal.md](../includes/open-source-devops-prereqs-create-service-principal.md)] +[!INCLUDE [ansible-prereqs-cloudshell-use-or-vm-creation2.md](includes/ansible-prereqs-cloudshell-use-or-vm-creation2.md)] + +## Create Azure VMs + +1. Sign in to the [Azure portal](https://go.microsoft.com/fwlink/p/?LinkID=525040). + +2. Open [Cloud Shell](/azure/cloud-shell/overview). + +3. Create an Azure resource group to hold the virtual machines for this tutorial. + + > [!IMPORTANT] + > The Azure resource group you create in this step must have a name that is entirely lower-case. Otherwise, the generation of the dynamic inventory will fail. + + # [Azure CLI](#tab/azure-cli) + ```azurecli-interactive + az group create --resource-group $RESOURCE_GROUP --location $REGION + ``` + # [Azure PowerShell] + ```azurepowershell + New-AzResourceGroup -Name $RESOURCE_GROUP -Location $REGION + ``` + +4. Create two virtual machines on Azure using one of the following techniques: + + - **Ansible playbook** – The articles [Create a basic Linux virtual machine in Azure with Ansible](./vm-configure.md) and [Create a basic Windows virtual machine in Azure with Ansible](./vm-configure-windows.md) illustrate how to create a virtual machine from an Ansible playbook. + + - **Azure CLI** – Issue each of the following commands in Cloud Shell to create the two virtual machines. Note that the --size parameter is provided to avoid unavailable SKU errors. + + # [Azure CLI](#tab/azure-cli) + ```azurecli-interactive + az vm create \ + --resource-group $RESOURCE_GROUP \ + --name win-vm \ + --image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest \ + --size Standard_B1s \ + --admin-username azureuser \ + --admin-password $ADMIN_PASSWORD + + az vm create \ + --resource-group $RESOURCE_GROUP \ + --name linux-vm \ + --image Ubuntu2204 \ + --size Standard_B1s \ + --admin-username azureuser \ + --admin-password $ADMIN_PASSWORD + ``` + + # [Azure PowerShell] + ```azurepowershell + $adminUsername = "azureuser" + $adminPassword = ConvertTo-SecureString $env:ADMIN_PASSWORD -AsPlainText -Force + $credential = New-Object System.Management.Automation.PSCredential ($adminUsername, $adminPassword); + + New-AzVM ` + -ResourceGroupName $RESOURCE_GROUP ` + -Location $REGION ` + -Image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest ` + -Name win-vm ` + -Size Standard_B1s ` + -OpenPorts 3389 ` + -Credential $credential + + New-AzVM ` + -ResourceGroupName $RESOURCE_GROUP ` + -Location $REGION ` + -Image Ubuntu2204 ` + -Name linux-vm ` + -Size Standard_B1s ` + -OpenPorts 22 ` + -Credential $credential + ``` + (Replace any password placeholders with the variable ADMIN_PASSWORD.) + +## Add application role tags + +Tags are used to organize and categorize Azure resources. Assigning the Azure VMs an application role allows you to use the tags as group names within the Azure dynamic inventory. + +Run the following commands to update the VM tags: + +# [Azure CLI](#tab/azure-cli) +```azurecli-interactive +az vm update \ + --resource-group $RESOURCE_GROUP \ + --name linux-vm \ + --set tags.applicationRole='message-broker' + +az vm update \ + --resource-group $RESOURCE_GROUP \ + --name win-vm \ + --set tags.applicationRole='web-server' +``` -## Create an Azure resource group +# [Azure PowerShell] +```azurepowershell +Update-AzVM -VM (Get-AzVM -Name win-vm -ResourceGroupName $RESOURCE_GROUP) -Tag @{applicationRole="web-server"} +Update-AzVM -VM (Get-AzVM -Name linux-vm -ResourceGroupName $RESOURCE_GROUP) -Tag @{applicationRole="message-broker"} +``` -To create an Azure OpenAI resource, you need an Azure resource group. When you create a new resource through the Azure CLI, you can also create a new resource group or instruct Azure to use an existing group. The following example shows how to create a new resource group named OAIResourceGroup with the az group create command. The resource group is created in the East US location. +Learn more about Azure tagging strategies at [Define your tagging strategy](/azure/cloud-adoption-framework/ready/azure-best-practices/resource-tagging). -```azurecli -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export REGION="eastus" -export OAI_RESOURCE_GROUP="OAIResourceGroup$RANDOM_SUFFIX" -az group create --name $OAI_RESOURCE_GROUP --location $REGION +## Generate a dynamic inventory + +Ansible provides an [Azure dynamic-inventory plug-in](https://github.com/ansible/ansible/blob/stable-2.9/lib/ansible/plugins/inventory/azure_rm.py). + +The following steps walk you through using the plug-in: + +1. Create a dynamic inventory named "myazure_rm.yml" with the basic configuration. + +```bash +cat < myazure_rm.yml +plugin: azure_rm +include_vm_resource_groups: + - ${RESOURCE_GROUP} +auth_source: auto +EOF ``` -Results: +2. Run the following command to query the VMs within the resource group. - +```bash +ansible-inventory -i myazure_rm.yml --graph +``` -```JSON -{ - "id": "/subscriptions/xxxxx/resourceGroups/OAIResourceGroupxxxxx", - "location": "eastus", - "managedBy": null, - "name": "OAIResourceGroupxxxxx", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" -} +Results: + + +```text +@all: + |--@ungrouped: + |--linux-vm_abc123 + |--win-vm_def456 ``` -## Create a resource +## Find Azure VM hostvars -Use the az cognitiveservices account create command to create an Azure OpenAI resource in the resource group. In the following example, you create a resource named MyOpenAIResource in the OAI_RESOURCE_GROUP resource group. When you try the example, update the code to use your desired values for the resource group and resource name. +Run the following command to view all the hostvars: -```azurecli -export OPENAI_RESOURCE_NAME="MyOpenAIResource$RANDOM_SUFFIX" -az cognitiveservices account create \ ---name $OPENAI_RESOURCE_NAME \ ---resource-group $OAI_RESOURCE_GROUP \ ---location $REGION \ ---kind OpenAI \ ---sku s0 +```bash +ansible-inventory -i myazure_rm.yml --list ``` -Results: +Results: - -```JSON +```json { - "id": "/subscriptions/xxxxx/resourceGroups/OAIResourceGroupxxxxx/providers/Microsoft.CognitiveServices/accounts/MyOpenAIResourcexxxxx", - "kind": "OpenAI", - "location": "eastus", - "name": "MyOpenAIResourcexxxxx", - "properties": { - "provisioningState": "Succeeded" - }, - "sku": { - "name": "s0" - }, - "type": "Microsoft.CognitiveServices/accounts" + "_meta": { + "hostvars": { + "linux-vm_abc123": { + "ansible_host": "x.x.x.x" + }, + "win-vm_def456": { + "ansible_host": "x.x.x.x" + } + } + } } ``` -## Retrieve information about the resource +## Assign group membership with conditional_groups + +Each conditional group is made of two parts: the name of the group and the condition for adding a member to the group. -After you create the resource, you can use different commands to find useful information about your Azure OpenAI Service instance. The following examples demonstrate how to retrieve the REST API endpoint base URL and the access keys for the new resource. +Use the property image.offer to create conditional group membership for the linux-vm. -### Get the endpoint URL +Open the myazure_rm.yml dynamic inventory and update it to include the following conditional_groups section. This overwrites the previous file. -Use the az cognitiveservices account show command to retrieve the REST API endpoint base URL for the resource. In this example, we direct the command output through the jq JSON processor to locate the .properties.endpoint value. +```bash +cat < myazure_rm.yml +plugin: azure_rm +include_vm_resource_groups: + - ${RESOURCE_GROUP} +auth_source: auto +conditional_groups: + linux: "'ubuntu' in image.offer" + windows: "'WindowsServer' in image.offer" +EOF +``` -When you try the example, update the code to use your values for the resource group and resource. +Run the ansible-inventory command with the --graph option: -```azurecli -az cognitiveservices account show \ ---name $OPENAI_RESOURCE_NAME \ ---resource-group $OAI_RESOURCE_GROUP \ -| jq -r .properties.endpoint +```bash +ansible-inventory -i myazure_rm.yml --graph ``` -Results: +Results: - ```text -https://openaiendpointxxxxx.cognitiveservices.azure.com/ +@all: + |--@linux: + |--linux-vm_abc123 + |--@ungrouped: + |--@windows: + |--win-vm_def456 ``` -### Get the primary API key +From the output, you can see the VMs are no longer associated with the "ungrouped" group. Instead, each VM is assigned to a new group created by the dynamic inventory. + +## Assign group membership with keyed_groups -To retrieve the access keys for the resource, use the az cognitiveservices account keys list command. In this example, we direct the command output through the jq JSON processor to locate the .key1 value. +Keyed groups assign group membership in a similar manner as conditional groups, but the group name is dynamically populated based on the resource tag. -When you try the example, update the code to use your values for the resource group and resource. +Update the myazure_rm.yml dynamic inventory to include the keyed_groups section. This overwrites the previous file. -```azurecli -az cognitiveservices account keys list \ ---name $OPENAI_RESOURCE_NAME \ ---resource-group $OAI_RESOURCE_GROUP \ -| jq -r .key1 +```bash +cat < myazure_rm.yml +plugin: azure_rm +include_vm_resource_groups: + - ${RESOURCE_GROUP} +auth_source: auto +conditional_groups: + linux: "'ubuntu' in image.offer" + windows: "'WindowsServer' in image.offer" +keyed_groups: + - key: tags.applicationRole +EOF ``` -Results: +Run the ansible-inventory command with the --graph option: - +```bash +ansible-inventory -i myazure_rm.yml --graph +``` + +Results: + ```text -xxxxxxxxxxxxxxxxxxxxxx +@all: + |--@_message_broker: + |--linux-vm_abc123 + |--@_web_server: + |--win-vm_def456 + |--@linux: + |--linux-vm_abc123 + |--@ungrouped: + |--@windows: + |--win-vm_def456 ``` -## Deploy a model +From the output, you see two more groups _message_broker and _web_server. By using a keyed group, the applicationRole tag populates the group names and group memberships. + +## Run playbooks with group name patterns -To deploy a model, use the az cognitiveservices account deployment create command. In the following example, you deploy an instance of the text-embedding-ada-002 model and give it the name MyModel. When you try the example, update the code to use your values for the resource group and resource. You don't need to change the model-version, model-format, sku-capacity, or sku-name values. +Use the groups created by the dynamic inventory to target subgroups. -```azurecli -export MODEL_DEPLOYMENT_NAME="MyModel" -az cognitiveservices account deployment create \ ---name $OPENAI_RESOURCE_NAME \ ---resource-group $OAI_RESOURCE_GROUP \ ---deployment-name $MODEL_DEPLOYMENT_NAME \ ---model-name text-embedding-ada-002 \ ---model-version "1" \ ---model-format OpenAI \ ---sku-capacity "1" \ ---sku-name "Standard" +1. Create a playbook called win_ping.yml with the following contents. Predefined variables are provided so that no interactive prompts occur. + +```bash +cat < win_ping.yml +--- +- hosts: windows + gather_facts: false + + vars: + username: "azureuser" + password: "${ADMIN_PASSWORD}" + ansible_user: "{{ username }}" + ansible_password: "{{ password }}" + ansible_connection: winrm + ansible_winrm_transport: ntlm + ansible_winrm_server_cert_validation: ignore + + tasks: + - name: run win_ping + win_ping: +EOF ``` -Results: +2. Run the win_ping.yml playbook. + +```bash +ansible-playbook win_ping.yml -i myazure_rm.yml +``` + +Results: +```text +PLAY [windows] ************************************************************* -```JSON -{ - "deploymentName": "MyModel", - "provisioningState": "Succeeded" -} +TASK [run win_ping] ******************************************************* +ok: [win-vm_def456] + +PLAY RECAP *************************************************************** +win-vm_def456 : ok=1 changed=0 unreachable=0 failed=0 ``` -> [!IMPORTANT] -> When you access the model via the API, you need to refer to the deployment name rather than the underlying model name in API calls, which is one of the [key differences](../how-to/switching-endpoints.yml) between OpenAI and Azure OpenAI. OpenAI only requires the model name. Azure OpenAI always requires deployment name, even when using the model parameter. In our docs, we often have examples where deployment names are represented as identical to model names to help indicate which model works with a particular API endpoint. Ultimately your deployment names can follow whatever naming convention is best for your use case. +If you get the error "winrm or requests is not installed: No module named 'winrm'", install pywinrm with the following command: -## Delete a model from your resource +```bash +pip install "pywinrm>=0.3.0" +``` + +3. Create a second playbook named ping.yml with the following contents. Predefined variables are provided so that no interactive prompts occur. + +```bash +cat < ping.yml +--- +- hosts: all + gather_facts: false + + vars: + username: "azureuser" + password: "${ADMIN_PASSWORD}" + ansible_user: "{{ username }}" + ansible_password: "{{ password }}" + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' + + tasks: + - name: run ping + ping: +EOF +``` + +4. Run the ping.yml playbook. -You can delete any model deployed from your resource with the az cognitiveservices account deployment delete command. In the following example, the original document provided instructions to delete a model named MyModel. When you try the example, update the code to use your values for the resource group, resource, and deployed model. +```bash +ansible-playbook ping.yml -i myazure_rm.yml +``` + +Results: -(Note: The deletion code block has been removed from this Exec Doc as deletion commands are not executed automatically in Exec Docs.) + +```text +PLAY [all] ************************************************************* +TASK [run ping] ******************************************************* +ok: [linux-vm_abc123] -## Delete a resource +PLAY RECAP ************************************************************* +linux-vm_abc123 : ok=1 changed=0 unreachable=0 failed=0 +``` -If you want to clean up after these exercises, you can remove your Azure OpenAI resource by deleting the resource through the Azure CLI. You can also delete the resource group. If you choose to delete the resource group, all resources contained in the group are also deleted. +## Clean up resources -To remove the resource group and its associated resources, the original document provided a command example. Be sure to update the example code to use your values for the resource group and resource. +(Note: In Exec Docs, deletion commands that remove resources are omitted to prevent accidental deletion during automated execution.) -(Note: The deletion code block has been removed from this Exec Doc as deletion commands are not executed automatically in Exec Docs.) +--- ## Next steps -- [Get started with the Azure OpenAI security building block](/azure/developer/ai/get-started-securing-your-ai-app?tabs=github-codespaces&pivots=python) -- Make API calls and generate text with [Azure OpenAI Service quickstarts](../quickstart.md). -- Learn more about the [Azure OpenAI Service models](../concepts/models.md). -- For information on pricing visit the [Azure OpenAI pricing page](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/) \ No newline at end of file +> [!div class="nextstepaction"] +> [Quickstart: Configure Linux virtual machines in Azure using Ansible](./vm-configure.md) \ No newline at end of file diff --git a/tools/execution_log.csv b/tools/execution_log.csv index 7de510738..29267d704 100644 --- a/tools/execution_log.csv +++ b/tools/execution_log.csv @@ -1266,3 +1266,56 @@ See vm create -h for more information on specifying an image. Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_sx95BEHUMfLmMWthesw8MpVq7FOIx45d' is not valid according to the validation procedure. The tracking id is '44b14b15-e2ea-4ac8-b5db-a9415338882f'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} ' StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_sx95BEHUMfLmMWthesw8MpVq7FOIx45d' is not valid according to the validation procedure. The tracking id is '44b14b15-e2ea-4ac8-b5db-a9415338882f'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}}",245.48310685157776,Success +2025-03-06 18:43:21,file,convert.md,convert_converted.md,11,"time=2025-03-06T18:01:49-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 0. +Error: unexpected end of JSON input +StdErr: + + time=2025-03-06T18:03:00-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. +Error: command exited with 'exit status 1' and the message 'WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. +ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_uwCOm6AsMFaPq38JK3iUeOr5GzysgPPQ' is not valid according to the validation procedure. The tracking id is '4b44146b-f9ec-45c0-b06e-4547a098c85d'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} +' +StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. +ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_uwCOm6AsMFaPq38JK3iUeOr5GzysgPPQ' is not valid according to the validation procedure. The tracking id is '4b44146b-f9ec-45c0-b06e-4547a098c85d'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} + + time=2025-03-06T18:04:04-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. +Error: command exited with 'exit status 1' and the message 'WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. +ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_NH1l0tnhOQitW1xREka8tnjlo6i9gBYS' is not valid according to the validation procedure. The tracking id is '53aa2916-7335-490c-bcf1-69953a136620'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} +' +StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. +ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_NH1l0tnhOQitW1xREka8tnjlo6i9gBYS' is not valid according to the validation procedure. The tracking id is '53aa2916-7335-490c-bcf1-69953a136620'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} + + time=2025-03-06T18:09:03-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 127' and the message 'bash: line 2: =: command not found +' +StdErr: bash: line 2: =: command not found + + time=2025-03-06T18:13:52-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. +Error: command exited with 'exit status 127' and the message 'bash: line 2: =: command not found +' +StdErr: bash: line 2: =: command not found + + time=2025-03-06T18:18:58-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. +Error: invalid character '{' after top-level value +StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. + + time=2025-03-06T18:23:55-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. +Error: invalid character '{' after top-level value +StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. + + time=2025-03-06T18:28:29-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. +Error: invalid character '{' after top-level value +StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. + + time=2025-03-06T18:34:32-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 2. +Error: command exited with 'exit status 127' and the message 'bash: line 2: ansible-inventory: command not found +' +StdErr: bash: line 2: ansible-inventory: command not found + + time=2025-03-06T18:35:22-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 0. +Error: command exited with 'exit status 1' and the message '' +StdErr: + + time=2025-03-06T18:43:21-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 2. +Error: command exited with 'exit status 127' and the message 'bash: line 2: ansible-inventory: command not found +' +StdErr: bash: line 2: ansible-inventory: command not found",2563.7570362091064,Failure diff --git a/tools/myazure_rm.yml b/tools/myazure_rm.yml new file mode 100644 index 000000000..57ce22eea --- /dev/null +++ b/tools/myazure_rm.yml @@ -0,0 +1,4 @@ +plugin: azure_rm +include_vm_resource_groups: + - ansibleinventorytestrg115b21 +auth_source: auto diff --git a/tools/myazure_rm.yml (initial version) b/tools/myazure_rm.yml (initial version) new file mode 100644 index 000000000..716364a7f --- /dev/null +++ b/tools/myazure_rm.yml (initial version) @@ -0,0 +1,4 @@ +plugin: azure_rm +include_vm_resource_groups: + - ${RESOURCE_GROUP} +auth_source: auto diff --git a/tools/myazure_rm.yml (with conditional_groups) b/tools/myazure_rm.yml (with conditional_groups) new file mode 100644 index 000000000..c5801d936 --- /dev/null +++ b/tools/myazure_rm.yml (with conditional_groups) @@ -0,0 +1,7 @@ +plugin: azure_rm +include_vm_resource_groups: + - ${RESOURCE_GROUP} +auth_source: auto +conditional_groups: + linux: "'ubuntu' in image.offer" + windows: "'WindowsServer' in image.offer" diff --git a/tools/myazure_rm.yml (with keyed_groups) b/tools/myazure_rm.yml (with keyed_groups) new file mode 100644 index 000000000..c32162213 --- /dev/null +++ b/tools/myazure_rm.yml (with keyed_groups) @@ -0,0 +1,9 @@ +plugin: azure_rm +include_vm_resource_groups: + - ${RESOURCE_GROUP} +auth_source: auto +conditional_groups: + linux: "'ubuntu' in image.offer" + windows: "'WindowsServer' in image.offer" +keyed_groups: + - key: tags.applicationRole diff --git a/tools/ping.yml b/tools/ping.yml new file mode 100644 index 000000000..e4a180cbd --- /dev/null +++ b/tools/ping.yml @@ -0,0 +1,9 @@ +- gather_facts: false + hosts: all + tasks: + - name: run ping + ping: null + vars: + ansible_password: '{{ lookup(''env'',''SSH_PASSWORD'') }}' + ansible_ssh_common_args: -o StrictHostKeyChecking=no + ansible_user: '{{ lookup(''env'',''SSH_USER'') }}' diff --git a/tools/win_ping.yml b/tools/win_ping.yml new file mode 100644 index 000000000..3b0bfcf00 --- /dev/null +++ b/tools/win_ping.yml @@ -0,0 +1,11 @@ +- gather_facts: false + hosts: windows + tasks: + - name: run win_ping + win_ping: null + vars: + ansible_connection: winrm + ansible_password: '{{ lookup(''env'',''WIN_PASSWORD'') }}' + ansible_user: '{{ lookup(''env'',''WIN_USER'') }}' + ansible_winrm_server_cert_validation: ignore + ansible_winrm_transport: ntlm From 74044b5eeee2e31544586eb1988b49e6792de556 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 00:44:16 -0700 Subject: [PATCH 17/87] updated documentation with technical constraints --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6e2dc6a0b..b6b9794c5 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters **Example:** ```markdown - ```bash - az group create --name myResourceGroup --location eastus - ``` + ```bash + az group create --name myResourceGroup --location eastus + ``` ``` >**Note:** This rule does not apply to output code blocks, which are used to display the results of commands, scripts, or other operations. These blocks help in illustrating what the expected output should look like. They include, but are not limited to, the following types: _output, json, yaml, console, text, and log._ @@ -57,10 +57,10 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters **Example of supported command:** ```markdown - ```bash - export VM_NAME="myVM" - az vm create --name $VM_NAME --resource-group myResourceGroup --image UbuntuLTS - ``` + ```bash + export VM_NAME="myVM" + az vm create --name $VM_NAME --resource-group myResourceGroup --image UbuntuLTS + ``` ``` 3. **Azure Portal Custom Cloud Shell Constraints** @@ -77,17 +77,17 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters **Example of supported command:** ```markdown - ```bash - az group create --name myResourceGroup --location eastus - ``` + ```bash + az group create --name myResourceGroup --location eastus + ``` ``` **Example of potentially unsupported command:** ```markdown - ```bash - # This requires elevated Graph API permissions and would fail - az ad app create --display-name myApp --native-app - ``` + ```bash + # This requires elevated Graph API permissions and would fail + az ad app create --display-name myApp --native-app + ``` ``` This filter system ensures that you select documentation that can be effectively transformed into executable docs that provide value through automated deployment capabilities. @@ -162,7 +162,7 @@ Check if all prerequisites below are met before writing the Exec Doc. ***If any **Example:** ```markdown - # Quickstart: Deploy an Azure Kubernetes Service (AKS) cluster using Azure CLI + # Quickstart: Deploy an Azure Kubernetes Service (AKS) cluster using Azure CLI ``` ### Writing Requirements @@ -404,12 +404,12 @@ Check if all prerequisites below are met before writing the Exec Doc. ***If any **Deeplink Template:** ```markdown - [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://ms.portal.azure.com/#view/Microsoft_Azure_CloudNative/SubscriptionSelectionPage.ReactView/isLearnMode~/true/referer/docs/tutorialKey/) + [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://ms.portal.azure.com/#view/Microsoft_Azure_CloudNative/SubscriptionSelectionPage.ReactView/isLearnMode~/true/referer/docs/tutorialKey/) ``` **Deeplink for Example Exec Doc:** ```markdown - [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://ms.portal.azure.com/#view/Microsoft_Azure_CloudNative/SubscriptionSelectionPage.ReactView/isLearnMode~/true/referer/docs/tutorialKey/azure-docs%2farticles%2faks%2fquick-kubernetes-deploy-cli.md) + [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://ms.portal.azure.com/#view/Microsoft_Azure_CloudNative/SubscriptionSelectionPage.ReactView/isLearnMode~/true/referer/docs/tutorialKey/azure-docs%2farticles%2faks%2fquick-kubernetes-deploy-cli.md) ``` **Example of Button in Live Exec Doc:** From 06fe6dee001077923da7fad06833badaeaf285ca Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 00:45:01 -0700 Subject: [PATCH 18/87] updated documentation with technical constraints --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b6b9794c5..154b2cdb3 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters - `azurecli` - `azure-cli-interactive` - `azurecli-interactive` + - `terraform` **Example:** ```markdown From c8515406b07af3a84a421087d208de2dd35b9e20 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 00:46:51 -0700 Subject: [PATCH 19/87] updated documentation with technical constraints --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 154b2cdb3..151ed79d3 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,6 @@ These experiences utilize [Innovation Engine](https://github.com/Azure/Innovatio Not all documentation is suitable for conversion to Exec Docs. Use these filters to determine if a document can be effectively converted: -### Technical Constraints - 1. **Supported Code Block Types** - The document must contain code blocks using at least one of these types: - `bash` @@ -70,7 +68,7 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters - Commands running within the user's subscription scope - Standard service deployments (VMs, storage, networking) - - **Not supported without special configuration:** + - **Not supported currrently:** - Commands requiring elevated Microsoft Graph API permissions - Operations needing KeyVault special access - Cross-subscription or tenant-level operations @@ -83,7 +81,7 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters ``` ``` - **Example of potentially unsupported command:** + **Example of unsupported command:** ```markdown ```bash # This requires elevated Graph API permissions and would fail @@ -91,7 +89,7 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters ``` ``` -This filter system ensures that you select documentation that can be effectively transformed into executable docs that provide value through automated deployment capabilities. +This filter system ensures that you select documentation that can be effectively transformed into executable docs that provide value through automated deployment capabilities. Please reach out to the [Exec Docs Team](#contact-information-for-exec-docs) if you have any questions about the suitability of a document for conversion to an Exec Doc. ## How to Write an Exec Doc From 105c55b92a8139040d12f82d40acfbd71ae146c3 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 00:47:36 -0700 Subject: [PATCH 20/87] updated documentation with technical constraints --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 151ed79d3..77e0692bc 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,13 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters ``` ``` + **Example of unsupported command:** + ```markdown + ```sql + SELECT * FROM myTable WHERE id = 1; + ``` + ``` + 3. **Azure Portal Custom Cloud Shell Constraints** - **Supported scenarios:** - Standard Azure resource operations (create, read, update, delete) From f3535316701e4238d5d57fe7cb23cb773dd413a3 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 10:47:37 -0700 Subject: [PATCH 21/87] updated docs --- tools/api_spec.md | 620 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 620 insertions(+) create mode 100644 tools/api_spec.md diff --git a/tools/api_spec.md b/tools/api_spec.md new file mode 100644 index 000000000..8628734cc --- /dev/null +++ b/tools/api_spec.md @@ -0,0 +1,620 @@ +# AI Documentation Assistant (ADA) REST API Specification + +## 1. Introduction + +This document provides the technical specifications for the AI Documentation Assistant (ADA) REST API. ADA enables users to generate, test, and validate executable documentation for Infrastructure as Code (IaC) deployments, focusing primarily on Linux and cloud native workloads. + +## 2. API Design Principles + +- **REST Architectural Style**: The API follows standard REST principles with resource-based URLs, appropriate HTTP methods, and stateless interactions +- **JSON**: All API requests and responses use JSON format +- **Authentication**: OAuth 2.0 integration with Azure AD +- **Performance**: Target response times under 2 seconds for document generation requests +- **Scalability**: Support for horizontal scaling to handle varying loads + +## 3. Base URL + +``` +https://ada.azure.com/api/v1 +``` + +## 4. Authentication and Authorization + +The API requires authentication for all requests using OAuth 2.0 with Azure Active Directory. + +**Headers**: +``` +Authorization: Bearer {token} +``` + +## 5. Resources and Endpoints + +### 5.1 Documents + +#### Create Document +``` +POST /documents +``` + +**Request Body**: +```json +{ + "title": "string", + "description": "string", + "prompt": "string", + "targetEnvironment": "string", // e.g., "azure", "aws", "local" + "infrastructureType": "string", // e.g., "terraform", "azcli", "bash" + "tags": ["string"], + "customizationParameters": { + "key": "value" + }, + "sourceDocument": "string", // Optional: Original markdown to convert + "sourceType": "string" // "prompt", "markdown", "script" +} +``` + +**Response** (201 Created): +```json +{ + "id": "string", + "title": "string", + "description": "string", + "content": "string", // Generated executable documentation + "createdAt": "string", + "status": "string", // "draft", "validated", "failed" + "_links": { + "self": {"href": "string"}, + "validate": {"href": "string"}, + "execute": {"href": "string"} + } +} +``` + +#### Get Document +``` +GET /documents/{id} +``` + +**Response** (200 OK): +```json +{ + "id": "string", + "title": "string", + "description": "string", + "content": "string", + "createdAt": "string", + "updatedAt": "string", + "status": "string", + "validationResult": { + "status": "string", + "details": "string", + "timestamp": "string" + }, + "dependencyFiles": [ + { + "filename": "string", + "content": "string", + "type": "string" + } + ], + "_links": { + "self": {"href": "string"}, + "validate": {"href": "string"}, + "execute": {"href": "string"}, + "revisions": {"href": "string"}, + "dependencies": {"href": "string"} + } +} +``` + +#### Update Document +``` +PUT /documents/{id} +``` + +**Request Body**: +```json +{ + "title": "string", + "description": "string", + "content": "string", + "tags": ["string"] +} +``` + +**Response** (200 OK): Same as GET response + +#### List Documents +``` +GET /documents +``` + +**Query Parameters**: +- `status` - Filter by validation status +- `tag` - Filter by tag +- `infrastructureType` - Filter by type +- `page` - Pagination page number +- `pageSize` - Items per page + +**Response** (200 OK): +```json +{ + "items": [ + { + "id": "string", + "title": "string", + "description": "string", + "status": "string", + "createdAt": "string", + "updatedAt": "string", + "_links": { + "self": {"href": "string"} + } + } + ], + "pagination": { + "totalItems": "number", + "totalPages": "number", + "currentPage": "number", + "pageSize": "number" + } +} +``` + +#### Delete Document +``` +DELETE /documents/{id} +``` + +**Response** (204 No Content) + +### 5.2 Validation and Testing + +#### Validate Document +``` +POST /documents/{id}/validate +``` + +**Request Body**: +```json +{ + "environmentParameters": { + "key": "value" + }, + "validateOnly": "boolean", // True for syntax check only, false for full execution test + "maxAttempts": "number", // Max number of auto-correction attempts (default: 3) + "timeoutSeconds": "number" // Execution timeout in seconds (default: 600) +} +``` + +**Response** (200 OK): +```json +{ + "id": "string", + "status": "string", // "in_progress", "success", "failed", "timed_out" + "details": "string", + "attempts": "number", // Number of attempts made + "validationSteps": [ + { + "step": "string", + "status": "string", + "output": "string", + "timestamp": "string", + "errorDetails": "string" + } + ], + "_links": { + "status": {"href": "string"}, + "document": {"href": "string"} + } +} +``` + +#### Get Validation Status +``` +GET /documents/{id}/validations/{validationId} +``` + +**Response** (200 OK): Same as validate response + +### 5.3 AI-Assisted Generation and Customization + +#### Generate Document from Prompt +``` +POST /ai/generate +``` + +**Request Body**: +```json +{ + "prompt": "string", // User's description of desired infrastructure + "targetEnvironment": "string", + "infrastructureType": "string", // "terraform", "azcli", "bash" + "expertiseLevel": "string", // "beginner", "intermediate", "expert" + "additionalContext": "string", + "sourceType": "string", // "prompt", "markdown", "script" + "sourceContent": "string" // Original content for conversion +} +``` + +**Response** (202 Accepted): +```json +{ + "requestId": "string", + "estimatedCompletionTime": "string", + "_links": { + "status": {"href": "string"} + } +} +``` + +#### Get Generation Status +``` +GET /ai/generate/{requestId} +``` + +**Response** (200 OK): +```json +{ + "status": "string", // "processing", "completed", "failed" + "progress": "number", // 0-100 + "document": { + // Document object if completed + }, + "error": "string" // If failed +} +``` + +#### AI-Assisted Document Repair +``` +POST /documents/{id}/repair +``` + +**Request Body**: +```json +{ + "validationErrors": ["string"], + "userGuidance": "string" +} +``` + +**Response** (200 OK): +```json +{ + "repairSuggestions": [ + { + "description": "string", + "modifiedContent": "string", + "confidence": "number" + } + ], + "_links": { + "apply": {"href": "string"}, + "document": {"href": "string"} + } +} +``` + +### 5.4 Dependency Files Management + +#### List Dependency Files +``` +GET /documents/{id}/dependencies +``` + +**Response** (200 OK): +```json +{ + "dependencies": [ + { + "filename": "string", + "type": "string", // "json", "yaml", "terraform", "shell", etc. + "content": "string" + } + ] +} +``` + +#### Create or Update Dependency File +``` +PUT /documents/{id}/dependencies/{filename} +``` + +**Request Body**: +```json +{ + "content": "string", + "type": "string" // "json", "yaml", "terraform", "shell", etc. +} +``` + +**Response** (200 OK): +```json +{ + "filename": "string", + "type": "string", + "content": "string", + "createdAt": "string", + "updatedAt": "string" +} +``` + +#### Generate Dependency Files +``` +POST /documents/{id}/dependencies/generate +``` + +**Response** (200 OK): +```json +{ + "generatedFiles": [ + { + "filename": "string", + "type": "string", + "content": "string" + } + ], + "documentUpdated": "boolean" +} +``` + +### 5.5 Security and Privacy + +#### Redact PII +``` +POST /documents/{id}/redact +``` + +**Request Body**: +```json +{ + "redactionLevel": "string" // "minimal", "standard", "strict" +} +``` + +**Response** (200 OK): +```json +{ + "id": "string", + "redactedContent": "string", + "redactionCount": "number", + "redactedTypes": ["string"] // Types of PII found and redacted +} +``` + +#### Security Analysis +``` +POST /documents/{id}/security-analysis +``` + +**Request Body**: +```json +{ + "analysisLevel": "string" // "basic", "standard", "comprehensive" +} +``` + +**Response** (202 Accepted): +```json +{ + "analysisId": "string", + "_links": { + "status": {"href": "string"} + } +} +``` + +#### Get Security Analysis Results +``` +GET /documents/{id}/security-analysis/{analysisId} +``` + +**Response** (200 OK): +```json +{ + "status": "string", // "in_progress", "completed", "failed" + "findings": [ + { + "severity": "string", // "critical", "high", "medium", "low" + "category": "string", + "description": "string", + "recommendation": "string", + "location": "string" // Location in document + } + ], + "summary": { + "criticalCount": "number", + "highCount": "number", + "mediumCount": "number", + "lowCount": "number" + } +} +``` + +## 6. Error Handling + +The API uses standard HTTP status codes and includes detailed error information in responses: + +```json +{ + "error": { + "code": "string", + "message": "string", + "details": [ + { + "code": "string", + "target": "string", + "message": "string" + } + ] + } +} +``` + +**Common Error Codes**: +- 400 Bad Request: Invalid input parameters +- 401 Unauthorized: Missing or invalid authentication +- 403 Forbidden: Insufficient permissions +- 404 Not Found: Resource not found +- 429 Too Many Requests: Rate limit exceeded +- 500 Internal Server Error: Server error + +## 7. Rate Limiting and Quotas + +- Rate limiting implemented with token bucket algorithm +- Default limits: + - 60 requests per minute per authenticated user + - 10 AI generation requests per hour per user + - 5 concurrent validation processes per user + +**Headers**: +``` +X-RateLimit-Limit: {limit} +X-RateLimit-Remaining: {remaining} +X-RateLimit-Reset: {reset_time} +``` + +## 8. Versioning Strategy + +- API versioning in URL path (/api/v1) +- Major version increments for breaking changes +- Support for at least one previous major version after a new version is released +- Deprecation notices provided 6 months before endpoint removal + +## 9. Security Considerations + +- Data Protection: + - All data encrypted in transit (TLS 1.3) + - Secrets and credentials never stored in generated documents + - Content scanning for sensitive information before storage + - Automatic PII redaction in result blocks and outputs + +- Access Controls: + - RBAC with Azure AD integration + - IP restrictions available for enterprise customers + - Audit logging for all API operations + +## 10. Integration Requirements + +### 10.1 Innovation Engine Integration + +The API must integrate with the Innovation Engine for document validation and execution: + +- Support for passing documents to Innovation Engine for testing +- Ability to receive and process validation results +- Support for debugging information when validation fails +- Iterative correction based on test failures + +### 10.2 LLM Integration + +- RAG implementation with weighting toward tested Executable Documents +- Capability to customize generation based on user expertise level +- Support for prompt engineering to improve generation quality +- Multi-turn conversations for iterative document improvement + +## 11. Monitoring and Observability + +The API should expose the following metrics: + +- Request latency +- Success/failure rates by endpoint +- Document generation success rates +- Validation success rates +- User adoption metrics +- Error frequency by type +- Validation attempts per document +- Common error patterns + +## 12. Implementation Roadmap + +1. **Phase 1 (3 months)**: + - Core CRUD operations for documents + - Basic validation capabilities + - OAuth authentication + - Terminal-based reference implementation + +2. **Phase 2 (6 months)**: + - AI-assisted document generation + - Integration with at least one partner UX (likely Azure Portal) + - Enhanced validation with detailed error reporting + - Dependency file management + +3. **Phase 3 (12 months)**: + - Full Copilot integration as an agent + - Self-healing document capabilities + - Support for additional IaC tools beyond Terraform and Azure CLI + - Security analysis and PII redaction + +## 13. Development Guidelines + +### 13.1 Technology Stack Recommendations + +- Backend: .NET Core or Node.js with TypeScript +- Database: Azure Cosmos DB (for document storage) +- Authentication: Azure AD OAuth 2.0 +- LLM: Azure OpenAI Service with custom RAG implementation +- Testing: Integration with Azure Innovation Engine + +### 13.2 Development Process + +- API-first development approach with OpenAPI/Swagger specifications +- CI/CD pipeline with automated testing +- Feature flags for gradual rollout of capabilities +- Comprehensive API documentation in Microsoft Learn + +## 14. Appendix + +### 14.1 Example Document Structure + +```markdown +# Deploy a Highly Available Web Application on Azure with Terraform + +This document will guide you through deploying a highly available web application +infrastructure on Azure using Terraform. + +## Prerequisites +- Azure CLI installed and configured +- Terraform v1.0+ installed +- Basic understanding of Terraform and Azure resources + +## Step 1: Configure Azure Provider + +```terraform +provider "azurerm" { + features {} +} + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + } +} +``` + +## Step 2: Create Resource Group + +```terraform +resource "azurerm_resource_group" "web_app_rg" { + name = "web-app-resources" + location = "East US" +} +``` + +# Additional steps would follow... +``` + +### 14.2 Recommended Testing Approaches + +- Unit tests for all API endpoints +- Integration tests with Innovation Engine +- Performance testing under load +- Security scanning for generated content +### 14.2 Recommended Testing Approaches + +- Unit tests for all API endpoints +- Integration tests with Innovation Engine +- Performance testing under load +- Security scanning for generated content \ No newline at end of file From 8a3a602485c448909d280b8b3a3316b2b763eab9 Mon Sep 17 00:00:00 2001 From: naman-msft <146123940+naman-msft@users.noreply.github.com> Date: Mon, 10 Mar 2025 12:03:22 -0700 Subject: [PATCH 22/87] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 77e0692bc..cc6cba4ce 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters >**Note:** While Innovation Engine can _parse_ a code block of any type, given its current features, it can only _execute_ code blocks of the types above. So, it is important to ensure that the code blocks in your Exec Doc are of the types above. 2. **Command Execution Limitations** - - **Not supported for direct execution (unless executed via bash):** + - **Not supported for direct execution:** - PowerShell scripts - Python, Ruby, or Node.js code - SQL commands From fe0baabf123c190a268902a91a5effc2478f92f4 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 14:50:33 -0700 Subject: [PATCH 23/87] updated docs --- tools/api_spec.md | 620 ---------------------------------------------- 1 file changed, 620 deletions(-) delete mode 100644 tools/api_spec.md diff --git a/tools/api_spec.md b/tools/api_spec.md deleted file mode 100644 index 8628734cc..000000000 --- a/tools/api_spec.md +++ /dev/null @@ -1,620 +0,0 @@ -# AI Documentation Assistant (ADA) REST API Specification - -## 1. Introduction - -This document provides the technical specifications for the AI Documentation Assistant (ADA) REST API. ADA enables users to generate, test, and validate executable documentation for Infrastructure as Code (IaC) deployments, focusing primarily on Linux and cloud native workloads. - -## 2. API Design Principles - -- **REST Architectural Style**: The API follows standard REST principles with resource-based URLs, appropriate HTTP methods, and stateless interactions -- **JSON**: All API requests and responses use JSON format -- **Authentication**: OAuth 2.0 integration with Azure AD -- **Performance**: Target response times under 2 seconds for document generation requests -- **Scalability**: Support for horizontal scaling to handle varying loads - -## 3. Base URL - -``` -https://ada.azure.com/api/v1 -``` - -## 4. Authentication and Authorization - -The API requires authentication for all requests using OAuth 2.0 with Azure Active Directory. - -**Headers**: -``` -Authorization: Bearer {token} -``` - -## 5. Resources and Endpoints - -### 5.1 Documents - -#### Create Document -``` -POST /documents -``` - -**Request Body**: -```json -{ - "title": "string", - "description": "string", - "prompt": "string", - "targetEnvironment": "string", // e.g., "azure", "aws", "local" - "infrastructureType": "string", // e.g., "terraform", "azcli", "bash" - "tags": ["string"], - "customizationParameters": { - "key": "value" - }, - "sourceDocument": "string", // Optional: Original markdown to convert - "sourceType": "string" // "prompt", "markdown", "script" -} -``` - -**Response** (201 Created): -```json -{ - "id": "string", - "title": "string", - "description": "string", - "content": "string", // Generated executable documentation - "createdAt": "string", - "status": "string", // "draft", "validated", "failed" - "_links": { - "self": {"href": "string"}, - "validate": {"href": "string"}, - "execute": {"href": "string"} - } -} -``` - -#### Get Document -``` -GET /documents/{id} -``` - -**Response** (200 OK): -```json -{ - "id": "string", - "title": "string", - "description": "string", - "content": "string", - "createdAt": "string", - "updatedAt": "string", - "status": "string", - "validationResult": { - "status": "string", - "details": "string", - "timestamp": "string" - }, - "dependencyFiles": [ - { - "filename": "string", - "content": "string", - "type": "string" - } - ], - "_links": { - "self": {"href": "string"}, - "validate": {"href": "string"}, - "execute": {"href": "string"}, - "revisions": {"href": "string"}, - "dependencies": {"href": "string"} - } -} -``` - -#### Update Document -``` -PUT /documents/{id} -``` - -**Request Body**: -```json -{ - "title": "string", - "description": "string", - "content": "string", - "tags": ["string"] -} -``` - -**Response** (200 OK): Same as GET response - -#### List Documents -``` -GET /documents -``` - -**Query Parameters**: -- `status` - Filter by validation status -- `tag` - Filter by tag -- `infrastructureType` - Filter by type -- `page` - Pagination page number -- `pageSize` - Items per page - -**Response** (200 OK): -```json -{ - "items": [ - { - "id": "string", - "title": "string", - "description": "string", - "status": "string", - "createdAt": "string", - "updatedAt": "string", - "_links": { - "self": {"href": "string"} - } - } - ], - "pagination": { - "totalItems": "number", - "totalPages": "number", - "currentPage": "number", - "pageSize": "number" - } -} -``` - -#### Delete Document -``` -DELETE /documents/{id} -``` - -**Response** (204 No Content) - -### 5.2 Validation and Testing - -#### Validate Document -``` -POST /documents/{id}/validate -``` - -**Request Body**: -```json -{ - "environmentParameters": { - "key": "value" - }, - "validateOnly": "boolean", // True for syntax check only, false for full execution test - "maxAttempts": "number", // Max number of auto-correction attempts (default: 3) - "timeoutSeconds": "number" // Execution timeout in seconds (default: 600) -} -``` - -**Response** (200 OK): -```json -{ - "id": "string", - "status": "string", // "in_progress", "success", "failed", "timed_out" - "details": "string", - "attempts": "number", // Number of attempts made - "validationSteps": [ - { - "step": "string", - "status": "string", - "output": "string", - "timestamp": "string", - "errorDetails": "string" - } - ], - "_links": { - "status": {"href": "string"}, - "document": {"href": "string"} - } -} -``` - -#### Get Validation Status -``` -GET /documents/{id}/validations/{validationId} -``` - -**Response** (200 OK): Same as validate response - -### 5.3 AI-Assisted Generation and Customization - -#### Generate Document from Prompt -``` -POST /ai/generate -``` - -**Request Body**: -```json -{ - "prompt": "string", // User's description of desired infrastructure - "targetEnvironment": "string", - "infrastructureType": "string", // "terraform", "azcli", "bash" - "expertiseLevel": "string", // "beginner", "intermediate", "expert" - "additionalContext": "string", - "sourceType": "string", // "prompt", "markdown", "script" - "sourceContent": "string" // Original content for conversion -} -``` - -**Response** (202 Accepted): -```json -{ - "requestId": "string", - "estimatedCompletionTime": "string", - "_links": { - "status": {"href": "string"} - } -} -``` - -#### Get Generation Status -``` -GET /ai/generate/{requestId} -``` - -**Response** (200 OK): -```json -{ - "status": "string", // "processing", "completed", "failed" - "progress": "number", // 0-100 - "document": { - // Document object if completed - }, - "error": "string" // If failed -} -``` - -#### AI-Assisted Document Repair -``` -POST /documents/{id}/repair -``` - -**Request Body**: -```json -{ - "validationErrors": ["string"], - "userGuidance": "string" -} -``` - -**Response** (200 OK): -```json -{ - "repairSuggestions": [ - { - "description": "string", - "modifiedContent": "string", - "confidence": "number" - } - ], - "_links": { - "apply": {"href": "string"}, - "document": {"href": "string"} - } -} -``` - -### 5.4 Dependency Files Management - -#### List Dependency Files -``` -GET /documents/{id}/dependencies -``` - -**Response** (200 OK): -```json -{ - "dependencies": [ - { - "filename": "string", - "type": "string", // "json", "yaml", "terraform", "shell", etc. - "content": "string" - } - ] -} -``` - -#### Create or Update Dependency File -``` -PUT /documents/{id}/dependencies/{filename} -``` - -**Request Body**: -```json -{ - "content": "string", - "type": "string" // "json", "yaml", "terraform", "shell", etc. -} -``` - -**Response** (200 OK): -```json -{ - "filename": "string", - "type": "string", - "content": "string", - "createdAt": "string", - "updatedAt": "string" -} -``` - -#### Generate Dependency Files -``` -POST /documents/{id}/dependencies/generate -``` - -**Response** (200 OK): -```json -{ - "generatedFiles": [ - { - "filename": "string", - "type": "string", - "content": "string" - } - ], - "documentUpdated": "boolean" -} -``` - -### 5.5 Security and Privacy - -#### Redact PII -``` -POST /documents/{id}/redact -``` - -**Request Body**: -```json -{ - "redactionLevel": "string" // "minimal", "standard", "strict" -} -``` - -**Response** (200 OK): -```json -{ - "id": "string", - "redactedContent": "string", - "redactionCount": "number", - "redactedTypes": ["string"] // Types of PII found and redacted -} -``` - -#### Security Analysis -``` -POST /documents/{id}/security-analysis -``` - -**Request Body**: -```json -{ - "analysisLevel": "string" // "basic", "standard", "comprehensive" -} -``` - -**Response** (202 Accepted): -```json -{ - "analysisId": "string", - "_links": { - "status": {"href": "string"} - } -} -``` - -#### Get Security Analysis Results -``` -GET /documents/{id}/security-analysis/{analysisId} -``` - -**Response** (200 OK): -```json -{ - "status": "string", // "in_progress", "completed", "failed" - "findings": [ - { - "severity": "string", // "critical", "high", "medium", "low" - "category": "string", - "description": "string", - "recommendation": "string", - "location": "string" // Location in document - } - ], - "summary": { - "criticalCount": "number", - "highCount": "number", - "mediumCount": "number", - "lowCount": "number" - } -} -``` - -## 6. Error Handling - -The API uses standard HTTP status codes and includes detailed error information in responses: - -```json -{ - "error": { - "code": "string", - "message": "string", - "details": [ - { - "code": "string", - "target": "string", - "message": "string" - } - ] - } -} -``` - -**Common Error Codes**: -- 400 Bad Request: Invalid input parameters -- 401 Unauthorized: Missing or invalid authentication -- 403 Forbidden: Insufficient permissions -- 404 Not Found: Resource not found -- 429 Too Many Requests: Rate limit exceeded -- 500 Internal Server Error: Server error - -## 7. Rate Limiting and Quotas - -- Rate limiting implemented with token bucket algorithm -- Default limits: - - 60 requests per minute per authenticated user - - 10 AI generation requests per hour per user - - 5 concurrent validation processes per user - -**Headers**: -``` -X-RateLimit-Limit: {limit} -X-RateLimit-Remaining: {remaining} -X-RateLimit-Reset: {reset_time} -``` - -## 8. Versioning Strategy - -- API versioning in URL path (/api/v1) -- Major version increments for breaking changes -- Support for at least one previous major version after a new version is released -- Deprecation notices provided 6 months before endpoint removal - -## 9. Security Considerations - -- Data Protection: - - All data encrypted in transit (TLS 1.3) - - Secrets and credentials never stored in generated documents - - Content scanning for sensitive information before storage - - Automatic PII redaction in result blocks and outputs - -- Access Controls: - - RBAC with Azure AD integration - - IP restrictions available for enterprise customers - - Audit logging for all API operations - -## 10. Integration Requirements - -### 10.1 Innovation Engine Integration - -The API must integrate with the Innovation Engine for document validation and execution: - -- Support for passing documents to Innovation Engine for testing -- Ability to receive and process validation results -- Support for debugging information when validation fails -- Iterative correction based on test failures - -### 10.2 LLM Integration - -- RAG implementation with weighting toward tested Executable Documents -- Capability to customize generation based on user expertise level -- Support for prompt engineering to improve generation quality -- Multi-turn conversations for iterative document improvement - -## 11. Monitoring and Observability - -The API should expose the following metrics: - -- Request latency -- Success/failure rates by endpoint -- Document generation success rates -- Validation success rates -- User adoption metrics -- Error frequency by type -- Validation attempts per document -- Common error patterns - -## 12. Implementation Roadmap - -1. **Phase 1 (3 months)**: - - Core CRUD operations for documents - - Basic validation capabilities - - OAuth authentication - - Terminal-based reference implementation - -2. **Phase 2 (6 months)**: - - AI-assisted document generation - - Integration with at least one partner UX (likely Azure Portal) - - Enhanced validation with detailed error reporting - - Dependency file management - -3. **Phase 3 (12 months)**: - - Full Copilot integration as an agent - - Self-healing document capabilities - - Support for additional IaC tools beyond Terraform and Azure CLI - - Security analysis and PII redaction - -## 13. Development Guidelines - -### 13.1 Technology Stack Recommendations - -- Backend: .NET Core or Node.js with TypeScript -- Database: Azure Cosmos DB (for document storage) -- Authentication: Azure AD OAuth 2.0 -- LLM: Azure OpenAI Service with custom RAG implementation -- Testing: Integration with Azure Innovation Engine - -### 13.2 Development Process - -- API-first development approach with OpenAPI/Swagger specifications -- CI/CD pipeline with automated testing -- Feature flags for gradual rollout of capabilities -- Comprehensive API documentation in Microsoft Learn - -## 14. Appendix - -### 14.1 Example Document Structure - -```markdown -# Deploy a Highly Available Web Application on Azure with Terraform - -This document will guide you through deploying a highly available web application -infrastructure on Azure using Terraform. - -## Prerequisites -- Azure CLI installed and configured -- Terraform v1.0+ installed -- Basic understanding of Terraform and Azure resources - -## Step 1: Configure Azure Provider - -```terraform -provider "azurerm" { - features {} -} - -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~>3.0" - } - } -} -``` - -## Step 2: Create Resource Group - -```terraform -resource "azurerm_resource_group" "web_app_rg" { - name = "web-app-resources" - location = "East US" -} -``` - -# Additional steps would follow... -``` - -### 14.2 Recommended Testing Approaches - -- Unit tests for all API endpoints -- Integration tests with Innovation Engine -- Performance testing under load -- Security scanning for generated content -### 14.2 Recommended Testing Approaches - -- Unit tests for all API endpoints -- Integration tests with Innovation Engine -- Performance testing under load -- Security scanning for generated content \ No newline at end of file From a19398b57104cb3bbb1d3d7287487fde007608f3 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 10 Mar 2025 14:54:25 -0700 Subject: [PATCH 24/87] updated docs --- LICENSE-CODE | 17 - ... on AKS with CloudNativePG_ai_generated.md | 156 ---- ...y Linux VM with SSH Access_ai_generated.md | 108 --- tools/aks.sh | 45 - tools/aks_documented.md | 144 ---- tools/convert_converted.md | 387 --------- tools/mongodb.md | 816 ------------------ tools/mongodb_redacted.md | 815 ----------------- tools/mongodb_security_report.md | 90 -- tools/myazure_rm.yml | 4 - tools/myazure_rm.yml (initial version) | 4 - .../myazure_rm.yml (with conditional_groups) | 7 - tools/myazure_rm.yml (with keyed_groups) | 9 - tools/ping.yml | 9 - tools/win_ping.yml | 11 - 15 files changed, 2622 deletions(-) delete mode 100644 LICENSE-CODE delete mode 100644 tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md delete mode 100644 tools/Deploy Linux VM with SSH Access_ai_generated.md delete mode 100644 tools/aks.sh delete mode 100644 tools/aks_documented.md delete mode 100644 tools/convert_converted.md delete mode 100644 tools/mongodb.md delete mode 100644 tools/mongodb_redacted.md delete mode 100644 tools/mongodb_security_report.md delete mode 100644 tools/myazure_rm.yml delete mode 100644 tools/myazure_rm.yml (initial version) delete mode 100644 tools/myazure_rm.yml (with conditional_groups) delete mode 100644 tools/myazure_rm.yml (with keyed_groups) delete mode 100644 tools/ping.yml delete mode 100644 tools/win_ping.yml diff --git a/LICENSE-CODE b/LICENSE-CODE deleted file mode 100644 index b17b032a4..000000000 --- a/LICENSE-CODE +++ /dev/null @@ -1,17 +0,0 @@ -The MIT License (MIT) -Copyright (c) Microsoft Corporation - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -associated documentation files (the "Software"), to deal in the Software without restriction, -including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial -portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT -NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md b/tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md deleted file mode 100644 index 0c15a1931..000000000 --- a/tools/Deploy Highly Available PostgreSQL on AKS with CloudNativePG_ai_generated.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -title: Deploy a Highly Available PostgreSQL Database on AKS using CloudNativePG Operator -description: This Exec Doc demonstrates how to deploy a highly available PostgreSQL database on an Azure Kubernetes Service (AKS) cluster using the CloudNativePG operator. It covers creating the necessary Azure resources, installing the operator via Helm, and deploying a multi-instance PostgreSQL cluster. -ms.topic: quickstart -ms.date: 10/12/2023 -author: yourgithubusername -ms.author: youralias -ms.custom: innovation-engine, akshighavailability, cloudnativepg ---- - -# Deploy a Highly Available PostgreSQL Database on AKS using CloudNativePG Operator - -This document guides you through deploying a highly available PostgreSQL database on an AKS cluster using the CloudNativePG operator. You will create an Azure resource group and an AKS cluster with a random suffix for uniqueness, install the CloudNativePG operator using Helm, and then deploy a PostgreSQL cluster configured for high availability. - -The following steps include environment variable declarations, Azure CLI commands, and Kubernetes commands executed via bash code blocks. Each code block includes an accompanying result block to verify that the commands execute with the expected output. - ---- - -## Step 1: Create an Azure Resource Group - -In this section, we declare environment variables for the deployment. The resource group name will have a random suffix appended to ensure uniqueness. We then create the resource group in the designated region (WestUS2). - -```bash -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export REGION="WestUS2" -export RESOURCE_GROUP="cnpg-rg$RANDOM_SUFFIX" -az group create --name $RESOURCE_GROUP --location $REGION -``` - -Results: - - - -```JSON -{ - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/cnpg-rgxxxxxxxxx", - "location": "WestUS2", - "name": "cnpg-rgxxxxxxxxx", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": {} -} -``` - ---- - -## Step 2: Create an AKS Cluster - -Now we create an AKS cluster in the resource group. The cluster name is also appended with a random suffix. This cluster will have 3 nodes to support deployment of a highly available PostgreSQL database. - -```bash -export AKS_CLUSTER="cnpg-aks$RANDOM_SUFFIX" -az aks create --resource-group $RESOURCE_GROUP --name $AKS_CLUSTER --node-count 3 --enable-addons monitoring --generate-ssh-keys --location $REGION -``` - -Results: - - - -```JSON -{ - "fqdn": "cnpg-aksxxxxxxxxx.hcp.westus2.azmk8s.io", - "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/cnpg-rgxxxxxxxxx/providers/Microsoft.ContainerService/managedClusters/cnpg-aksxxxxxxxxx", - "location": "WestUS2", - "name": "cnpg-aksxxxxxxxxx", - "provisioningState": "Succeeded", - "tags": {} -} -``` - -After creating the cluster, download its credentials so that kubectl can interact with it: - -```bash -az aks get-credentials --resource-group $RESOURCE_GROUP --name $AKS_CLUSTER -``` - -Results: - - - -```console -Merged "cnpg-aksxxxxxxxxx" as current context in /home/xxxxx/.kube/config -``` - ---- - -## Step 3: Install the CloudNativePG Operator - -The CloudNativePG operator is installed via Helm. This section adds the CloudNativePG Helm repository and deploys the operator into its own namespace (cnpg-system). - -```bash -helm repo add cloudnative-pg https://cloudnative-pg.io/charts -helm repo update -helm install cnpg cloudnative-pg/cnpg --namespace cnpg-system --create-namespace -``` - -Results: - - - -```console -NAME: cnpg -LAST DEPLOYED: Wed Oct 11 2023 12:34:56 PM -NAMESPACE: cnpg-system -STATUS: deployed -REVISION: 1 -``` - ---- - -## Step 4: Deploy a Highly Available PostgreSQL Cluster - -In this step, you'll deploy a PostgreSQL cluster using CloudNativePG. The configuration specifies three instances to achieve high availability, and a minimal storage allocation is used for demonstration purposes. - -First, create the PostgreSQL cluster manifest file named "ha-postgresql.yaml". This file should reside in the same folder as this Exec Doc. - -```bash -cat << 'EOF' > ha-postgresql.yaml -apiVersion: postgresql.cnpg.io/v1 -kind: Cluster -metadata: - name: ha-postgres -spec: - instances: 3 - storage: - size: 1Gi - postgresVersion: 14 -EOF -``` - -Results: - - - -```console -ha-postgresql.yaml created -``` - -Now, apply the YAML file to deploy the PostgreSQL cluster. - -```bash -kubectl apply -f ha-postgresql.yaml -``` - -Results: - - - -```console -cluster.postgresql.cnpg.io/ha-postgres created -``` - ---- - -In this Exec Doc, you've created an Azure resource group and an AKS cluster, installed the CloudNativePG operator using Helm, and deployed a highly available PostgreSQL database on the cluster using a custom YAML manifest. This automated, one-click deployment is repeatable and ensures that the resources are unique for every run. \ No newline at end of file diff --git a/tools/Deploy Linux VM with SSH Access_ai_generated.md b/tools/Deploy Linux VM with SSH Access_ai_generated.md deleted file mode 100644 index 7994ce6a6..000000000 --- a/tools/Deploy Linux VM with SSH Access_ai_generated.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -title: 'Quickstart: Create a Linux VM and SSH into it' -description: Learn how to create a Linux virtual machine in Azure using Azure CLI and then SSH into it. -ms.topic: quickstart -ms.date: 10/12/2023 -author: yourgithubusername -ms.author: yourgithubusername -ms.custom: innovation-engine, azurecli, linux-related-content ---- - -# Quickstart: Create a Linux VM and SSH into it - -This Exec Doc demonstrates how to create a resource group, deploy a Linux VM using a supported Ubuntu image, retrieve its public IP address, and then SSH into the VM. The process uses environment variables to manage configuration details and appends a random suffix to resource names to ensure uniqueness. - -The following sections walk through each step with code blocks. Remember that you must already be logged in to Azure and have your subscription set. - -## Step 1: Create a Resource Group - -In this section, we declare environment variables necessary for the deployment and create a resource group in the "centralindia" region. A random suffix is appended to the resource group name to guarantee uniqueness. - -```bash -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export REGION="centralindia" -export RG_NAME="LinuxRG$RANDOM_SUFFIX" -az group create --name $RG_NAME --location $REGION -``` - -Results: - - -```JSON -{ - "id": "/subscriptions/xxxxx/resourceGroups/LinuxRGabc123", - "location": "centralindia", - "managedBy": null, - "name": "LinuxRGabc123", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" -} -``` - -## Step 2: Create a Linux Virtual Machine - -Now we create a Linux VM using a supported Ubuntu image ('Ubuntu2204'). In this example, we use a Standard_B1s VM size. We also set an administrator username and let Azure generate SSH key pairs automatically. A random suffix is used in the VM name for uniqueness. - -```bash -export VM_NAME="LinuxVM$RANDOM_SUFFIX" -export ADMIN_USERNAME="azureuser" -az vm create \ - --resource-group $RG_NAME \ - --name $VM_NAME \ - --image Ubuntu2204 \ - --size Standard_B1s \ - --admin-username $ADMIN_USERNAME \ - --generate-ssh-keys -``` - -Results: - - -```JSON -{ - "fqdns": "", - "id": "/subscriptions/xxxxx/resourceGroups/LinuxRGabc123/providers/Microsoft.Compute/virtualMachines/LinuxVMabc123", - "location": "centralindia", - "macAddress": "00-0X-0X-0X-0X-0X", - "machineId": "xxxxx", - "name": "LinuxVMabc123", - "powerState": "VM running", - "privateIpAddress": "10.0.0.4", - "publicIpAddress": "13.92.xxx.xxx", - "resourceGroup": "LinuxRGabc123", - "zones": "" -} -``` - -## Step 3: Retrieve the VM Public IP Address - -This step retrieves the public IP address of the newly created VM. The public IP is stored in an environment variable to be used in the SSH step. - -```bash -export VM_PUBLIC_IP=$(az vm list-ip-addresses --resource-group $RG_NAME --name $VM_NAME --query "[].virtualMachine.network.publicIpAddresses[0].ipAddress" --output tsv) -echo "The public IP address of the VM is: $VM_PUBLIC_IP" -``` - -Results: - - -```text -The public IP address of the VM is: 13.92.xxx.xxx -``` - -## Step 4: SSH into the Linux VM - -Finally, once you have retrieved the public IP address, you can SSH into your Linux VM using the generated SSH key pair. This command establishes an SSH connection without prompting for host key verification. - -```bash -ssh -o StrictHostKeyChecking=no $ADMIN_USERNAME@$VM_PUBLIC_IP -``` - -When executed, this command initiates an SSH session with your Linux VM. After connecting, you will have full access to manage and configure the VM as needed. - ---- - -This Exec Doc has successfully deployed a Linux VM in Azure using a supported Ubuntu image and shown how to connect to it using SSH, all accomplished with a series of Azure CLI commands executed via the Innovation Engine. \ No newline at end of file diff --git a/tools/aks.sh b/tools/aks.sh deleted file mode 100644 index 48719c724..000000000 --- a/tools/aks.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# This script creates an AKS cluster using Azure CLI - -# Exit on error -set -e - -# Configuration variables -RESOURCE_GROUP="myAKSResourceGroup" -LOCATION="eastus" -CLUSTER_NAME="myAKSCluster" -NODE_COUNT=3 -NODE_VM_SIZE="Standard_DS2_v2" -KUBERNETES_VERSION="1.26.3" # Check available versions with: az aks get-versions --location $LOCATION --output table - -# Login to Azure (uncomment if not already logged in) -# az login - -# Create resource group -echo "Creating resource group $RESOURCE_GROUP in $LOCATION..." -az group create --name $RESOURCE_GROUP --location $LOCATION - -# Create AKS cluster -echo "Creating AKS cluster $CLUSTER_NAME..." -az aks create \ - --resource-group $RESOURCE_GROUP \ - --name $CLUSTER_NAME \ - --node-count $NODE_COUNT \ - --node-vm-size $NODE_VM_SIZE \ - --kubernetes-version $KUBERNETES_VERSION \ - --generate-ssh-keys \ - --enable-managed-identity \ - --enable-cluster-autoscaler \ - --min-count 1 \ - --max-count 5 - -# Get credentials for the Kubernetes cluster -echo "Getting credentials for cluster $CLUSTER_NAME..." -az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME - -echo "AKS cluster $CLUSTER_NAME has been created successfully!" -echo "You can now use kubectl to manage your cluster" - -# Verify connection to the cluster -echo "Verifying connection to the cluster..." -kubectl get nodes \ No newline at end of file diff --git a/tools/aks_documented.md b/tools/aks_documented.md deleted file mode 100644 index 74fb71d55..000000000 --- a/tools/aks_documented.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -title: Explanation: AKS Cluster Creation Script -description: This Exec Doc explains a shell script that creates an AKS cluster using Azure CLI. The document walks you through each functional block to help you understand the purpose of the script and how each section contributes to the overall process. -ms.topic: article -ms.date: 2023-10-12 -author: chatgpt -ms.author: chatgpt -ms.custom: innovation-engine, ms-learn, azure, cluster-creation ---- - -# Explanation: AKS Cluster Creation Script - -In this Exec Doc, we examine a shell script that automates the process of creating an Azure Kubernetes Service (AKS) cluster. The script covers several key tasks: setting safe execution options, defining configuration variables, creating a resource group, deploying the AKS cluster, retrieving credentials, and finally verifying the cluster connectivity. Read on to understand the purpose and function of each block. - ---- - -## Script Header and Safety Settings - -Below the shebang line, the script uses `set -e` to ensure that the script exits immediately upon encountering any error. This helps prevent cascading failures during the deployment process. - -```bash -#!/bin/bash -# This script creates an AKS cluster using Azure CLI - -# Exit on error -set -e -``` - -The above code ensures that any failure in subsequent commands stops the script, thereby protecting against unintended side effects. - ---- - -## Configuration Variables - -This section defines the necessary configuration variables for the deployment. These variables include the resource group name, location, cluster name, node count, node VM size, and the Kubernetes version. The comments also guide you on how to check for available Kubernetes versions using the Azure CLI. - -```bash -# Configuration variables -RESOURCE_GROUP="myAKSResourceGroup" -LOCATION="eastus" -CLUSTER_NAME="myAKSCluster" -NODE_COUNT=3 -NODE_VM_SIZE="Standard_DS2_v2" -KUBERNETES_VERSION="1.26.3" # Check available versions with: az aks get-versions --location $LOCATION --output table -``` - -Each variable is critical for the subsequent commands that create and configure the AKS cluster. Note that these values are hardcoded; changing them will adjust the deployment accordingly. - ---- - -## (Optional) Azure Login Comment - -The script includes a commented-out Azure login command. This serves as a reminder to log in if you aren’t already authenticated. Since the Exec Doc guidelines do not allow login commands, the line remains commented out. - -```bash -# Login to Azure (uncomment if not already logged in) -# az login -``` - -This block is informational and does not affect the execution when the script is run in a pre-authenticated session. - ---- - -## Creating the Resource Group - -Before deploying the AKS cluster, the script creates a resource group in the specified location. This resource group will contain all the resources associated with the AKS cluster. - -```bash -# Create resource group -echo "Creating resource group $RESOURCE_GROUP in $LOCATION..." -az group create --name $RESOURCE_GROUP --location $LOCATION -``` - -The echo statement provides user feedback, while the `az group create` command creates the resource group if it does not already exist. - ---- - -## Deploying the AKS Cluster - -The next functional block involves the creation of the AKS cluster. The script uses several parameters to customize the deployment, such as node count, VM size, Kubernetes version, SSH key generation, managed identity, and autoscaling settings. - -```bash -# Create AKS cluster -echo "Creating AKS cluster $CLUSTER_NAME..." -az aks create \ - --resource-group $RESOURCE_GROUP \ - --name $CLUSTER_NAME \ - --node-count $NODE_COUNT \ - --node-vm-size $NODE_VM_SIZE \ - --kubernetes-version $KUBERNETES_VERSION \ - --generate-ssh-keys \ - --enable-managed-identity \ - --enable-cluster-autoscaler \ - --min-count 1 \ - --max-count 5 -``` - -This block deploys the AKS cluster with the defined specifications. It also enables cluster autoscaling between 1 and 5 nodes to adapt to workload demands. - ---- - -## Retrieving Cluster Credentials - -Once the AKS cluster is deployed, the script retrieves the cluster's credentials. This allows you to manage the Kubernetes cluster using the `kubectl` command-line tool. - -```bash -# Get credentials for the Kubernetes cluster -echo "Getting credentials for cluster $CLUSTER_NAME..." -az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME -``` - -The credentials command updates your local kubeconfig file, enabling seamless interaction with your cluster. - ---- - -## Final Confirmation and Cluster Verification - -After the credentials are fetched, the script prints success messages and then verifies the cluster connection by listing the cluster nodes using `kubectl`. - -```bash -echo "AKS cluster $CLUSTER_NAME has been created successfully!" -echo "You can now use kubectl to manage your cluster" - -# Verify connection to the cluster -echo "Verifying connection to the cluster..." -kubectl get nodes -``` - -This verification confirms that the cluster is operational and that the kubectl context is correctly set up. - -Results: - - - -```console -NAME STATUS ROLES AGE VERSION -aks-nodepool1-abcdef12-vmss000000 Ready agent 5m v1.26.3 -``` - -The above result block illustrates a typical output from `kubectl get nodes`, indicating that at least one node in the AKS cluster is ready and connected. - ---- - -This Exec Doc provides a short and sweet explanation of every major functional block in the AKS cluster creation script. By following the annotated steps, you gain a clearer understanding of how cloud resources are provisioned in a streamlined, automated manner. \ No newline at end of file diff --git a/tools/convert_converted.md b/tools/convert_converted.md deleted file mode 100644 index 8637dec59..000000000 --- a/tools/convert_converted.md +++ /dev/null @@ -1,387 +0,0 @@ ---- -title: Tutorial - Configure dynamic inventories for Azure Virtual Machines using Ansible -description: Learn how to populate your Ansible inventory dynamically from information in Azure -keywords: ansible, azure, devops, bash, cloudshell, dynamic inventory -ms.topic: tutorial -ms.date: 08/14/2024 -ms.custom: devx-track-ansible, devx-track-azurecli, devx-track-azurepowershell, linux-related-content -author: ansibleexpert -ms.author: ansibleexpert ---- - -# Tutorial: Configure dynamic inventories of your Azure resources using Ansible - -[!INCLUDE [ansible-28-note.md](includes/ansible-28-note.md)] - -Before you begin, ensure that your environment has Ansible installed. - -Set the following environment variables. These declarations ensure unique resource names and provide needed configuration so that the Exec Doc runs non-interactively. - -```bash -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export RESOURCE_GROUP="ansibleinventorytestrg${RANDOM_SUFFIX}" -export REGION="centralindia" -export ADMIN_PASSWORD="P@ssw0rd123!" -``` - -The [Ansible dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_dynamic_inventory.html) feature removes the burden of maintaining static inventory files. - -In this tutorial, you use Azure's dynamic-inventory plug-in to populate your Ansible inventory. - -In this article, you learn how to: - -> [!div class="checklist"] -> * Configure two test virtual machines. -> * Add tags to Azure virtual machines. -> * Generate a dynamic inventory. -> * Use conditional and keyed groups to populate group memberships. -> * Run playbooks against groups within the dynamic inventory. - -## Prerequisites - -[!INCLUDE [open-source-devops-prereqs-azure-subscription.md](../includes/open-source-devops-prereqs-azure-subscription.md)] -[!INCLUDE [open-source-devops-prereqs-create-service-principal.md](../includes/open-source-devops-prereqs-create-service-principal.md)] -[!INCLUDE [ansible-prereqs-cloudshell-use-or-vm-creation2.md](includes/ansible-prereqs-cloudshell-use-or-vm-creation2.md)] - -## Create Azure VMs - -1. Sign in to the [Azure portal](https://go.microsoft.com/fwlink/p/?LinkID=525040). - -2. Open [Cloud Shell](/azure/cloud-shell/overview). - -3. Create an Azure resource group to hold the virtual machines for this tutorial. - - > [!IMPORTANT] - > The Azure resource group you create in this step must have a name that is entirely lower-case. Otherwise, the generation of the dynamic inventory will fail. - - # [Azure CLI](#tab/azure-cli) - ```azurecli-interactive - az group create --resource-group $RESOURCE_GROUP --location $REGION - ``` - # [Azure PowerShell] - ```azurepowershell - New-AzResourceGroup -Name $RESOURCE_GROUP -Location $REGION - ``` - -4. Create two virtual machines on Azure using one of the following techniques: - - - **Ansible playbook** – The articles [Create a basic Linux virtual machine in Azure with Ansible](./vm-configure.md) and [Create a basic Windows virtual machine in Azure with Ansible](./vm-configure-windows.md) illustrate how to create a virtual machine from an Ansible playbook. - - - **Azure CLI** – Issue each of the following commands in Cloud Shell to create the two virtual machines. Note that the --size parameter is provided to avoid unavailable SKU errors. - - # [Azure CLI](#tab/azure-cli) - ```azurecli-interactive - az vm create \ - --resource-group $RESOURCE_GROUP \ - --name win-vm \ - --image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest \ - --size Standard_B1s \ - --admin-username azureuser \ - --admin-password $ADMIN_PASSWORD - - az vm create \ - --resource-group $RESOURCE_GROUP \ - --name linux-vm \ - --image Ubuntu2204 \ - --size Standard_B1s \ - --admin-username azureuser \ - --admin-password $ADMIN_PASSWORD - ``` - - # [Azure PowerShell] - ```azurepowershell - $adminUsername = "azureuser" - $adminPassword = ConvertTo-SecureString $env:ADMIN_PASSWORD -AsPlainText -Force - $credential = New-Object System.Management.Automation.PSCredential ($adminUsername, $adminPassword); - - New-AzVM ` - -ResourceGroupName $RESOURCE_GROUP ` - -Location $REGION ` - -Image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest ` - -Name win-vm ` - -Size Standard_B1s ` - -OpenPorts 3389 ` - -Credential $credential - - New-AzVM ` - -ResourceGroupName $RESOURCE_GROUP ` - -Location $REGION ` - -Image Ubuntu2204 ` - -Name linux-vm ` - -Size Standard_B1s ` - -OpenPorts 22 ` - -Credential $credential - ``` - (Replace any password placeholders with the variable ADMIN_PASSWORD.) - -## Add application role tags - -Tags are used to organize and categorize Azure resources. Assigning the Azure VMs an application role allows you to use the tags as group names within the Azure dynamic inventory. - -Run the following commands to update the VM tags: - -# [Azure CLI](#tab/azure-cli) -```azurecli-interactive -az vm update \ - --resource-group $RESOURCE_GROUP \ - --name linux-vm \ - --set tags.applicationRole='message-broker' - -az vm update \ - --resource-group $RESOURCE_GROUP \ - --name win-vm \ - --set tags.applicationRole='web-server' -``` - -# [Azure PowerShell] -```azurepowershell -Update-AzVM -VM (Get-AzVM -Name win-vm -ResourceGroupName $RESOURCE_GROUP) -Tag @{applicationRole="web-server"} -Update-AzVM -VM (Get-AzVM -Name linux-vm -ResourceGroupName $RESOURCE_GROUP) -Tag @{applicationRole="message-broker"} -``` - -Learn more about Azure tagging strategies at [Define your tagging strategy](/azure/cloud-adoption-framework/ready/azure-best-practices/resource-tagging). - -## Generate a dynamic inventory - -Ansible provides an [Azure dynamic-inventory plug-in](https://github.com/ansible/ansible/blob/stable-2.9/lib/ansible/plugins/inventory/azure_rm.py). - -The following steps walk you through using the plug-in: - -1. Create a dynamic inventory named "myazure_rm.yml" with the basic configuration. - -```bash -cat < myazure_rm.yml -plugin: azure_rm -include_vm_resource_groups: - - ${RESOURCE_GROUP} -auth_source: auto -EOF -``` - -2. Run the following command to query the VMs within the resource group. - -```bash -ansible-inventory -i myazure_rm.yml --graph -``` - -Results: - - -```text -@all: - |--@ungrouped: - |--linux-vm_abc123 - |--win-vm_def456 -``` - -## Find Azure VM hostvars - -Run the following command to view all the hostvars: - -```bash -ansible-inventory -i myazure_rm.yml --list -``` - -Results: - - -```json -{ - "_meta": { - "hostvars": { - "linux-vm_abc123": { - "ansible_host": "x.x.x.x" - }, - "win-vm_def456": { - "ansible_host": "x.x.x.x" - } - } - } -} -``` - -## Assign group membership with conditional_groups - -Each conditional group is made of two parts: the name of the group and the condition for adding a member to the group. - -Use the property image.offer to create conditional group membership for the linux-vm. - -Open the myazure_rm.yml dynamic inventory and update it to include the following conditional_groups section. This overwrites the previous file. - -```bash -cat < myazure_rm.yml -plugin: azure_rm -include_vm_resource_groups: - - ${RESOURCE_GROUP} -auth_source: auto -conditional_groups: - linux: "'ubuntu' in image.offer" - windows: "'WindowsServer' in image.offer" -EOF -``` - -Run the ansible-inventory command with the --graph option: - -```bash -ansible-inventory -i myazure_rm.yml --graph -``` - -Results: - - -```text -@all: - |--@linux: - |--linux-vm_abc123 - |--@ungrouped: - |--@windows: - |--win-vm_def456 -``` - -From the output, you can see the VMs are no longer associated with the "ungrouped" group. Instead, each VM is assigned to a new group created by the dynamic inventory. - -## Assign group membership with keyed_groups - -Keyed groups assign group membership in a similar manner as conditional groups, but the group name is dynamically populated based on the resource tag. - -Update the myazure_rm.yml dynamic inventory to include the keyed_groups section. This overwrites the previous file. - -```bash -cat < myazure_rm.yml -plugin: azure_rm -include_vm_resource_groups: - - ${RESOURCE_GROUP} -auth_source: auto -conditional_groups: - linux: "'ubuntu' in image.offer" - windows: "'WindowsServer' in image.offer" -keyed_groups: - - key: tags.applicationRole -EOF -``` - -Run the ansible-inventory command with the --graph option: - -```bash -ansible-inventory -i myazure_rm.yml --graph -``` - -Results: - - -```text -@all: - |--@_message_broker: - |--linux-vm_abc123 - |--@_web_server: - |--win-vm_def456 - |--@linux: - |--linux-vm_abc123 - |--@ungrouped: - |--@windows: - |--win-vm_def456 -``` - -From the output, you see two more groups _message_broker and _web_server. By using a keyed group, the applicationRole tag populates the group names and group memberships. - -## Run playbooks with group name patterns - -Use the groups created by the dynamic inventory to target subgroups. - -1. Create a playbook called win_ping.yml with the following contents. Predefined variables are provided so that no interactive prompts occur. - -```bash -cat < win_ping.yml ---- -- hosts: windows - gather_facts: false - - vars: - username: "azureuser" - password: "${ADMIN_PASSWORD}" - ansible_user: "{{ username }}" - ansible_password: "{{ password }}" - ansible_connection: winrm - ansible_winrm_transport: ntlm - ansible_winrm_server_cert_validation: ignore - - tasks: - - name: run win_ping - win_ping: -EOF -``` - -2. Run the win_ping.yml playbook. - -```bash -ansible-playbook win_ping.yml -i myazure_rm.yml -``` - -Results: - - -```text -PLAY [windows] ************************************************************* - -TASK [run win_ping] ******************************************************* -ok: [win-vm_def456] - -PLAY RECAP *************************************************************** -win-vm_def456 : ok=1 changed=0 unreachable=0 failed=0 -``` - -If you get the error "winrm or requests is not installed: No module named 'winrm'", install pywinrm with the following command: - -```bash -pip install "pywinrm>=0.3.0" -``` - -3. Create a second playbook named ping.yml with the following contents. Predefined variables are provided so that no interactive prompts occur. - -```bash -cat < ping.yml ---- -- hosts: all - gather_facts: false - - vars: - username: "azureuser" - password: "${ADMIN_PASSWORD}" - ansible_user: "{{ username }}" - ansible_password: "{{ password }}" - ansible_ssh_common_args: '-o StrictHostKeyChecking=no' - - tasks: - - name: run ping - ping: -EOF -``` - -4. Run the ping.yml playbook. - -```bash -ansible-playbook ping.yml -i myazure_rm.yml -``` - -Results: - - -```text -PLAY [all] ************************************************************* -TASK [run ping] ******************************************************* -ok: [linux-vm_abc123] - -PLAY RECAP ************************************************************* -linux-vm_abc123 : ok=1 changed=0 unreachable=0 failed=0 -``` - -## Clean up resources - -(Note: In Exec Docs, deletion commands that remove resources are omitted to prevent accidental deletion during automated execution.) - ---- - -## Next steps - -> [!div class="nextstepaction"] -> [Quickstart: Configure Linux virtual machines in Azure using Ansible](./vm-configure.md) \ No newline at end of file diff --git a/tools/mongodb.md b/tools/mongodb.md deleted file mode 100644 index a61019c32..000000000 --- a/tools/mongodb.md +++ /dev/null @@ -1,816 +0,0 @@ ---- -title: 'Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS)' -description: In this article, you configure and deploy a MongoDB cluster on AKS. -ms.topic: how-to -ms.date: 01/07/2025 -author: fossygirl -ms.author: carols -ms.custom: aks-related-content ---- - -# Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS) - -In this article, you configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS). - -## Configure a workload identity - -1. Create a namespace for the MongoDB cluster using the `kubectl create namespace` command. - - ```bash - kubectl create namespace ${AKS_MONGODB_NAMESPACE} --dry-run=client --output yaml | kubectl apply -f - - ``` - - Example output: - - - ```output - namespace/mongodb created - ``` - -2. Create a service account and configure a workload identity using the `kubectl apply` command. - - ```bash - export TENANT_ID=$(az account show --query tenantId -o tsv) - cat < - ```output - serviceaccount/mongodb created - ``` - -## Install External Secrets Operator - -In this section, you use Helm to install External Secrets Operator. External Secrets Operator is a Kubernetes operator that manages the life cycle of external secrets stored in external secret stores like Azure Key Vault. - -1. Add the External Secrets Helm repository and update the repository using the `helm repo add` and `helm repo update` commands. - - ```bash - helm repo add external-secrets https://charts.external-secrets.io - helm repo update - ``` - - Example output: - - - ```output - Hang tight while we grab the latest from your chart repositories... - ...Successfully got an update from the "external-secrets" chart repository - ``` - -2. Install External Secrets Operator using the `helm install` command. - - ```bash - helm install external-secrets \ - external-secrets/external-secrets \ - --namespace ${AKS_MONGODB_NAMESPACE} \ - --create-namespace \ - --set installCRDs=true \ - --wait \ - --set nodeSelector."kubernetes\.azure\.com/agentpool"=userpool - ``` - - Example output: - - - ```output - NAME: external-secrets - LAST DEPLOYED: Tue Jun 11 11:55:32 2024 - NAMESPACE: mongodb - STATUS: deployed - REVISION: 1 - TEST SUITE: None - NOTES: - external-secrets has been deployed successfully in namespace mongodb! - - In order to begin using ExternalSecrets, you will need to set up a SecretStore - or ClusterSecretStore resource (for example, by creating a 'vault' SecretStore). - - More information on the different types of SecretStores and how to configure them - can be found in our Github: https://github.com/external-secrets/external-secrets - ``` - -3. Generate a random password for the MongoDB cluster and store it in Azure Key Vault using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - #MongoDB connection strings can contain special characters in the password, which need to be URL encoded. - #This is because the connection string is a URI, and special characters can interfere with the URI structure. - #This function generates secrets of 32 characters using only alphanumeric characters. - - generateRandomPasswordString() { - cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 - } - ``` - -## Create MongoDB secrets - -1. Create a MongoDB [backup user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#backup-and-restoration-roles) secret to use for any backup and restore operations using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-USER --value MONGODB_BACKUP_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -2. Create a MongoDB [database admin user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#all-database-roles) secret for database administration using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-USER --value MONGODB_DATABASE_ADMIN_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -3. Create a MongoDB [cluster administration user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterAdmin) secret for a cluster administration role that provides administration for more than one database using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-USER --value MONGODB_CLUSTER_ADMIN_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -4. Create a MongoDB [cluster monitoring user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterMonitor) secret for cluster monitoring using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-USER --value MONGODB_CLUSTER_MONITOR_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -5. Create a user and password secret for [user administration](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-userAdminAnyDatabase) using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-USER --value MONGODB_USER_ADMIN_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -6. Create a secret for the API key used to access the Percona Monitoring and Management (PMM) server using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. You update the value of this secret later when you deploy the PMM server. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name PMM-SERVER-API-KEY --value $(openssl rand -base64 32) --output table - ``` - -7. Add `AZURE-STORAGE-ACCOUNT-NAME` to use later for backups using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name AZURE-STORAGE-ACCOUNT-NAME --value $AKS_MONGODB_BACKUP_STORAGE_ACCOUNT_NAME --output table - ``` - -## Create secrets resources - -1. Create a `SecretStore` resource to access the MongoDB passwords stored in your key vault using the `kubectl apply` command. - - ```bash - kubectl apply -f - < - ```output - secretstore.external-secrets.io/azure-store created - ``` - -2. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace with the MongoDB secrets stored in your key vault. - - ```bash - kubectl apply -f - < - ```output - externalsecret.external-secrets.io/cluster-aks-mongodb-secrets created - ``` - -3. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace for Azure Blob Storage secrets stored in your key vault. - - ```bash - kubectl apply -f - < - ```output - externalsecret.external-secrets.io/cluster-aks-azure-secrets created - ``` - -4. Create a federated credential using the [`az identity federated-credential create`](/cli/azure/identity/federated-credential#az-identity-federated-credential-create) command. - - ```azurecli-interactive - az identity federated-credential create \ - --name external-secret-operator \ - --identity-name ${MY_IDENTITY_NAME} \ - --resource-group ${MY_RESOURCE_GROUP_NAME} \ - --issuer ${OIDC_URL} \ - --subject system:serviceaccount:${AKS_MONGODB_NAMESPACE}:${SERVICE_ACCOUNT_NAME} \ - --output table - ``` - - Example output: - - - ```output - Issuer Name ResourceGroup Subject - ----------------------------------------------------------------------------------------------------------------------- ------------------------ -------------------------------- ------------------------------------- - https://australiaeast.oic.prod-aks.azure.com/aaaa0a0a-bb1b-cc2c-dd3d-eeeeee4e4e4e/a0a0a0a0-bbbb-cccc-dddd-e1e1e1e1e1e1/ external-secret-operator myResourceGroup-rg-australiaeast system:serviceaccount:mongodb:mongodb - ``` - -5. Give permission to the user-assigned identity to access the secret using the [`az keyvault set-policy`](/cli/azure/keyvault#az-keyvault-set-policy) command. - - ```azurecli-interactive - az keyvault set-policy --name $MY_KEYVAULT_NAME --object-id $MY_IDENTITY_NAME_PRINCIPAL_ID --secret-permissions get --output table - ``` - - Example output: - - - ```output - Location Name ResourceGroup - ------------- -------------- -------------------------------- - australiaeast vault-cjcfc-kv myResourceGroup-rg-australiaeast - ``` - -## Install the Percona Operator and CRDs - -The Percona Operator is typically distributed as a Kubernetes `Deployment` or `Operator`. You can deploy it by using a `kubectl apply -f` command with a manifest file. You can find the latest manifests in the [Percona GitHub repository](https://github.com/percona/percona-server-mongodb-operator) or the [official documentation](https://docs.percona.com/percona-operator-for-mongodb/aks.html). - -* Deploy the Percona Operator and custom resource definitions (CRDs) using the `kubectl apply` command. - - ```bash - kubectl apply --server-side -f https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/v1.16.0/deploy/bundle.yaml -n "${AKS_MONGODB_NAMESPACE}" - ``` - - Example output: - - - ```output - customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied - customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied - customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied - role.rbac.authorization.k8s.io/percona-server-mongodb-operator serverside-applied - serviceaccount/percona-server-mongodb-operator serverside-applied - rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator serverside-applied - deployment.apps/percona-server-mongodb-operator serverside-applied - ``` - -## Deploy the MongoDB cluster - -1. Deploy a MongoDB cluster with the Percona Operator using the `kubectl apply` command. To help ensure high availability, you deploy the MongoDB cluster with a replica set, with sharding enabled, in multiple availability zones, and with a backup solution that stores the backups in an Azure Blob Storage account. - - ```bash - kubectl apply -f - < - ```output - perconaservermongodb.psmdb.percona.com/cluster-aks-mongodb created - ``` - -2. Finish the MongoDB cluster deployment process using the following script: - - ```bash - while [ "$(kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for MongoDB cluster to be ready"; sleep 10; done - ``` - -3. When the process finishes, your cluster shows the `Ready` status. You can view the status using the `kubectl get` command. - - ```bash - kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME ENDPOINT STATUS AGE - cluster-aks-mongodb cluster-aks-mongodb-mongos.mongodb.svc.cluster.local ready 3m1s - ``` - -4. View the availability zones of the nodes in your cluster using the `kubectl get` command. - - ```bash - kubectl get node -o custom-columns=Name:.metadata.name,Zone:".metadata.labels.topology\.kubernetes\.io/zone" - ``` - - Example output: - - - ```output - Name Zone - aks-systempool-30094695-vmss000000 australiaeast-1 - aks-nodepool1-28994785-vmss000000 australiaeast-1 - aks-nodepool1-28994785-vmss000001 australiaeast-2 - aks-nodepool1-28994785-vmss000002 australiaeast-3 - ``` - -## Connect to the Percona Server - -To connect to Percona Server for MongoDB, you need to construct the MongoDB connection URI string. It includes the credentials of the admin user, which are stored in the `Secrets` object. - -1. List the `Secrets` objects using the `kubectl get` command. - - ```bash - kubectl get secrets -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME TYPE DATA AGE - cluster-aks-azure-secrets Opaque 2 2m56s - cluster-aks-mongodb-mongodb-keyfile Opaque 1 2m54s - cluster-aks-mongodb-secrets Opaque 11 2m56s - cluster-aks-mongodb-secrets-mongodb-encryption-key Opaque 1 2m54s - cluster-aks-mongodb-ssl kubernetes.io/tls 3 2m55s - cluster-aks-mongodb-ssl-internal kubernetes.io/tls 3 2m54s - external-secrets-webhook Opaque 4 3m49s - internal-cluster-aks-mongodb-users Opaque 11 2m56s - sh.helm.release.v1.external-secrets.v1 helm.sh/release.v1 1 3m49s - ``` - -2. View the `Secrets` contents to retrieve the admin user credentials using the `kubectl get` command. - - ```bash - kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -o yaml -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - apiVersion: v1 - data: - MONGODB_BACKUP_PASSWORD: aB1cD2eF-3gH... - MONGODB_BACKUP_USER: cD2eF3gH4iJ... - MONGODB_CLUSTER_ADMIN_PASSWORD: eF3gH4iJ5kL6mN7oP... - MONGODB_CLUSTER_ADMIN_USER: gH4iJ5kL6mN7oP8... - MONGODB_CLUSTER_MONITOR_PASSWORD: iJ5kL6mN7oP8qR9sT0-u... - MONGODB_CLUSTER_MONITOR_USER: kL6mN7oP8qR9sT0... - MONGODB_DATABASE_ADMIN_PASSWORD: mN7oP8qR9sT0uV1... - MONGODB_DATABASE_ADMIN_USER: A1bC2dE3fH4iJ5kL... - MONGODB_USER_ADMIN_PASSWORD: C2dE3fH4iJ5kL6mN7oP... - MONGODB_USER_ADMIN_USER: E3fH4iJ5kL6mN7... - immutable: false - kind: Secret - metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: | - {"apiVersion":"external-secrets.io/v1beta1","kind":"ExternalSecret","metadata":{"annotations":{},"name":"cluster-aks-mongodb-secrets","namespace":"mongodb"},"spec":{"data":[{"remoteRef":{"key":"MONGODB-BACKUP-USER"},"secretKey":"MONGODB_BACKUP_USER"},{"remoteRef":{"key":"MONGODB-BACKUP-PASSWORD"},"secretKey":"MONGODB_BACKUP_PASSWORD"},{"remoteRef":{"key":"MONGODB-DATABASE-ADMIN-USER"},"secretKey":"MONGODB_DATABASE_ADMIN_USER"},{"remoteRef":{"key":"MONGODB-DATABASE-ADMIN-PASSWORD"},"secretKey":"MONGODB_DATABASE_ADMIN_PASSWORD"},{"remoteRef":{"key":"MONGODB-CLUSTER-ADMIN-USER"},"secretKey":"MONGODB_CLUSTER_ADMIN_USER"},{"remoteRef":{"key":"MONGODB-CLUSTER-ADMIN-PASSWORD"},"secretKey":"MONGODB_CLUSTER_ADMIN_PASSWORD"},{"remoteRef":{"key":"MONGODB-CLUSTER-MONITOR-USER"},"secretKey":"MONGODB_CLUSTER_MONITOR_USER"},{"remoteRef":{"key":"MONGODB-CLUSTER-MONITOR-PASSWORD"},"secretKey":"MONGODB_CLUSTER_MONITOR_PASSWORD"},{"remoteRef":{"key":"MONGODB-USER-ADMIN-USER"},"secretKey":"MONGODB_USER_ADMIN_USER"},{"remoteRef":{"key":"MONGODB-USER-ADMIN-PASSWORD"},"secretKey":"MONGODB_USER_ADMIN_PASSWORD"}],"refreshInterval":"1h","secretStoreRef":{"kind":"SecretStore","name":"azure-store"},"target":{"creationPolicy":"Owner","name":"cluster-aks-mongodb-secrets"}}} - reconcile.external-secrets.io/data-hash: aB1cD2eF-3gH4iJ5kL6-mN7oP8qR= - creationTimestamp: "2024-07-01T12:24:38Z" - labels: - reconcile.external-secrets.io/created-by: N7oP8qR9sT0uV1wX2yZ3aB4cD5eF6g - name: cluster-aks-mongodb-secrets - namespace: mongodb - ownerReferences: - - apiVersion: external-secrets.io/v1beta1 - blockOwnerDeletion: true - controller: true - kind: ExternalSecret - name: cluster-aks-mongodb-secrets - uid: aaaaaaaa-0b0b-1c1c-2d2d-333333333333 - resourceVersion: "1872" - uid: bbbbbbbb-1c1c-2d2d-3e3e-444444444444 - type: Opaque - ``` - -3. Decode the Base64-encoded login name and password from the output using the following commands: - - ```bash - #Decode login name and password on the output, which are Base64-encoded - export databaseAdmin=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_USER}" | base64 --decode) - export databaseAdminPassword=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_PASSWORD}" | base64 --decode) - - echo $databaseAdmin - echo $databaseAdminPassword - echo $AKS_MONGODB_CLUSTER_NAME - ``` - - Example output: - - - ```output - MONGODB_DATABASE_ADMIN_USER - gH4iJ5kL6mN7oP8... - cluster-aks-mongodb - ``` - -## Verify the MongoDB cluster - -In this section, you verify your MongoDB cluster by running a container with a MongoDB client and connect its console output to your terminal. - -1. Create a pod named `percona-client` under the `${AKS_MONGODB_NAMESPACE}` namespace in your cluster using the `kubectl run` command. - - ```bash - kubectl -n "${AKS_MONGODB_NAMESPACE}" run -i --rm --tty percona-client --image=${MY_ACR_REGISTRY}.azurecr.io/percona-server-mongodb:7.0.8-5 --restart=Never -- bash -il - ``` - -2. In a different terminal window, verify the pod was successfully created using the `kubectl get` command. - - ```bash - kubectl get pod percona-client -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME READY STATUS RESTARTS AGE - percona-client 1/1 Running 0 39s - ``` - -3. Connect to the MongoDB cluster using the admin user credentials from the previous section in the terminal window that you used to create the `percona-client` pod. - - ```bash - # Note: Replace variables `databaseAdmin` , `databaseAdminPassword` and `AKS_MONGODB_CLUSTER_NAME` with actual values printed in step 3. - - mongosh "mongodb://${databaseAdmin}:${databaseAdminPassword}@${AKS_MONGODB_CLUSTER_NAME}-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=false&directConnection=true" - ``` - - Example output: - - - ```output - Current Mongosh Log ID: L6mN7oP8qR9sT0uV1wX2yZ3a - Connecting to: mongodb://@cluster-aks-mongodb-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=false&directConnection=true&appName=mongosh+2.1.5 - Using MongoDB: 7.0.8-5 - Using Mongosh: 2.1.5 - - For mongosh info see: https://docs.mongodb.com/mongodb-shell/ - ... - ``` - -4. List the databases in your cluster using the `show dbs` command. - - ```bash - show dbs - ``` - - Example output: - - - ```output - rs0 [direct: mongos] admin> show dbs - admin 960.00 KiB - config 3.45 MiB - rs0 [direct: mongos] admin> - ``` - -## Create a MongoDB backup - -You can back up your data to Azure using one of the following methods: - -* **Manual**: Manually back up your data at any time. -* **Scheduled**: Configure backups and their schedules in the CRD YAML. The Percona Operator makes the backups automatically according to the specified schedule. - -The Percona Operator can perform either of the following backup types: - -* **Logical backup**: Query Percona Server for MongoDB for the database data, and then write the retrieved data to the remote backup storage. -* **Physical backup**: Copy physical files from the Percona Server for MongoDB `dbPath` data directory to the remote backup storage. - -Logical backups use less storage but are slower than physical backups. - -To store backups on Azure Blob Storage using Percona, you need to create a secret. You completed this step in an earlier command. For detailed instructions, follow the steps in the [Percona documentation about Azure Blob Storage](https://docs.percona.com/percona-operator-for-mongodb/backups-storage.html#microsoft-azure-blob-storage). - -### Configure scheduled backups - -You can define the backup schedule in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: - -* Set the `backup.enabled` key to `true`. -* Ensure that the `backup.storages` subsection contains at least one configured storage resource. -* Ensure that the `backup.tasks` subsection enables backup scheduling. - -For more information, see [Making scheduled backups](https://docs.percona.com/percona-operator-for-mongodb/backups-scheduled.html). - -### Perform manual backups - -You can make a manual, on-demand backup in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: - -* Set the `backup.enabled` key to `true`. -* Ensure that the `backup.storages` subsection contains at least one configured storage resource. - -For more information, see [Making on-demand backups](https://docs.percona.com/percona-operator-for-mongodb/backups-ondemand.html). - -## Deploy a MongoDB backup - -1. Deploy your MongoDB backup using the `kubectl apply` command. - - ```bash - kubectl apply -f - < - ```output - perconaservermongodbbackup.psmdb.percona.com/az-backup1 created - ``` - -2. Finish the MongoDB backup deployment process using the following script: - - ```bash - while [ "$(kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for the backup to be ready"; sleep 10; done - ``` - - Example output: - - - ```output - waiting for the backup to be ready - ``` - -3. When the process finishes, the backup should return the `Ready` status. Verify the backup deployment was successful using the `kubectl get` command. - - ```bash - kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE - az-backup1 cluster-aks-mongodb azure-blob https://mongodbsacjcfc.blob.core.windows.net/backups/psmdb/2024-07-01T12:27:57Z logical ready 3h3m 3h3m - ``` - -4. If you have any problems with the backup, you can view logs from the `backup-agent` container of the appropriate pod using the `kubectl logs` command. - - ```bash - kubectl logs pod/${AKS_MONGODB_CLUSTER_NAME}-rs0-0 -c backup-agent -n ${AKS_MONGODB_NAMESPACE} - ``` - -## Next step - -> [!div class="nextstepaction"] -> [Deploy a client application (Mongo Express)][validate-mongodb-cluster] - - -[validate-mongodb-cluster]: ./validate-mongodb-cluster.md \ No newline at end of file diff --git a/tools/mongodb_redacted.md b/tools/mongodb_redacted.md deleted file mode 100644 index 291975a60..000000000 --- a/tools/mongodb_redacted.md +++ /dev/null @@ -1,815 +0,0 @@ ---- -title: 'Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS)' -description: In this article, you configure and deploy a MongoDB cluster on AKS. -ms.topic: how-to -ms.date: 01/07/2025 -author: xxxxx -ms.author: xxxxx -ms.custom: aks-related-content ---- - -# Configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS) - -In this article, you configure and deploy a MongoDB cluster on Azure Kubernetes Service (AKS). - -## Configure a workload identity - -1. Create a namespace for the MongoDB cluster using the `kubectl create namespace` command. - - ```bash - kubectl create namespace ${AKS_MONGODB_NAMESPACE} --dry-run=client --output yaml | kubectl apply -f - - ``` - - Example output: - - - ```output - namespace/xxxxx created - ``` - -2. Create a service account and configure a workload identity using the `kubectl apply` command. - - ```bash - export TENANT_ID=$(az account show --query tenantId -o tsv) - cat < - ```output - serviceaccount/xxxxx created - ``` - -## Install External Secrets Operator - -In this section, you use Helm to install External Secrets Operator. External Secrets Operator is a Kubernetes operator that manages the life cycle of external secrets stored in external secret stores like Azure Key Vault. - -1. Add the External Secrets Helm repository and update the repository using the `helm repo add` and `helm repo update` commands. - - ```bash - helm repo add external-secrets https://charts.external-secrets.io - helm repo update - ``` - - Example output: - - - ```output - Hang tight while we grab the latest from your chart repositories... - ...Successfully got an update from the "external-secrets" chart repository - ``` - -2. Install External Secrets Operator using the `helm install` command. - - ```bash - helm install external-secrets \ - external-secrets/external-secrets \ - --namespace ${AKS_MONGODB_NAMESPACE} \ - --create-namespace \ - --set installCRDs=true \ - --wait \ - --set nodeSelector."kubernetes\.azure\.com/agentpool"=userpool - ``` - - Example output: - - - ```output - NAME: external-secrets - LAST DEPLOYED: Tue Jun 11 11:55:32 2024 - NAMESPACE: xxxxx - STATUS: deployed - REVISION: 1 - TEST SUITE: None - NOTES: - external-secrets has been deployed successfully in namespace xxxxx! - - In order to begin using ExternalSecrets, you will need to set up a SecretStore - or ClusterSecretStore resource (for example, by creating a 'vault' SecretStore). - - More information on the different types of SecretStores and how to configure them - can be found in our Github: https://github.com/external-secrets/external-secrets - ``` - -3. Generate a random password for the MongoDB cluster and store it in Azure Key Vault using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - #MongoDB connection strings can contain special characters in the password, which need to be URL encoded. - #This is because the connection string is a URI, and special characters can interfere with the URI structure. - #This function generates secrets of 32 characters using only alphanumeric characters. - - generateRandomPasswordString() { - cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 - } - ``` - -## Create MongoDB secrets - -1. Create a MongoDB [backup user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#backup-and-restoration-roles) secret to use for any backup and restore operations using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-USER --value MONGODB_BACKUP_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-BACKUP-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -2. Create a MongoDB [database admin user and password](https://www.mongodb.com/docs/manual/reference/built-in-roles/#all-database-roles) secret for database administration using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-USER --value MONGODB_DATABASE_ADMIN_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-DATABASE-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -3. Create a MongoDB [cluster administration user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterAdmin) secret for a cluster administration role that provides administration for more than one database using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-USER --value MONGODB_CLUSTER_ADMIN_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -4. Create a MongoDB [cluster monitoring user and admin](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-clusterMonitor) secret for cluster monitoring using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-USER --value MONGODB_CLUSTER_MONITOR_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-CLUSTER-MONITOR-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -5. Create a user and password secret for [user administration](https://www.mongodb.com/docs/manual/reference/built-in-roles/#mongodb-authrole-userAdminAnyDatabase) using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-USER --value MONGODB_USER_ADMIN_USER --output table - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name MONGODB-USER-ADMIN-PASSWORD --value $(generateRandomPasswordString) --output table - ``` - -6. Create a secret for the API key used to access the Percona Monitoring and Management (PMM) server using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. You update the value of this secret later when you deploy the PMM server. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name PMM-SERVER-API-KEY --value $(openssl rand -base64 32) --output table - ``` - -7. Add `AZURE-STORAGE-ACCOUNT-NAME` to use later for backups using the [`az keyvault secret set`](/cli/azure/keyvault/secret#az-keyvault-secret-set) command. - - ```azurecli-interactive - az keyvault secret set --vault-name $MY_KEYVAULT_NAME --name AZURE-STORAGE-ACCOUNT-NAME --value $AKS_MONGODB_BACKUP_STORAGE_ACCOUNT_NAME --output table - ``` - -## Create secrets resources - -1. Create a `SecretStore` resource to access the MongoDB passwords stored in your key vault using the `kubectl apply` command. - - ```bash - kubectl apply -f - < - ```output - secretstore.external-secrets.io/xxxxx created - ``` - -2. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace with the MongoDB secrets stored in your key vault. - - ```bash - kubectl apply -f - < - ```output - externalsecret.external-secrets.io/xxxxx created - ``` - -3. Create an `ExternalSecret` resource using the `kubectl apply` command. This resource creates a Kubernetes secret in the `mongodb` namespace for Azure Blob Storage secrets stored in your key vault. - - ```bash - kubectl apply -f - < - ```output - externalsecret.external-secrets.io/xxxxx created - ``` - -4. Create a federated credential using the [`az identity federated-credential create`](/cli/azure/identity/federated-credential#az-identity-federated-credential-create) command. - - ```azurecli-interactive - az identity federated-credential create \ - --name external-secret-operator \ - --identity-name ${MY_IDENTITY_NAME} \ - --resource-group ${MY_RESOURCE_GROUP_NAME} \ - --issuer ${OIDC_URL} \ - --subject system:serviceaccount:${AKS_MONGODB_NAMESPACE}:${SERVICE_ACCOUNT_NAME} \ - --output table - ``` - - Example output: - - - ```output - Issuer Name ResourceGroup Subject - ----------------------------------------------------------------------------------------------------------------------- ------------------------ -------------------------------- ------------------------------------- - https://australiaeast.oic.prod-aks.azure.com/xxxxx/xxxxx/ xxxxx xxxxx system:serviceaccount:xxxxx:xxxxx - ``` - -5. Give permission to the user-assigned identity to access the secret using the [`az keyvault set-policy`](/cli/azure/keyvault#az-keyvault-set-policy) command. - - ```azurecli-interactive - az keyvault set-policy --name $MY_KEYVAULT_NAME --object-id $MY_IDENTITY_NAME_PRINCIPAL_ID --secret-permissions get --output table - ``` - - Example output: - - - ```output - Location Name ResourceGroup - ------------- -------------- -------------------------------- - australiaeast xxxxx xxxxx - ``` - -## Install the Percona Operator and CRDs - -The Percona Operator is typically distributed as a Kubernetes `Deployment` or `Operator`. You can deploy it by using a `kubectl apply -f` command with a manifest file. You can find the latest manifests in the [Percona GitHub repository](https://github.com/percona/percona-server-mongodb-operator) or the [official documentation](https://docs.percona.com/percona-operator-for-mongodb/aks.html). - -* Deploy the Percona Operator and custom resource definitions (CRDs) using the `kubectl apply` command. - - ```bash - kubectl apply --server-side -f https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/v1.16.0/deploy/bundle.yaml -n "${AKS_MONGODB_NAMESPACE}" - ``` - - Example output: - - - ```output - customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied - customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied - customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied - role.rbac.authorization.k8s.io/percona-server-mongodb-operator serverside-applied - serviceaccount/percona-server-mongodb-operator serverside-applied - rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator serverside-applied - deployment.apps/percona-server-mongodb-operator serverside-applied - ``` - -## Deploy the MongoDB cluster - -1. Deploy a MongoDB cluster with the Percona Operator using the `kubectl apply` command. To help ensure high availability, you deploy the MongoDB cluster with a replica set, with sharding enabled, in multiple availability zones, and with a backup solution that stores the backups in an Azure Blob Storage account. - - ```bash - kubectl apply -f - < - ```output - perconaservermongodb.psmdb.percona.com/xxxxx created - ``` - -2. Finish the MongoDB cluster deployment process using the following script: - - ```bash - while [ "$(kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for MongoDB cluster to be ready"; sleep 10; done - ``` - -3. When the process finishes, your cluster shows the `Ready` status. You can view the status using the `kubectl get` command. - - ```bash - kubectl get psmdb -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME ENDPOINT STATUS AGE - xxxxx xxxxx ready 3m1s - ``` - -4. View the availability zones of the nodes in your cluster using the `kubectl get` command. - - ```bash - kubectl get node -o custom-columns=Name:.metadata.name,Zone:".metadata.labels.topology\.kubernetes\.io/zone" - ``` - - Example output: - - - ```output - Name Zone - xxxxx australiaeast-1 - xxxxx australiaeast-1 - xxxxx australiaeast-2 - xxxxx australiaeast-3 - ``` - -## Connect to the Percona Server - -To connect to Percona Server for MongoDB, you need to construct the MongoDB connection URI string. It includes the credentials of the admin user, which are stored in the `Secrets` object. - -1. List the `Secrets` objects using the `kubectl get` command. - - ```bash - kubectl get secrets -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME TYPE DATA AGE - xxxxx Opaque 2 2m56s - xxxxx Opaque 1 2m54s - xxxxx Opaque 11 2m56s - xxxxx Opaque 1 2m54s - xxxxx kubernetes.io/tls 3 2m55s - xxxxx kubernetes.io/tls 3 2m54s - external-secrets-webhook Opaque 4 3m49s - xxxxx Opaque 11 2m56s - xxxxx helm.sh/release.v1 1 3m49s - ``` - -2. View the `Secrets` contents to retrieve the admin user credentials using the `kubectl get` command. - - ```bash - kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -o yaml -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - apiVersion: v1 - data: - MONGODB_BACKUP_PASSWORD: xxxxx - MONGODB_BACKUP_USER: xxxxx - MONGODB_CLUSTER_ADMIN_PASSWORD: xxxxx - MONGODB_CLUSTER_ADMIN_USER: xxxxx - MONGODB_CLUSTER_MONITOR_PASSWORD: xxxxx - MONGODB_CLUSTER_MONITOR_USER: xxxxx - MONGODB_DATABASE_ADMIN_PASSWORD: xxxxx - MONGODB_DATABASE_ADMIN_USER: xxxxx - MONGODB_USER_ADMIN_PASSWORD: xxxxx - MONGODB_USER_ADMIN_USER: xxxxx - immutable: false - kind: Secret - metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: xxxxx - reconcile.external-secrets.io/data-hash: xxxxx - creationTimestamp: "xxxxx" - labels: - reconcile.external-secrets.io/created-by: xxxxx - name: xxxxx - namespace: mongodb - ownerReferences: - - apiVersion: external-secrets.io/v1beta1 - blockOwnerDeletion: true - controller: true - kind: ExternalSecret - name: xxxxx - uid: xxxxx - resourceVersion: "xxxxx" - uid: xxxxx - type: Opaque - ``` - -3. Decode the Base64-encoded login name and password from the output using the following commands: - - ```bash - #Decode login name and password on the output, which are Base64-encoded - export databaseAdmin=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_USER}" | base64 --decode) - export databaseAdminPassword=$(kubectl get secret ${AKS_MONGODB_SECRETS_NAME} -n ${AKS_MONGODB_NAMESPACE} -o jsonpath="{.data.MONGODB_DATABASE_ADMIN_PASSWORD}" | base64 --decode) - - echo $databaseAdmin - echo $databaseAdminPassword - echo $AKS_MONGODB_CLUSTER_NAME - ``` - - Example output: - - - ```output - xxxxx - xxxxx - xxxxx - ``` - -## Verify the MongoDB cluster - -In this section, you verify your MongoDB cluster by running a container with a MongoDB client and connect its console output to your terminal. - -1. Create a pod named `percona-client` under the `${AKS_MONGODB_NAMESPACE}` namespace in your cluster using the `kubectl run` command. - - ```bash - kubectl -n "${AKS_MONGODB_NAMESPACE}" run -i --rm --tty percona-client --image=${MY_ACR_REGISTRY}.azurecr.io/percona-server-mongodb:7.0.8-5 --restart=Never -- bash -il - ``` - -2. In a different terminal window, verify the pod was successfully created using the `kubectl get` command. - - ```bash - kubectl get pod percona-client -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME READY STATUS RESTARTS AGE - xxxxx 1/1 Running 0 39s - ``` - -3. Connect to the MongoDB cluster using the admin user credentials from the previous section in the terminal window that you used to create the `percona-client` pod. - - ```bash - # Note: Replace variables `databaseAdmin` , `databaseAdminPassword` and `AKS_MONGODB_CLUSTER_NAME` with actual values printed in step 3. - - mongosh "mongodb://${databaseAdmin}:${databaseAdminPassword}@${AKS_MONGODB_CLUSTER_NAME}-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=false&directConnection=true" - ``` - - Example output: - - - ```output - Current Mongosh Log ID: xxxxx - Connecting to: mongodb://@xxxxx/admin?replicaSet=rs0&ssl=false&directConnection=true&appName=mongosh+2.1.5 - Using MongoDB: 7.0.8-5 - Using Mongosh: 2.1.5 - - For mongosh info see: https://docs.mongodb.com/mongodb-shell/ - ... - ``` - -4. List the databases in your cluster using the `show dbs` command. - - ```bash - show dbs - ``` - - Example output: - - - ```output - rs0 [direct: mongos] admin> show dbs - admin 960.00 KiB - config 3.45 MiB - rs0 [direct: mongos] admin> - ``` - -## Create a MongoDB backup - -You can back up your data to Azure using one of the following methods: - -* **Manual**: Manually back up your data at any time. -* **Scheduled**: Configure backups and their schedules in the CRD YAML. The Percona Operator makes the backups automatically according to the specified schedule. - -The Percona Operator can perform either of the following backup types: - -* **Logical backup**: Query Percona Server for MongoDB for the database data, and then write the retrieved data to the remote backup storage. -* **Physical backup**: Copy physical files from the Percona Server for MongoDB `dbPath` data directory to the remote backup storage. - -Logical backups use less storage but are slower than physical backups. - -To store backups on Azure Blob Storage using Percona, you need to create a secret. You completed this step in an earlier command. For detailed instructions, follow the steps in the [Percona documentation about Azure Blob Storage](https://docs.percona.com/percona-operator-for-mongodb/backups-storage.html#microsoft-azure-blob-storage). - -### Configure scheduled backups - -You can define the backup schedule in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: - -* Set the `backup.enabled` key to `true`. -* Ensure that the `backup.storages` subsection contains at least one configured storage resource. -* Ensure that the `backup.tasks` subsection enables backup scheduling. - -For more information, see [Making scheduled backups](https://docs.percona.com/percona-operator-for-mongodb/backups-scheduled.html). - -### Perform manual backups - -You can make a manual, on-demand backup in the backup section of the CRD in *mongodb-cr.yaml* using the following guidance: - -* Set the `backup.enabled` key to `true`. -* Ensure that the `backup.storages` subsection contains at least one configured storage resource. - -For more information, see [Making on-demand backups](https://docs.percona.com/percona-operator-for-mongodb/backups-ondemand.html). - -## Deploy a MongoDB backup - -1. Deploy your MongoDB backup using the `kubectl apply` command. - - ```bash - kubectl apply -f - < - ```output - perconaservermongodbbackup.psmdb.percona.com/xxxxx created - ``` - -2. Finish the MongoDB backup deployment process using the following script: - - ```bash - while [ "$(kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} -o jsonpath='{.items[0].status.state}')" != "ready" ]; do echo "waiting for the backup to be ready"; sleep 10; done - ``` - - Example output: - - - ```output - waiting for the backup to be ready - ``` - -3. When the process finishes, the backup should return the `Ready` status. Verify the backup deployment was successful using the `kubectl get` command. - - ```bash - kubectl get psmdb-backup -n ${AKS_MONGODB_NAMESPACE} - ``` - - Example output: - - - ```output - NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE - xxxxx xxxxx xxxxx https://xxxxx.blob.core.windows.net/backups/psmdb/xxxxx logical ready 3h3m 3h3m - ``` - -4. If you have any problems with the backup, you can view logs from the `backup-agent` container of the appropriate pod using the `kubectl logs` command. - - ```bash - kubectl logs pod/${AKS_MONGODB_CLUSTER_NAME}-rs0-0 -c backup-agent -n ${AKS_MONGODB_NAMESPACE} - ``` - -## Next step - -> [!div class="nextstepaction"] -> [Deploy a client application (Mongo Express)][validate-mongodb-cluster] - - -[validate-mongodb-cluster]: ./validate-mongodb-cluster.md \ No newline at end of file diff --git a/tools/mongodb_security_report.md b/tools/mongodb_security_report.md deleted file mode 100644 index 937116796..000000000 --- a/tools/mongodb_security_report.md +++ /dev/null @@ -1,90 +0,0 @@ -Below is the complete security vulnerability analysis report for the provided Exec Doc. This analysis covers both static (code review) and dynamic (runtime environment) aspects using industry frameworks such as the OWASP Top 10 and cloud security best practices. - ------------------------------------------------------------- - -# Security Vulnerability Analysis Report - -## 1. Executive Summary - -This document outlines a comprehensive security review of the MongoDB cluster deployment instructions on Azure Kubernetes Service (AKS) using Percona Operator and External Secrets Operator. Overall, most risks are related to misconfigurations and reliance on external secret management. In particular, several areas require improvement regarding authentication and authorization settings, network security (e.g., non-enforced TLS), input validation, command injection risk in shell helpers, and secret management practices. While no immediate critical code-level injection was found, proper remediation and adherence to best practices are recommended to prevent potential privilege escalation, data leakage, and cloud exposure risks. - -## 2. Methodology - -The analysis was performed in two main phases: - -• Static Code Review: -– A manual review of the YAML manifests, shell scripts, Helm commands, and embedded Kubernetes objects. -– Assessment based on configuration best practices (namespace isolation, RBAC, workload identity annotations). -– Evaluation of inline scripts (e.g., password generation) for command injection and proper use of environment variable substitution. - -• Dynamic/Runtime Assessment: -– Consideration of how the deployment behaves (runtime secret handling, federated credential use, network communication). -– Review of cloud-specific operations such as creation of federated credentials, key vault secret policies, and external secret polling frequency. -– Evaluation of network configurations (unencrypted MongoDB connection string and cross-namespace secret accessibility). - -## 3. Findings - -The following table summarizes the identified vulnerabilities along with their severity, exact locations (where applicable), description, potential impact, and recommended fixes. - -| Severity | Location / Context | Description | Potential Impact | Recommended Fix / Code Example | -|----------|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Critical | MongoDB connection URI in client connection | The connection string uses “ssl=false”, disabling encrypted communication between clients and the MongoDB service. | Sensitive credentials and data transmissions are exposed to eavesdropping and man-in-the-middle attacks. | Enforce TLS/SSL by setting ssl=true and ensuring certificates are properly configured. Example:
``` -mongosh "mongodb://${databaseAdmin}:${databaseAdminPassword}@${AKS_MONGODB_CLUSTER_NAME}-mongos.mongodb.svc.cluster.local/admin?replicaSet=rs0&ssl=true&directConnection=true" -``` | -| High | Workload Identity & Service Account Manifest | The ServiceAccount YAML includes annotations for workload identity (client-id, tenant-id) and creates federated credentials. If misconfigured (e.g., allowing overly broad access or not restricted to the intended namespace), it could allow unauthorized access or abuse of privileges in the cluster. | Potential privilege escalation and unauthorized access to resources in the AKS cluster and Azure Key Vault. | Limit the scope of the service account by using minimal RBAC privileges and enforce strict validation on annotations. Additionally, ensure the federated credential subject is tightly scoped. | -| High | Kubernetes RBAC and Secret Storage | Kubernetes Secrets are stored base64-encoded and referenced in multiple YAML files. Without proper encryption at rest (e.g., ETCD encryption) or strict RBAC restrictions, there is a risk that unauthorized users could read sensitive data. | Exposure of credentials (MongoDB admin, backup, cluster users) if an attacker gains read access to secrets. | Enable encryption at rest for Kubernetes secrets and restrict access via RBAC. Use tools such as Kubernetes Secret Encryption Providers and audit logs to monitor accesses. | -| Medium | Shell Function “generateRandomPasswordString” | The helper function uses /dev/urandom piped to tr and fold. Although the randomness is sufficient, interpolation of environment variables around this function (if uncontrolled) could allow local command injection in other contexts. | If an attacker controls input or environment variables, it could inject commands that compromise the system. | Validate or hard-code the allowed character set and ensure that environment variables used in the script (e.g., for names) are sanitized before use. | -| Medium | External Commands with Environment Variables | Many commands depend on environment variables (e.g., ${AKS_MONGODB_NAMESPACE}, ${MY_IDENTITY_NAME_CLIENT_ID}). Misconfiguration or injection in these variables (if not validated earlier) might lead to unintended command execution or resource exposure. | Unintended namespace creation, malicious resource targeting, or command injection if variables contain unsafe input.| Validate and sanitize environment variables prior to use. For example, using regex checks in your shell script before passing these values to kubectl or helm commands. | -| Medium | Federated Credential Creation (az identity) | The federation subject is constructed with a variable reference to the namespace and service account. If manipulated, attackers might elevate privileges by targeting the wrong subject, especially if OIDC endpoints are misconfigured. | Privilege escalation leading to unauthorized access to Azure resources. | Double-check the correctness of the issuer URL and subject field. Use strict identity policies and consider auditing the federated credential creation process for unusual modifications. | -| Low | Logging and Secret Disclosure in Shell Scripts | The documentation shows echoing of environment variables such as $databaseAdmin and $databaseAdminPassword directly on the console output. | Risk of leaking sensitive information to local logs or process history, especially in shared environments. | Remove unnecessary echo commands that print secret values. Use secure logging that obfuscates sensitive data. | -| Low | Backup and Cloud Storage Secrets | While backup operations and storage account access are configured via secrets, the lifecycle of these secrets is not discussed and could lead to outdated or leaked credentials if not rotated properly. | Persistent storage credentials might be exploited if not rotated; manual intervention needed for secret rotations. | Implement automated secret rotation and periodic audits of backup and storage credentials. Ensure that backups themselves are encrypted and access is strictly limited. | -| Low | Certificate and TLS Usage in Internal Communications | The YAML mostly does not enforce TLS for internal connections between pods (example: “ssl=false” in the MongoDB connection URI) and does not detail the use of mutual TLS between components such as the External Secrets Operator and Key Vault. | Risk of interception in a compromised cluster network or lateral movement if an attacker gains in-cluster access. | Enforce TLS between all cluster components (both intra-cluster and external communications). Configure mutual TLS (mTLS) for sensitive operations between operators and API servers where possible. | - -## 4. Recommendations - -Based on the findings above, the following steps are recommended: - -1. Secure Communication: - • Update the MongoDB connection string to enforce TLS (ssl=true). - • Configure certificates and enable mutual TLS for intra-cluster communications. - -2. Harden Identity and Access Management: - • Restrict ServiceAccount scopes using strict RBAC policies. - • Validate and lock down annotations used for workload identities. - • Review and minimize federated credential subject claims ensuring they match the intended namespace/service account. - -3. Protect Kubernetes Secrets: - • Enable encryption at rest for Kubernetes secrets. - • Tighten RBAC to limit secret read/write permissions only to required pods/users. - • Audit etcd and secret access logs for anomalous behavior. - -4. Sanitize Environment Variables and Shell Scripts: - • Validate all environment variables (namespaces, registry names, etc.) before use in commands. - • Refactor shell helpers to ensure they are protected against command injection by avoiding unsanitized interpolation. - • Remove or mask secret outputs in logs/echo commands. - -5. Improve Secret Management and Rotation: - • Ensure Azure Key Vault access policies are tightly controlled and secrets are rotated periodically. - • Monitor the use of External Secrets Operator and the secret sync frequency, ensuring timely updates and minimizing exposure if a secret is compromised. - -6. Monitor and Audit Cloud Configurations: - • Regularly audit federated credentials, backup policies, and Key Vault permissions. - • Enable logging and alerting on unusual configuration changes in the cloud environment. - -## 5. Best Practices - -To further improve the security posture of the deployment, consider the following general security best practices: - -• Adopt the Principle of Least Privilege (PoLP) for all identities and resources. -• Enable network segmentation and enforce security policies between namespaces. -• Implement regular vulnerability scans and penetration testing on both the Kubernetes infrastructure and deployed applications. -• Use automation for secret rotations and configuration audits. -• Integrate continuous monitoring and logging solutions (e.g., cloud-native SIEM) to detect abnormal behaviors quickly. -• Stay up-to-date with security patches for all deployed software components (Kubernetes, Operators, Helm charts). -• Educate users and administrators on secure configuration and incident response procedures. - ------------------------------------------------------------- - -By addressing the above recommendations and following best practices, the overall security posture of the MongoDB on AKS deployment can be significantly hardened against common vulnerabilities and misconfiguration risks. - -This concludes the security vulnerability analysis report. \ No newline at end of file diff --git a/tools/myazure_rm.yml b/tools/myazure_rm.yml deleted file mode 100644 index 57ce22eea..000000000 --- a/tools/myazure_rm.yml +++ /dev/null @@ -1,4 +0,0 @@ -plugin: azure_rm -include_vm_resource_groups: - - ansibleinventorytestrg115b21 -auth_source: auto diff --git a/tools/myazure_rm.yml (initial version) b/tools/myazure_rm.yml (initial version) deleted file mode 100644 index 716364a7f..000000000 --- a/tools/myazure_rm.yml (initial version) +++ /dev/null @@ -1,4 +0,0 @@ -plugin: azure_rm -include_vm_resource_groups: - - ${RESOURCE_GROUP} -auth_source: auto diff --git a/tools/myazure_rm.yml (with conditional_groups) b/tools/myazure_rm.yml (with conditional_groups) deleted file mode 100644 index c5801d936..000000000 --- a/tools/myazure_rm.yml (with conditional_groups) +++ /dev/null @@ -1,7 +0,0 @@ -plugin: azure_rm -include_vm_resource_groups: - - ${RESOURCE_GROUP} -auth_source: auto -conditional_groups: - linux: "'ubuntu' in image.offer" - windows: "'WindowsServer' in image.offer" diff --git a/tools/myazure_rm.yml (with keyed_groups) b/tools/myazure_rm.yml (with keyed_groups) deleted file mode 100644 index c32162213..000000000 --- a/tools/myazure_rm.yml (with keyed_groups) +++ /dev/null @@ -1,9 +0,0 @@ -plugin: azure_rm -include_vm_resource_groups: - - ${RESOURCE_GROUP} -auth_source: auto -conditional_groups: - linux: "'ubuntu' in image.offer" - windows: "'WindowsServer' in image.offer" -keyed_groups: - - key: tags.applicationRole diff --git a/tools/ping.yml b/tools/ping.yml deleted file mode 100644 index e4a180cbd..000000000 --- a/tools/ping.yml +++ /dev/null @@ -1,9 +0,0 @@ -- gather_facts: false - hosts: all - tasks: - - name: run ping - ping: null - vars: - ansible_password: '{{ lookup(''env'',''SSH_PASSWORD'') }}' - ansible_ssh_common_args: -o StrictHostKeyChecking=no - ansible_user: '{{ lookup(''env'',''SSH_USER'') }}' diff --git a/tools/win_ping.yml b/tools/win_ping.yml deleted file mode 100644 index 3b0bfcf00..000000000 --- a/tools/win_ping.yml +++ /dev/null @@ -1,11 +0,0 @@ -- gather_facts: false - hosts: windows - tasks: - - name: run win_ping - win_ping: null - vars: - ansible_connection: winrm - ansible_password: '{{ lookup(''env'',''WIN_PASSWORD'') }}' - ansible_user: '{{ lookup(''env'',''WIN_USER'') }}' - ansible_winrm_server_cert_validation: ignore - ansible_winrm_transport: ntlm From e863fd6c63a317daba5ee8eba0fa7b6eaf10214a Mon Sep 17 00:00:00 2001 From: naman-msft Date: Tue, 11 Mar 2025 16:22:55 -0700 Subject: [PATCH 25/87] updated readme with content filters --- README.md | 13 +- tools/user_stories.md | 355 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 4 deletions(-) create mode 100644 tools/user_stories.md diff --git a/README.md b/README.md index cc6cba4ce..13923b460 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ These experiences utilize [Innovation Engine](https://github.com/Azure/Innovatio - [Frequently Asked Questions (FAQs)](#frequently-asked-questions-faqs) - [Contact Information for Exec Docs](#contact-information-for-exec-docs) +I'll update the highlighted section with the clarified information about command execution limitations: + ## Selecting Documentation for Exec Docs Not all documentation is suitable for conversion to Exec Docs. Use these filters to determine if a document can be effectively converted: @@ -44,15 +46,16 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters 2. **Command Execution Limitations** - **Not supported for direct execution:** - - PowerShell scripts - - Python, Ruby, or Node.js code - - SQL commands + - PowerShell scripts - GUI-based instructions + - Direct code blocks containing Python, SQL, or other languages (these should be executed via BASH commands) - **Supported execution context:** - Commands that run in a Linux/bash environment - Azure CLI commands - - Terraform commands (with appropriate setup) + - Terraform commands (works without any special setup) + - Python scripts executed via BASH (e.g., `python myApp.py`) + - SQL queries executed via database CLI tools **Example of supported command:** ```markdown @@ -69,6 +72,8 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters ``` ``` + >**Note:** The key principle is that if a code block can be executed in a BASH terminal as written (the way a human would execute it), then it will work with Exec Docs. + 3. **Azure Portal Custom Cloud Shell Constraints** - **Supported scenarios:** - Standard Azure resource operations (create, read, update, delete) diff --git a/tools/user_stories.md b/tools/user_stories.md new file mode 100644 index 000000000..ed58e4b01 --- /dev/null +++ b/tools/user_stories.md @@ -0,0 +1,355 @@ +# ADA (AI Documentation Assistant) API User Stories with Detailed Flows + +## 1. Cloud Architect (Aarya) + +### User Story 1: Infrastructure Documentation Generation + +As a Cloud Architect, I want to quickly generate executable documentation for my infrastructure design +So that my operations team can reliably deploy and understand our cloud architecture. + +**Experience:** Aarya accesses the ADA API through the Azure Portal integration. She enters a description of her desired infrastructure - "a highly available web application with Azure Kubernetes Service, Azure Container Registry, and Azure Front Door." The API generates a complete executable document with properly formatted code blocks, environment variables, and step-by-step instructions. Each code block is already validated to work correctly, saving her hours of documentation time while ensuring operational reliability. + +**Detailed API Flow:** +1. Aarya accesses the Azure Portal integration with ADA +2. The portal frontend makes a `POST` request to `/api/v1/ai/generate` with payload: + ```json + { + "prompt": "Create a highly available web application with Azure Kubernetes Service, Azure Container Registry, and Azure Front Door", + "targetEnvironment": "azure", + "infrastructureType": "terraform", + "expertiseLevel": "intermediate", + "additionalContext": "This is for a financial services company with 99.99% uptime requirements" + } + ``` +3. API returns a `202 Accepted` response with `requestId: "gen-12345"` +4. Portal polls `GET /api/v1/ai/generate/gen-12345` every 3 seconds to check generation status +5. When complete, API returns a document object with Terraform infrastructure code and explanations +6. Portal then calls `POST /api/v1/documents/doc-12345/validate` to verify the document works +7. Innovation Engine runs the document in a sandbox environment and returns validation results +8. If validation succeeds, document status is updated to "validated" and displayed to Aarya +9. If validation fails, error details are shown with an option to call `POST /api/v1/documents/doc-12345/repair` + +### User Story 2: Custom Terraform Documentation + +As a Cloud Architect, I want to convert my existing Terraform scripts into educational, executable documentation so that new team members can understand our infrastructure approach while deploying resources. + +**Experience:** Aarya has complex Terraform files that work but lack explanation. She submits them to the ADA API's document conversion endpoint with context about their purpose. The API returns a comprehensive markdown document that preserves all the functional code while adding clear explanations, proper environment variable declarations, appropriate headers, and validation checkpoints that can be executed by Innovation Engine. The document becomes both a deployment tool and training material. + +**Detailed API Flow:** +1. Aarya has existing Terraform files in a GitHub repository +2. She integrates ADA with her CI/CD using a GitHub Action that triggers on PRs to the `/terraform` folder +3. The Action calls `POST /api/v1/documents` with: + ```json + { + "title": "Production Kubernetes Infrastructure", + "description": "Terraform modules for our production Kubernetes deployment", + "content": "<>", + "infrastructureType": "terraform", + "tags": ["production", "kubernetes", "documentation"] + } + ``` +4. The API processes the files and adds explanations between code blocks by analyzing the Terraform structure +5. The Action then calls `GET /api/v1/documents/{documentId}` to retrieve the generated documentation +6. For each code block, the Action verifies syntax by calling `POST /api/v1/documents/{documentId}/validate` with `validateOnly: true` +7. The completed document is committed to a new branch and a PR is created for review +8. Upon approval, the executable documentation is published to their internal knowledge base via webhook + +## 2. Product Manager (Alex) + +### User Story 3: Demo Script to Executable Document + +As a Product Manager, I want to convert my demo scripts into customer-ready, executable documents so that customers can understand and deploy our product features without engineering support + +**Experience:** Alex has created a basic demo script showing a new Azure feature. He calls the ADA API with this script and selects the option to generate dependency files. The API transforms his simple commands into a comprehensive, educational document with proper metadata, environment variables with random suffixes for idempotency, detailed explanations of each step, and expected output blocks for verification. The document automatically passes all innovation engine tests, making it immediately shareable with customers. + +**Detailed API Flow:** +1. Alex uses the ADA CLI tool that wraps the REST API +2. He runs `ada convert demo.sh --target-format=executabledoc --customer-ready` +3. The CLI tool reads his demo.sh file and makes a `POST` request to `/api/v1/documents` with: + ```json + { + "title": "New Feature Demo", + "description": "Demonstration of our new feature X", + "content": "<>", + "infrastructureType": "bash", + "tags": ["demo", "customer-ready"], + "customizationParameters": { + "generateIdempotentResourceNames": true, + "includeExpectedOutput": true, + "addPrerequisiteChecks": true + } + } + ``` +4. API processes the script and returns a document ID +5. CLI tool then calls `POST /api/v1/documents/{documentId}/validate` to test the generated document +6. The API runs the document through Innovation Engine which executes each code block sequentially +7. API captures the output from each step and verifies it matches expected results +8. CLI tool receives the validated document and writes it to `demo-executable.md` +9. Alex reviews the document with embedded expected outputs and resource name randomization + +### User Story 4: User Experience Documentation + +As a Product Manager, I want to document customer experiences in an executable format early in the development process so that engineering teams can understand the expected behavior and validate it works as designed. + +**Experience:** Alex creates a basic description of a new feature workflow, then submits it to the ADA API. The service generates a document-driven design specification that includes both narrative explanation and executable code. Engineers can run this document to see the expected behavior, while Alex can verify the feature matches his design intent. The document becomes both a specification and a validation tool. + +**Detailed API Flow:** +1. Alex uses the Azure Portal ADA integration +2. He creates a new document description and submits it through a form that calls `POST /api/v1/ai/generate` +3. The request includes: + ```json + { + "prompt": "Document the user experience for containerizing and deploying a Node.js app to AKS", + "targetEnvironment": "azure", + "infrastructureType": "azcli", + "expertiseLevel": "beginner", + "additionalContext": "Target audience is developers with no Kubernetes experience" + } + ``` +4. The API generates a document and returns a document ID +5. The portal displays a preview and Alex makes edits through the UI +6. Each edit triggers a `PUT /api/v1/documents/{documentId}` call with the updated content +7. Alex shares the document with engineering by clicking "Share", which calls `GET /api/v1/documents/{documentId}?format=markdown` +8. The API returns a shareable link with appropriate permissions set +9. Engineering team members access the document and can execute it section-by-section using the Innovation Engine integration + +## 3. Content Author + +### User Story 5: Automated Documentation Testing + +As a Content Author, I want to automatically test my infrastructure documentation for errors so that customers don't encounter issues when following our guidance. + +**Experience:** A content author submits their markdown document to the ADA API's validation endpoint. The service identifies several issues: missing environment variables, commands that would fail with certain Azure regions, and dependencies not properly defined. The API automatically fixes these issues through multiple attempts, each time applying more sophisticated troubleshooting strategies until all tests pass. The author receives a fully functional document with detailed explanations of what was fixed. + +**Detailed API Flow:** +1. Content author submits a document through the MS Learn authoring tool +2. The tool calls `POST /api/v1/documents` with the markdown content +3. After creation, it immediately calls `POST /api/v1/documents/{documentId}/validate` with: + ```json + { + "environmentParameters": { + "azureRegion": "eastus", + "subscriptionType": "pay-as-you-go" + }, + "validateOnly": false + } + ``` +4. The API returns validation status with error details for failing steps +5. For each error, the tool calls `POST /api/v1/documents/{documentId}/repair` with: + ```json + { + "validationErrors": ["Step 3 failed: Error: Resource group name already exists"], + "userGuidance": "Please fix the resource naming to be unique" + } + ``` +6. The API returns suggested fixes and the tool applies them with `PUT /api/v1/documents/{documentId}` +7. This process repeats until all validation passes or user intervention is required +8. The tool then executes a final validation call with different parameters to test idempotency: + ```json + { + "environmentParameters": { + "azureRegion": "westeurope", + "subscriptionType": "enterprise" + }, + "validateOnly": false + } + ``` +9. Once all validations pass, the document is marked as validated and ready for publishing + +### User Story 6: Shell Script Documentation + +As a Content Author, I want to convert technical shell scripts into educational documents so that users understand what each part of the script accomplishes. + +**Experience:** The author has a complex deployment shell script but no documentation. They call the ADA API's script documentation endpoint with the script path and receive a fully structured markdown document with proper headings, detailed explanations between code blocks, all required metadata, and preserved functionality. The document explains each section's purpose while maintaining all the original functionality. + +**Detailed API Flow:** +1. Content author uses the ADA VS Code Extension +2. They open their complex deployment shell script and right-click to select "Generate Executable Documentation" +3. The extension calls `POST /api/v1/ai/generate` with: + ```json + { + "prompt": "Convert this shell script into an educational executable document", + "targetEnvironment": "bash", + "infrastructureType": "bash", + "additionalContext": "<>", + "expertiseLevel": "beginner" + } + ``` +4. The API processes the request and returns a generated document +5. VS Code extension displays the document in a side panel with syntax highlighting +6. Author makes edits which are synchronized with the API via `PUT /api/v1/documents/{documentId}` +7. They click "Test" in the extension, which calls `POST /api/v1/documents/{documentId}/validate` +8. The API runs each code block in isolation and returns success/error for each section +9. Author can see exactly which parts of their script need improvement before publishing +10. Final document is exported as markdown with embedded executable code blocks + +## 4. Developer/Engineer + +### User Story 7: API-Driven Document Generation Pipeline + +As a Developer, I want to integrate the ADA API into our CI/CD pipeline so that our infrastructure documentation is automatically generated and tested alongside code changes. + +**Experience:** A developer creates a GitHub Action that calls the ADA API whenever infrastructure code is changed. The action submits the updated Terraform files to the API, which generates an updated executable document. This document is then automatically tested using the Innovation Engine. If validation passes, the updated documentation is published to their internal knowledge base. This ensures their documentation always reflects the current infrastructure state. + +**Detailed API Flow:** +1. Developer creates a GitHub Action workflow file (.github/workflows/update-docs.yml) +2. The workflow triggers whenever infrastructure files change in the repo +3. Action authenticates with Azure AD and obtains a token for the ADA API +4. It identifies changed Terraform files and calls `POST /api/v1/documents` with: + ```json + { + "title": "Infrastructure Documentation", + "description": "Auto-generated from CI/CD pipeline", + "content": "<>", + "infrastructureType": "terraform", + "tags": ["cicd", "auto-generated"] + } + ``` +5. The API processes the Terraform files and generates a structured document +6. Action then calls `POST /api/v1/documents/{documentId}/validate` to verify functionality +7. If validation succeeds, it calls `GET /api/v1/documents/{documentId}?format=markdown` +8. The action commits the generated markdown to the repo's `/docs` folder +9. For any validation failures, it opens an issue with details from the validation response +10. A scheduled job runs weekly to revalidate all documents by calling: + ``` + GET /api/v1/documents?tag=cicd&status=validated + ``` + And then validating each document to ensure continued functionality + +### User Story 8: Self-Healing Documentation + +As a Developer, I want to automatically fix broken documentation when changes occur in the underlying cloud services so that our documents remain functional even when Azure features evolve + +**Experience:** The developer has scheduled a job that periodically validates all executive documents against the Innovation Engine. When a service API change breaks a document, the job sends the failing document to the ADA API's repair endpoint with the validation errors. The API analyzes the errors, makes intelligent corrections to the document, and returns an updated version that works with the changed service. The system attempts multiple strategies until it finds one that passes all tests. + +**Detailed API Flow:** +1. Operations team has a scheduled Azure Function that runs nightly +2. Function retrieves all published documents via `GET /api/v1/documents?status=published` +3. For each document, it calls `POST /api/v1/documents/{documentId}/validate` +4. When a document fails validation, function calls `POST /api/v1/documents/{documentId}/repair` with: + ```json + { + "validationErrors": ["Error in step 4: Azure CLI command 'az containerapp create' failed with 'unrecognized argument --environment'"], + "userGuidance": "The Container Apps CLI commands may have changed" + } + ``` +5. The API analyzes the error, consults updated Azure CLI documentation, and generates a fix +6. Function retrieves the suggested fix and applies it with `PUT /api/v1/documents/{documentId}` +7. Function then re-validates the document and, if successful, updates the published version +8. If repair fails, function creates a ticket in Azure DevOps with detailed diagnostics +9. Function maintains a revision history of all repairs by calling `GET /api/v1/documents/{documentId}/revisions` +10. Monthly summary reports are generated showing repair success rates and common failure patterns + +## 5. Operations Engineer + +### User Story 9: Security Analysis of Documentation + +As an Operations Engineer, I want to analyze my deployment documents for security vulnerabilities so that I don't inadvertently introduce security risks during deployments + +**Experience:** An ops engineer submits their deployment document to the ADA API's security analysis endpoint. The service returns a comprehensive report identifying several issues: overly permissive access controls, sensitive data not being properly handled, and resources exposed to the internet unnecessarily. The report provides specific remediation steps for each issue with code examples, enabling the engineer to secure their deployment process. + +**Detailed API Flow:** +1. Operations engineer submits their deployment document through the ADA web portal +2. Portal uploads the document via `POST /api/v1/documents` with security scanning flag enabled: + ```json + { + "title": "Production Deployment", + "content": "<>", + "infrastructureType": "terraform", + "securityScanEnabled": true, + "complianceFrameworks": ["NIST", "CIS"] + } + ``` +3. API creates the document and returns the document ID +4. Portal immediately calls `POST /api/v1/documents/{documentId}/securityAnalysis` with: + ```json + { + "depth": "comprehensive", + "includeRemediation": true + } + ``` +5. API analyzes the document against security best practices and selected compliance frameworks +6. Results are returned with specific line numbers and security issues identified +7. For each issue, API provides `POST /api/v1/documents/{documentId}/securityFix/{issueId}` endpoint +8. Engineer reviews each issue and selects which fixes to apply +9. Portal calls the appropriate fix endpoints to apply selected security improvements +10. Final secure document is validated with `POST /api/v1/documents/{documentId}/validate` + +### User Story 10: PII Detection and Redaction + +As an Operations Engineer, I want to automatically detect and redact sensitive information from my deployment logs and documents so that I can safely share them with team members and in support tickets + +Experience: An engineer has troubleshooting documents containing output from production systems. Before sharing them, they submit these documents to the ADA API's redaction endpoint. The service identifies and replaces all subscription IDs, resource names, IP addresses, and other sensitive information with appropriate placeholders. The engineer receives a cleaned document that maintains all the technical context while removing security-sensitive details. + + +**Detailed API Flow:** +1. Operations engineer uses the ADA CLI tool to process sensitive logs +2. They run `ada redact sensitive-logs.md --output=redacted-logs.md --sensitivity=high` +3. CLI tool calls `POST /api/v1/documents` to upload the sensitive document with: + ```json + { + "title": "Sensitive Troubleshooting Logs", + "content": "<>", + "temporary": true, + "retentionPeriod": "1h" + } + ``` +4. Upon successful upload, CLI calls `POST /api/v1/documents/{documentId}/redact` with: + ```json + { + "sensitivityLevel": "high", + "redactionTypes": [ + "subscriptionIds", + "resourceNames", + "ipAddresses", + "connectionStrings", + "emails" + ], + "replacementFormat": "descriptive-placeholder" + } + ``` +5. API processes the document using NER (Named Entity Recognition) to identify sensitive data +6. Redacted document is returned with each sensitive item replaced with a descriptive placeholder +7. CLI saves the redacted content to the output file +8. After redaction is complete, CLI calls `DELETE /api/v1/documents/{documentId}` to ensure sensitive data is removed +9. An audit log of the redaction (without sensitive data) is maintained for compliance purposes + +## 6. Enterprise Architect + +### User Story 11: Custom Documentation Templates + +As an Enterprise Architect, I want to generate documentation that follows our corporate standards and patterns so that all infrastructure documentation is consistent across the organization. + +**Experience:** The architect provides the ADA API with their organization's documentation template and standards along with a workload description. The service generates executable documentation that not only works correctly but follows all company-specific naming conventions, security practices, and formatting guidelines. This ensures consistency across hundreds of projects while maintaining the executable nature of the documents. + +**Detailed API Flow:** +1. Enterprise architect first registers their company template via `POST /api/v1/templates` with: + ```json + { + "name": "Contoso Enterprise Template", + "template": "<>", + "rules": [ + {"type": "naming", "pattern": "contoso-{service}-{env}-{region}"}, + {"type": "security", "rule": "all-resources-require-tagging"}, + {"type": "formatting", "rule": "section-structure-preserved"} + ] + } + ``` +2. API returns a template ID they can reference +3. When generating new documents, architect uses `POST /api/v1/ai/generate` with: + ```json + { + "prompt": "Create infrastructure for a three-tier web application", + "templateId": "template-12345", + "infrastructureType": "terraform", + "organizationSpecificParameters": { + "businessUnit": "finance", + "costCenter": "cc-12345", + "environment": "production" + } + } + ``` +4. API generates documentation following all company-specific naming conventions, security practices, and formatting guidelines +5. Architect reviews and publishes the document with `PUT /api/v1/documents/{documentId}` and `status: "approved"` +6. The document is automatically distributed via webhook to their knowledge management system +7. Monthly template compliance is checked via `GET /api/v1/templates/{templateId}/compliance` \ No newline at end of file From 459bfa5996722c895803f22776420b7f65191005 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Tue, 11 Mar 2025 16:23:08 -0700 Subject: [PATCH 26/87] updated readme with content filters --- tools/user_stories.md | 355 ------------------------------------------ 1 file changed, 355 deletions(-) delete mode 100644 tools/user_stories.md diff --git a/tools/user_stories.md b/tools/user_stories.md deleted file mode 100644 index ed58e4b01..000000000 --- a/tools/user_stories.md +++ /dev/null @@ -1,355 +0,0 @@ -# ADA (AI Documentation Assistant) API User Stories with Detailed Flows - -## 1. Cloud Architect (Aarya) - -### User Story 1: Infrastructure Documentation Generation - -As a Cloud Architect, I want to quickly generate executable documentation for my infrastructure design -So that my operations team can reliably deploy and understand our cloud architecture. - -**Experience:** Aarya accesses the ADA API through the Azure Portal integration. She enters a description of her desired infrastructure - "a highly available web application with Azure Kubernetes Service, Azure Container Registry, and Azure Front Door." The API generates a complete executable document with properly formatted code blocks, environment variables, and step-by-step instructions. Each code block is already validated to work correctly, saving her hours of documentation time while ensuring operational reliability. - -**Detailed API Flow:** -1. Aarya accesses the Azure Portal integration with ADA -2. The portal frontend makes a `POST` request to `/api/v1/ai/generate` with payload: - ```json - { - "prompt": "Create a highly available web application with Azure Kubernetes Service, Azure Container Registry, and Azure Front Door", - "targetEnvironment": "azure", - "infrastructureType": "terraform", - "expertiseLevel": "intermediate", - "additionalContext": "This is for a financial services company with 99.99% uptime requirements" - } - ``` -3. API returns a `202 Accepted` response with `requestId: "gen-12345"` -4. Portal polls `GET /api/v1/ai/generate/gen-12345` every 3 seconds to check generation status -5. When complete, API returns a document object with Terraform infrastructure code and explanations -6. Portal then calls `POST /api/v1/documents/doc-12345/validate` to verify the document works -7. Innovation Engine runs the document in a sandbox environment and returns validation results -8. If validation succeeds, document status is updated to "validated" and displayed to Aarya -9. If validation fails, error details are shown with an option to call `POST /api/v1/documents/doc-12345/repair` - -### User Story 2: Custom Terraform Documentation - -As a Cloud Architect, I want to convert my existing Terraform scripts into educational, executable documentation so that new team members can understand our infrastructure approach while deploying resources. - -**Experience:** Aarya has complex Terraform files that work but lack explanation. She submits them to the ADA API's document conversion endpoint with context about their purpose. The API returns a comprehensive markdown document that preserves all the functional code while adding clear explanations, proper environment variable declarations, appropriate headers, and validation checkpoints that can be executed by Innovation Engine. The document becomes both a deployment tool and training material. - -**Detailed API Flow:** -1. Aarya has existing Terraform files in a GitHub repository -2. She integrates ADA with her CI/CD using a GitHub Action that triggers on PRs to the `/terraform` folder -3. The Action calls `POST /api/v1/documents` with: - ```json - { - "title": "Production Kubernetes Infrastructure", - "description": "Terraform modules for our production Kubernetes deployment", - "content": "<>", - "infrastructureType": "terraform", - "tags": ["production", "kubernetes", "documentation"] - } - ``` -4. The API processes the files and adds explanations between code blocks by analyzing the Terraform structure -5. The Action then calls `GET /api/v1/documents/{documentId}` to retrieve the generated documentation -6. For each code block, the Action verifies syntax by calling `POST /api/v1/documents/{documentId}/validate` with `validateOnly: true` -7. The completed document is committed to a new branch and a PR is created for review -8. Upon approval, the executable documentation is published to their internal knowledge base via webhook - -## 2. Product Manager (Alex) - -### User Story 3: Demo Script to Executable Document - -As a Product Manager, I want to convert my demo scripts into customer-ready, executable documents so that customers can understand and deploy our product features without engineering support - -**Experience:** Alex has created a basic demo script showing a new Azure feature. He calls the ADA API with this script and selects the option to generate dependency files. The API transforms his simple commands into a comprehensive, educational document with proper metadata, environment variables with random suffixes for idempotency, detailed explanations of each step, and expected output blocks for verification. The document automatically passes all innovation engine tests, making it immediately shareable with customers. - -**Detailed API Flow:** -1. Alex uses the ADA CLI tool that wraps the REST API -2. He runs `ada convert demo.sh --target-format=executabledoc --customer-ready` -3. The CLI tool reads his demo.sh file and makes a `POST` request to `/api/v1/documents` with: - ```json - { - "title": "New Feature Demo", - "description": "Demonstration of our new feature X", - "content": "<>", - "infrastructureType": "bash", - "tags": ["demo", "customer-ready"], - "customizationParameters": { - "generateIdempotentResourceNames": true, - "includeExpectedOutput": true, - "addPrerequisiteChecks": true - } - } - ``` -4. API processes the script and returns a document ID -5. CLI tool then calls `POST /api/v1/documents/{documentId}/validate` to test the generated document -6. The API runs the document through Innovation Engine which executes each code block sequentially -7. API captures the output from each step and verifies it matches expected results -8. CLI tool receives the validated document and writes it to `demo-executable.md` -9. Alex reviews the document with embedded expected outputs and resource name randomization - -### User Story 4: User Experience Documentation - -As a Product Manager, I want to document customer experiences in an executable format early in the development process so that engineering teams can understand the expected behavior and validate it works as designed. - -**Experience:** Alex creates a basic description of a new feature workflow, then submits it to the ADA API. The service generates a document-driven design specification that includes both narrative explanation and executable code. Engineers can run this document to see the expected behavior, while Alex can verify the feature matches his design intent. The document becomes both a specification and a validation tool. - -**Detailed API Flow:** -1. Alex uses the Azure Portal ADA integration -2. He creates a new document description and submits it through a form that calls `POST /api/v1/ai/generate` -3. The request includes: - ```json - { - "prompt": "Document the user experience for containerizing and deploying a Node.js app to AKS", - "targetEnvironment": "azure", - "infrastructureType": "azcli", - "expertiseLevel": "beginner", - "additionalContext": "Target audience is developers with no Kubernetes experience" - } - ``` -4. The API generates a document and returns a document ID -5. The portal displays a preview and Alex makes edits through the UI -6. Each edit triggers a `PUT /api/v1/documents/{documentId}` call with the updated content -7. Alex shares the document with engineering by clicking "Share", which calls `GET /api/v1/documents/{documentId}?format=markdown` -8. The API returns a shareable link with appropriate permissions set -9. Engineering team members access the document and can execute it section-by-section using the Innovation Engine integration - -## 3. Content Author - -### User Story 5: Automated Documentation Testing - -As a Content Author, I want to automatically test my infrastructure documentation for errors so that customers don't encounter issues when following our guidance. - -**Experience:** A content author submits their markdown document to the ADA API's validation endpoint. The service identifies several issues: missing environment variables, commands that would fail with certain Azure regions, and dependencies not properly defined. The API automatically fixes these issues through multiple attempts, each time applying more sophisticated troubleshooting strategies until all tests pass. The author receives a fully functional document with detailed explanations of what was fixed. - -**Detailed API Flow:** -1. Content author submits a document through the MS Learn authoring tool -2. The tool calls `POST /api/v1/documents` with the markdown content -3. After creation, it immediately calls `POST /api/v1/documents/{documentId}/validate` with: - ```json - { - "environmentParameters": { - "azureRegion": "eastus", - "subscriptionType": "pay-as-you-go" - }, - "validateOnly": false - } - ``` -4. The API returns validation status with error details for failing steps -5. For each error, the tool calls `POST /api/v1/documents/{documentId}/repair` with: - ```json - { - "validationErrors": ["Step 3 failed: Error: Resource group name already exists"], - "userGuidance": "Please fix the resource naming to be unique" - } - ``` -6. The API returns suggested fixes and the tool applies them with `PUT /api/v1/documents/{documentId}` -7. This process repeats until all validation passes or user intervention is required -8. The tool then executes a final validation call with different parameters to test idempotency: - ```json - { - "environmentParameters": { - "azureRegion": "westeurope", - "subscriptionType": "enterprise" - }, - "validateOnly": false - } - ``` -9. Once all validations pass, the document is marked as validated and ready for publishing - -### User Story 6: Shell Script Documentation - -As a Content Author, I want to convert technical shell scripts into educational documents so that users understand what each part of the script accomplishes. - -**Experience:** The author has a complex deployment shell script but no documentation. They call the ADA API's script documentation endpoint with the script path and receive a fully structured markdown document with proper headings, detailed explanations between code blocks, all required metadata, and preserved functionality. The document explains each section's purpose while maintaining all the original functionality. - -**Detailed API Flow:** -1. Content author uses the ADA VS Code Extension -2. They open their complex deployment shell script and right-click to select "Generate Executable Documentation" -3. The extension calls `POST /api/v1/ai/generate` with: - ```json - { - "prompt": "Convert this shell script into an educational executable document", - "targetEnvironment": "bash", - "infrastructureType": "bash", - "additionalContext": "<>", - "expertiseLevel": "beginner" - } - ``` -4. The API processes the request and returns a generated document -5. VS Code extension displays the document in a side panel with syntax highlighting -6. Author makes edits which are synchronized with the API via `PUT /api/v1/documents/{documentId}` -7. They click "Test" in the extension, which calls `POST /api/v1/documents/{documentId}/validate` -8. The API runs each code block in isolation and returns success/error for each section -9. Author can see exactly which parts of their script need improvement before publishing -10. Final document is exported as markdown with embedded executable code blocks - -## 4. Developer/Engineer - -### User Story 7: API-Driven Document Generation Pipeline - -As a Developer, I want to integrate the ADA API into our CI/CD pipeline so that our infrastructure documentation is automatically generated and tested alongside code changes. - -**Experience:** A developer creates a GitHub Action that calls the ADA API whenever infrastructure code is changed. The action submits the updated Terraform files to the API, which generates an updated executable document. This document is then automatically tested using the Innovation Engine. If validation passes, the updated documentation is published to their internal knowledge base. This ensures their documentation always reflects the current infrastructure state. - -**Detailed API Flow:** -1. Developer creates a GitHub Action workflow file (.github/workflows/update-docs.yml) -2. The workflow triggers whenever infrastructure files change in the repo -3. Action authenticates with Azure AD and obtains a token for the ADA API -4. It identifies changed Terraform files and calls `POST /api/v1/documents` with: - ```json - { - "title": "Infrastructure Documentation", - "description": "Auto-generated from CI/CD pipeline", - "content": "<>", - "infrastructureType": "terraform", - "tags": ["cicd", "auto-generated"] - } - ``` -5. The API processes the Terraform files and generates a structured document -6. Action then calls `POST /api/v1/documents/{documentId}/validate` to verify functionality -7. If validation succeeds, it calls `GET /api/v1/documents/{documentId}?format=markdown` -8. The action commits the generated markdown to the repo's `/docs` folder -9. For any validation failures, it opens an issue with details from the validation response -10. A scheduled job runs weekly to revalidate all documents by calling: - ``` - GET /api/v1/documents?tag=cicd&status=validated - ``` - And then validating each document to ensure continued functionality - -### User Story 8: Self-Healing Documentation - -As a Developer, I want to automatically fix broken documentation when changes occur in the underlying cloud services so that our documents remain functional even when Azure features evolve - -**Experience:** The developer has scheduled a job that periodically validates all executive documents against the Innovation Engine. When a service API change breaks a document, the job sends the failing document to the ADA API's repair endpoint with the validation errors. The API analyzes the errors, makes intelligent corrections to the document, and returns an updated version that works with the changed service. The system attempts multiple strategies until it finds one that passes all tests. - -**Detailed API Flow:** -1. Operations team has a scheduled Azure Function that runs nightly -2. Function retrieves all published documents via `GET /api/v1/documents?status=published` -3. For each document, it calls `POST /api/v1/documents/{documentId}/validate` -4. When a document fails validation, function calls `POST /api/v1/documents/{documentId}/repair` with: - ```json - { - "validationErrors": ["Error in step 4: Azure CLI command 'az containerapp create' failed with 'unrecognized argument --environment'"], - "userGuidance": "The Container Apps CLI commands may have changed" - } - ``` -5. The API analyzes the error, consults updated Azure CLI documentation, and generates a fix -6. Function retrieves the suggested fix and applies it with `PUT /api/v1/documents/{documentId}` -7. Function then re-validates the document and, if successful, updates the published version -8. If repair fails, function creates a ticket in Azure DevOps with detailed diagnostics -9. Function maintains a revision history of all repairs by calling `GET /api/v1/documents/{documentId}/revisions` -10. Monthly summary reports are generated showing repair success rates and common failure patterns - -## 5. Operations Engineer - -### User Story 9: Security Analysis of Documentation - -As an Operations Engineer, I want to analyze my deployment documents for security vulnerabilities so that I don't inadvertently introduce security risks during deployments - -**Experience:** An ops engineer submits their deployment document to the ADA API's security analysis endpoint. The service returns a comprehensive report identifying several issues: overly permissive access controls, sensitive data not being properly handled, and resources exposed to the internet unnecessarily. The report provides specific remediation steps for each issue with code examples, enabling the engineer to secure their deployment process. - -**Detailed API Flow:** -1. Operations engineer submits their deployment document through the ADA web portal -2. Portal uploads the document via `POST /api/v1/documents` with security scanning flag enabled: - ```json - { - "title": "Production Deployment", - "content": "<>", - "infrastructureType": "terraform", - "securityScanEnabled": true, - "complianceFrameworks": ["NIST", "CIS"] - } - ``` -3. API creates the document and returns the document ID -4. Portal immediately calls `POST /api/v1/documents/{documentId}/securityAnalysis` with: - ```json - { - "depth": "comprehensive", - "includeRemediation": true - } - ``` -5. API analyzes the document against security best practices and selected compliance frameworks -6. Results are returned with specific line numbers and security issues identified -7. For each issue, API provides `POST /api/v1/documents/{documentId}/securityFix/{issueId}` endpoint -8. Engineer reviews each issue and selects which fixes to apply -9. Portal calls the appropriate fix endpoints to apply selected security improvements -10. Final secure document is validated with `POST /api/v1/documents/{documentId}/validate` - -### User Story 10: PII Detection and Redaction - -As an Operations Engineer, I want to automatically detect and redact sensitive information from my deployment logs and documents so that I can safely share them with team members and in support tickets - -Experience: An engineer has troubleshooting documents containing output from production systems. Before sharing them, they submit these documents to the ADA API's redaction endpoint. The service identifies and replaces all subscription IDs, resource names, IP addresses, and other sensitive information with appropriate placeholders. The engineer receives a cleaned document that maintains all the technical context while removing security-sensitive details. - - -**Detailed API Flow:** -1. Operations engineer uses the ADA CLI tool to process sensitive logs -2. They run `ada redact sensitive-logs.md --output=redacted-logs.md --sensitivity=high` -3. CLI tool calls `POST /api/v1/documents` to upload the sensitive document with: - ```json - { - "title": "Sensitive Troubleshooting Logs", - "content": "<>", - "temporary": true, - "retentionPeriod": "1h" - } - ``` -4. Upon successful upload, CLI calls `POST /api/v1/documents/{documentId}/redact` with: - ```json - { - "sensitivityLevel": "high", - "redactionTypes": [ - "subscriptionIds", - "resourceNames", - "ipAddresses", - "connectionStrings", - "emails" - ], - "replacementFormat": "descriptive-placeholder" - } - ``` -5. API processes the document using NER (Named Entity Recognition) to identify sensitive data -6. Redacted document is returned with each sensitive item replaced with a descriptive placeholder -7. CLI saves the redacted content to the output file -8. After redaction is complete, CLI calls `DELETE /api/v1/documents/{documentId}` to ensure sensitive data is removed -9. An audit log of the redaction (without sensitive data) is maintained for compliance purposes - -## 6. Enterprise Architect - -### User Story 11: Custom Documentation Templates - -As an Enterprise Architect, I want to generate documentation that follows our corporate standards and patterns so that all infrastructure documentation is consistent across the organization. - -**Experience:** The architect provides the ADA API with their organization's documentation template and standards along with a workload description. The service generates executable documentation that not only works correctly but follows all company-specific naming conventions, security practices, and formatting guidelines. This ensures consistency across hundreds of projects while maintaining the executable nature of the documents. - -**Detailed API Flow:** -1. Enterprise architect first registers their company template via `POST /api/v1/templates` with: - ```json - { - "name": "Contoso Enterprise Template", - "template": "<>", - "rules": [ - {"type": "naming", "pattern": "contoso-{service}-{env}-{region}"}, - {"type": "security", "rule": "all-resources-require-tagging"}, - {"type": "formatting", "rule": "section-structure-preserved"} - ] - } - ``` -2. API returns a template ID they can reference -3. When generating new documents, architect uses `POST /api/v1/ai/generate` with: - ```json - { - "prompt": "Create infrastructure for a three-tier web application", - "templateId": "template-12345", - "infrastructureType": "terraform", - "organizationSpecificParameters": { - "businessUnit": "finance", - "costCenter": "cc-12345", - "environment": "production" - } - } - ``` -4. API generates documentation following all company-specific naming conventions, security practices, and formatting guidelines -5. Architect reviews and publishes the document with `PUT /api/v1/documents/{documentId}` and `status: "approved"` -6. The document is automatically distributed via webhook to their knowledge management system -7. Monthly template compliance is checked via `GET /api/v1/templates/{templateId}/compliance` \ No newline at end of file From 35ec0f1585512cb11e947cbbaafe5033aedb9428 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 11:29:58 -0700 Subject: [PATCH 27/87] updated documentation --- README.md | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 13923b460..686e991cb 100644 --- a/README.md +++ b/README.md @@ -36,43 +36,56 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters **Example:** ```markdown ```bash - az group create --name myResourceGroup --location eastus + export REGION="eastus" + export RESOURCE_GROUP="myResourceGroup" + az group create --name $RESOURCE_GROUP --location $REGION ``` ``` - >**Note:** This rule does not apply to output code blocks, which are used to display the results of commands, scripts, or other operations. These blocks help in illustrating what the expected output should look like. They include, but are not limited to, the following types: _output, json, yaml, console, text, and log._ - - >**Note:** While Innovation Engine can _parse_ a code block of any type, given its current features, it can only _execute_ code blocks of the types above. So, it is important to ensure that the code blocks in your Exec Doc are of the types above. + >**Note:** You can include code blocks of any type in your documentation for human readers, but only the types listed above will be executed by Innovation Engine. Other code block types will be displayed but ignored during execution. + + >**Note:** There is a special kind of code block called a "result block" that's used to validate command execution. We'll cover result blocks in detail later in section 11. 2. **Command Execution Limitations** - - **Not supported for direct execution:** + - **Not supported:** - PowerShell scripts - GUI-based instructions - - Direct code blocks containing Python, SQL, or other languages (these should be executed via BASH commands) + - Commands requiring `sudo` privileges + - Direct code blocks of languages that aren't bash/shell commands - - **Supported execution context:** - - Commands that run in a Linux/bash environment + - **Supported:** + - Any command that can run in a BASH terminal - Azure CLI commands - - Terraform commands (works without any special setup) + - Terraform commands - Python scripts executed via BASH (e.g., `python myApp.py`) - SQL queries executed via database CLI tools **Example of supported command:** ```markdown ```bash - export VM_NAME="myVM" - az vm create --name $VM_NAME --resource-group myResourceGroup --image UbuntuLTS + export VM_NAME="my-virtual-machine" + export RESOURCE_GROUP="my-resource-group" + az vm create --name $VM_NAME --resource-group $RESOURCE_GROUP --image UbuntuLTS ``` ``` - **Example of unsupported command:** - ```markdown - ```sql - SELECT * FROM myTable WHERE id = 1; - ``` - ``` + **Example of unsupported SQL query (won't work):** + ```markdown + ```sql + INSERT INTO myTable (name, value) VALUES ('test', 123); + ``` + ``` + + **Example of supported SQL command (will work):** + ```markdown + ```bash + export DATABASE_NAME="mydb" + export TABLE_NAME="myTable" + psql -d $DATABASE_NAME -c "INSERT INTO $TABLE_NAME (name, value) VALUES ('test', 123);" + ``` + ``` - >**Note:** The key principle is that if a code block can be executed in a BASH terminal as written (the way a human would execute it), then it will work with Exec Docs. + >**Note:** The key principle is simple: if you can run it in a BASH terminal as written, it will work with Exec Docs (although at this time `sudo` is not supported). Code blocks in other languages won't be executed directly but can be included for human readers. 3. **Azure Portal Custom Cloud Shell Constraints** - **Supported scenarios:** From 71fad605d1936dc4210aa5f26586d1cc746b38b2 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 13:21:52 -0700 Subject: [PATCH 28/87] updated documentation --- .../obtain-performance-metrics-linux-system.md | 5 ++++- scenarios/metadata.json | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scenarios/ObtainPerformanceMetricsLinuxSustem/obtain-performance-metrics-linux-system.md b/scenarios/ObtainPerformanceMetricsLinuxSustem/obtain-performance-metrics-linux-system.md index 7e8499928..2424ff0dd 100644 --- a/scenarios/ObtainPerformanceMetricsLinuxSustem/obtain-performance-metrics-linux-system.md +++ b/scenarios/ObtainPerformanceMetricsLinuxSustem/obtain-performance-metrics-linux-system.md @@ -50,7 +50,10 @@ export MY_VM_NAME="myVM89f292" The full command for installation of the `sysstat` package on some popular Distros is: ```bash -az vm run-command invoke --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --command-id RunShellScript --scripts "/bin/bash -c 'OS=\$(cat /etc/os-release|grep NAME|head -1|cut -d= -f2 | sed \"s/\\\"//g\"); if [[ \$OS =~ \"Ubuntu\" ]] || [[ \$OS =~ \"Debian\" ]]; then sudo apt install sysstat -y; elif [[ \$OS =~ \"Red Hat\" ]]; then sudo dnf install sysstat -y; elif [[ \$OS =~ \"SUSE\" ]]; then sudo zypper install sysstat --non-interactive; else echo \"Unknown distribution\"; fi'" +output=$(az vm run-command invoke --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --command-id RunShellScript --scripts "/bin/bash -c 'OS=\$(cat /etc/os-release|grep NAME|head -1|cut -d= -f2 | sed \"s/\\\"//g\"); if [[ \$OS =~ \"Ubuntu\" ]] || [[ \$OS =~ \"Debian\" ]]; then sudo apt install sysstat -y; elif [[ \$OS =~ \"Red Hat\" ]]; then sudo dnf install sysstat -y; elif [[ \$OS =~ \"SUSE\" ]]; then sudo zypper install sysstat --non-interactive; else echo \"Unknown distribution\"; fi'") +value=$(echo "$output" | jq -r '.value[0].message') +extracted=$(echo "$value" | awk '/\[stdout\]/,/\[stderr\]/' | sed '/\[stdout\]/d' | sed '/\[stderr\]/d') +echo "$extracted" ``` ## CPU diff --git a/scenarios/metadata.json b/scenarios/metadata.json index a17768364..b1045bd87 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -949,8 +949,8 @@ { "status": "active", "key": "KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md", - "title": "Troubleshoot Linux VM boot issues due to fstab errors", - "description": "Explains why Linux VM cannot start and how to solve the problem.", + "title": "Recover Azure Linux VM from kernel panic due to missing initramfs", + "description": "Provides solutions to an issue in which a Linux virtual machine (VM) can't boot after applying kernel changes.", "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md", "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/kernel-related-boot-issues#missing-initramfs-alar", From 7722d6a60f6f6fc96c2f5c6679beff16f33ff11f Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 13:25:31 -0700 Subject: [PATCH 29/87] updated documentation --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 686e991cb..30a160355 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,6 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters ``` >**Note:** You can include code blocks of any type in your documentation for human readers, but only the types listed above will be executed by Innovation Engine. Other code block types will be displayed but ignored during execution. - - >**Note:** There is a special kind of code block called a "result block" that's used to validate command execution. We'll cover result blocks in detail later in section 11. 2. **Command Execution Limitations** - **Not supported:** @@ -102,15 +100,18 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters **Example of supported command:** ```markdown ```bash - az group create --name myResourceGroup --location eastus + export RESOURCE_GROUP="myResourceGroup" + export LOCATION="eastus" + az group create --name $RESOURCE_GROUP --location $LOCATION ``` ``` **Example of unsupported command:** ```markdown ```bash + export APP_NAME="myApp" # This requires elevated Graph API permissions and would fail - az ad app create --display-name myApp --native-app + az ad app create --display-name $APP_NAME --native-app ``` ``` From 0042b2321c2165a469f7559de88eebb88de4fea8 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 13:28:35 -0700 Subject: [PATCH 30/87] updated documentation --- scenarios/metadata.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/metadata.json b/scenarios/metadata.json index b1045bd87..0ed737882 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1113,8 +1113,8 @@ { "status": "active", "key": "azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md", - "title": "Deploy container group to Azure virtual network", - "description": "Learn how to deploy a container group to a new or existing Azure virtual network via the Azure CLI.", + "title": "Create a Linux VM in Azure with multiple NICs", + "description": "Learn how to create a Linux VM with multiple NICs attached to it using the Azure CLI or Resource Manager templates.", "stackDetails": [ ], "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md", From 308232e05a929aa038fda63b3a525a3e1ccadbc2 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 13:43:45 -0700 Subject: [PATCH 31/87] updated documentation --- README.md | 44 ++++++++++++-------------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 30a160355..4ddbbfec4 100644 --- a/README.md +++ b/README.md @@ -25,56 +25,35 @@ I'll update the highlighted section with the clarified information about command Not all documentation is suitable for conversion to Exec Docs. Use these filters to determine if a document can be effectively converted: -1. **Supported Code Block Types** - - The document must contain code blocks using at least one of these types: - - `bash` - - `azurecli` - - `azure-cli-interactive` - - `azurecli-interactive` - - `terraform` - - **Example:** - ```markdown - ```bash - export REGION="eastus" - export RESOURCE_GROUP="myResourceGroup" - az group create --name $RESOURCE_GROUP --location $REGION - ``` - ``` - - >**Note:** You can include code blocks of any type in your documentation for human readers, but only the types listed above will be executed by Innovation Engine. Other code block types will be displayed but ignored during execution. +1. **Command Execution Limitations** + - **Supported:** + - Any command that can run in a BASH terminal + - Azure CLI commands (e.g. azurecli, azure-cli-interactive, azurecli-interactive) + - Terraform commands -2. **Command Execution Limitations** - **Not supported:** - PowerShell scripts - GUI-based instructions - Commands requiring `sudo` privileges - Direct code blocks of languages that aren't bash/shell commands - - **Supported:** - - Any command that can run in a BASH terminal - - Azure CLI commands - - Terraform commands - - Python scripts executed via BASH (e.g., `python myApp.py`) - - SQL queries executed via database CLI tools - **Example of supported command:** ```markdown ```bash - export VM_NAME="my-virtual-machine" - export RESOURCE_GROUP="my-resource-group" - az vm create --name $VM_NAME --resource-group $RESOURCE_GROUP --image UbuntuLTS + export REGION="eastus" + export RESOURCE_GROUP="myResourceGroup" + az group create --name $RESOURCE_GROUP --location $REGION ``` ``` - **Example of unsupported SQL query (won't work):** + **Example of unsupported command (SQL query below won't work):** ```markdown ```sql INSERT INTO myTable (name, value) VALUES ('test', 123); ``` ``` - **Example of supported SQL command (will work):** + **Example of supported command (SQL query below will work):** ```markdown ```bash export DATABASE_NAME="mydb" @@ -82,10 +61,11 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters psql -d $DATABASE_NAME -c "INSERT INTO $TABLE_NAME (name, value) VALUES ('test', 123);" ``` ``` + >**Note:** You can include code blocks of any type in your documentation for human readers, but only the types listed above in the "Supported Code Block Types" section will be executed by Innovation Engine. Other code block types will be displayed but ignored during execution. >**Note:** The key principle is simple: if you can run it in a BASH terminal as written, it will work with Exec Docs (although at this time `sudo` is not supported). Code blocks in other languages won't be executed directly but can be included for human readers. -3. **Azure Portal Custom Cloud Shell Constraints** +2. **Azure Portal Custom Cloud Shell Constraints** - **Supported scenarios:** - Standard Azure resource operations (create, read, update, delete) - Commands running within the user's subscription scope From 21a854ec96c90008f21023fa2975d00148c1abad Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 13:48:38 -0700 Subject: [PATCH 32/87] updated documentation --- .../kernel-related-boot-issues-repairvm.md | 4 ++-- .../troubleshoot-vm-grub-error-repairvm.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md b/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md index 354e5e38e..3b230795c 100644 --- a/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md +++ b/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md @@ -32,7 +32,7 @@ export MY_VM_NAME="myVM89f292" Make sure the [serial console](serial-console-linux.md) is enabled and functional in the Linux VM. -## Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0) +## Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0) This error occurs because of a recent system update (kernel). It's most commonly seen in RHEL-based distributions. You can [identify this issue from the Azure serial console](#identify-kernel-boot-issue). You'll see any of the following error messages: @@ -64,7 +64,7 @@ You can [identify this issue from the Azure serial console](#identify-kernel-boo This kind of error indicates that the initramfs file isn't generated, the GRUB configuration file has the initrd entry missing after a patching process, or a GRUB manual misconfiguration. -### Regenerate missing initramfs by using Azure Repair VM ALAR scripts +### Regenerate missing initramfs by using Azure Repair VM ALAR scripts 1. Create a repair VM by running the following Bash command line with [Azure Cloud Shell](/azure/cloud-shell/overview). For more information, see [Use Azure Linux Auto Repair (ALAR) to fix a Linux VM - initrd option](repair-linux-vm-using-ALAR.md#initrd). This command will regenerate the initrd/initramfs image, regenerate the GRUB configuration file if it has the initrd entry missing, and swap the OS disk diff --git a/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md b/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md index 48750ef8e..2bbc7dc15 100644 --- a/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md +++ b/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md @@ -35,7 +35,7 @@ This article discusses multiple conditions that cause GRUB rescue issues and pro During the boot process, the boot loader tries to locate the Linux kernel and hand off the boot control. If this handoff can't be performed, the virtual machine (VM) enters a GRUB rescue console. The GRUB rescue console prompt isn't shown in the Azure serial console log, but it can be shown in the [Azure boot diagnostics screenshot](/azure/virtual-machines/boot-diagnostics#boot-diagnostics-view). -## Identify GRUB rescue issue +## Identify GRUB rescue issue [View a boot diagnostics screenshot](/azure/virtual-machines/boot-diagnostics#boot-diagnostics-view) in the VM **Boot diagnostics** page of the Azure portal. This screenshot helps diagnose the GRUB rescue issue and determine if a boot error causes the issue. @@ -47,7 +47,7 @@ Entering rescue mode... grub rescue> ``` -## Troubleshoot GRUB rescue issue offline +## Troubleshoot GRUB rescue issue offline 1. To troubleshoot a GRUB rescue issue, a rescue/repair VM is required. Use [vm repair commands](repair-linux-vm-using-azure-virtual-machine-repair-commands.md) to create a repair VM that has a copy of the affected VM's OS disk attached. Mount the copy of the OS file systems in the repair VM by using [chroot](chroot-environment-linux.md). @@ -78,7 +78,7 @@ See the following sections for detailed errors, possible causes, and solutions. > [!NOTE] > In the commands mentioned in the following sections, replace `/dev/sdX` with the corresponding Operating System (OS) disk device. -### Reinstall GRUB and regenerate GRUB configuration file using Auto Repair (ALAR) +### Reinstall GRUB and regenerate GRUB configuration file using Auto Repair (ALAR) Azure Linux Auto Repair (ALAR) scripts are part of the VM repair extension described in [Use Azure Linux Auto Repair (ALAR) to fix a Linux VM](./repair-linux-vm-using-alar.md). ALAR covers the automation of multiple repair scenarios, including GRUB rescue issues. From f7430a9fb7e56f4603b8a2a44585b736130ebe48 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 14:01:22 -0700 Subject: [PATCH 33/87] updated documentation --- README.md | 4 +--- .../tutorial-modify-scale-sets-cli.md | 7 ++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4ddbbfec4..1903fd7e7 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,7 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters 1. **Command Execution Limitations** - **Supported:** - - Any command that can run in a BASH terminal - - Azure CLI commands (e.g. azurecli, azure-cli-interactive, azurecli-interactive) - - Terraform commands + - Any command that can run in a BASH terminal (e.g. azurecli, azure-cli-interactive, azurecli-interactive, terraform commands) - **Not supported:** - PowerShell scripts diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md index 1ffa46d52..6977bb9b4 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md @@ -287,7 +287,8 @@ These properties describe the configuration of a VM instance within a scale set, You can perform updates to individual VM instances in a scale set just like you would a standalone VM. For example, attaching a new data disk to instance 1: ```azurecli-interactive -az vm disk attach --resource-group $MY_RESOURCE_GROUP_NAME --vm-name $INSTANCE_NAME --name disk_name1 --new +export DISK_NAME="disk_name$RANDOM_SUFFIX" +az vm disk attach --resource-group $MY_RESOURCE_GROUP_NAME --vm-name $INSTANCE_NAME --name $DISK_NAME --new ``` Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the VM instance has the new disk attached. @@ -303,11 +304,11 @@ Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the V "diskSizeGb": 1023, "lun": 0, "managedDisk": { - "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/disks/disk_name1", + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/disks/disk_namexxxx", "resourceGroup": "myResourceGroupxxx", "storageAccountType": "Premium_LRS" }, - "name": "disk_name1", + "name": "disk_namexxxx", "toBeDetached": false } ] From 63865223d3ae2651cf12ec0d15fd8b2078c30217 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 14:03:22 -0700 Subject: [PATCH 34/87] updated documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1903fd7e7..109e8a71c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Not all documentation is suitable for conversion to Exec Docs. Use these filters 1. **Command Execution Limitations** - **Supported:** - - Any command that can run in a BASH terminal (e.g. azurecli, azure-cli-interactive, azurecli-interactive, terraform commands) + - Any command that can run in a BASH terminal (e.g. azurecli, azure-cli-interactive, azurecli-interactive commands) - **Not supported:** - PowerShell scripts From 42f76a8bf2d7ab4598b6d3d6e046b79ca06836db Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 15:16:33 -0700 Subject: [PATCH 35/87] updated documentation --- .../articles/container-instances/container-instances-vnet.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md index 06ca507fd..96f08fec8 100644 --- a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md @@ -99,6 +99,7 @@ az container create \ --vnet-address-prefix 10.0.0.0/16 \ --subnet $MY_SUBNET_NAME \ --subnet-address-prefix 10.0.0.0/24 + --os-type Linux ``` A successful operation should produce output similar to the following JSON: @@ -286,7 +287,7 @@ Deploy the container group with the [az container create][az-container-create] c ```azurecli-interactive az container create --resource-group $MY_RESOURCE_GROUP_NAME \ - --file container-instances-vnet.yaml + --file container-instances-vnet.yaml --os-type Linux ``` The following Bash command is for the automated deployment pathway. @@ -360,6 +361,7 @@ az container create \ --restart-policy never \ --vnet $MY_VNET_NAME \ --subnet $MY_SUBNET_NAME + --os-type Linux ``` After this second container deployment completes, pull its logs so you can see the output of the `wget` command it executed: From edabbaa56999aa3ee5d759905b25c93ed42cf0de Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 15:22:03 -0700 Subject: [PATCH 36/87] updated documentation --- .../tutorial-modify-scale-sets-cli.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md index 6977bb9b4..69937aaf2 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md @@ -199,7 +199,8 @@ az vmss create \ --admin-username azureuser \ --generate-ssh-keys \ --instance-count 5 \ - --os-disk-size-gb 64 + --os-disk-size-gb 64 \ + --admin-username azureuser ``` ## Updating individual VM instances in a scale set @@ -320,7 +321,7 @@ Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the V There are times where you might want to add a new VM to your scale set but want different configuration options than those listed in the scale set model. VMs can be added to a scale set during creation by using the [az vm create](/cli/azure/vmss#az-vmss-create) command and specifying the scale set name you want the instance added to. ```azurecli-interactive -az vm create --name $NEW_INSTANCE_NAME --resource-group $MY_RESOURCE_GROUP_NAME --vmss $SCALE_SET_NAME --image RHELRaw8LVMGen2 +az vm create --name $NEW_INSTANCE_NAME --resource-group $MY_RESOURCE_GROUP_NAME --vmss $SCALE_SET_NAME --image RHELRaw8LVMGen2 --admin-username azureuser ``` ```output From 682fee71b9e1cd33d103343f5de1910f5cd6ebc5 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 15:32:00 -0700 Subject: [PATCH 37/87] updated documentation --- .../container-instances/container-instances-vnet.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md index 96f08fec8..ea04015cf 100644 --- a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md @@ -98,7 +98,7 @@ az container create \ --vnet $MY_VNET_NAME \ --vnet-address-prefix 10.0.0.0/16 \ --subnet $MY_SUBNET_NAME \ - --subnet-address-prefix 10.0.0.0/24 + --subnet-address-prefix 10.0.0.0/24 \ --os-type Linux ``` @@ -287,7 +287,8 @@ Deploy the container group with the [az container create][az-container-create] c ```azurecli-interactive az container create --resource-group $MY_RESOURCE_GROUP_NAME \ - --file container-instances-vnet.yaml --os-type Linux + --file container-instances-vnet.yaml \ + --os-type Linux ``` The following Bash command is for the automated deployment pathway. @@ -360,7 +361,7 @@ az container create \ --command-line "wget 10.0.0.4" \ --restart-policy never \ --vnet $MY_VNET_NAME \ - --subnet $MY_SUBNET_NAME + --subnet $MY_SUBNET_NAME \ --os-type Linux ``` From 7a8dbba8e249c119ad62276c290421ea79a6aa3f Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 15:34:09 -0700 Subject: [PATCH 38/87] updated documentation --- .../tutorial-modify-scale-sets-cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md index 69937aaf2..ed996f0ef 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md @@ -16,7 +16,7 @@ Throughout the lifecycle of your applications, you may need to modify or update Below, we declare environment variables that will be used throughout this document. A random suffix is appended to resource names that need to be unique for each deployment. The REGION is set to WestUS2. ```bash -export RANDOM_SUFFIX=adcc95 +export RANDOM_SUFFIX=$(openssl rand -hex 3) export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" export SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" export NEW_INSTANCE_NAME="myNewInstance$RANDOM_SUFFIX" From f184fb4dfaa5decc2d7f857fb51e2781b090596a Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 15:37:42 -0700 Subject: [PATCH 39/87] updated documentation --- .../container-instances/container-instances-vnet.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md index ea04015cf..ef4d36fa2 100644 --- a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md @@ -99,7 +99,9 @@ az container create \ --vnet-address-prefix 10.0.0.0/16 \ --subnet $MY_SUBNET_NAME \ --subnet-address-prefix 10.0.0.0/24 \ - --os-type Linux + --os-type Linux \ + --cpu 1.0 \ + --memory 1.5 ``` A successful operation should produce output similar to the following JSON: @@ -362,7 +364,9 @@ az container create \ --restart-policy never \ --vnet $MY_VNET_NAME \ --subnet $MY_SUBNET_NAME \ - --os-type Linux + --os-type Linux \ + --cpu 1.0 \ + --memory 1.5 ``` After this second container deployment completes, pull its logs so you can see the output of the `wget` command it executed: From aa56f28d83fc432331d957492094555ab617fd03 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 21:16:40 -0700 Subject: [PATCH 40/87] updated commands --- .../articles/container-instances/container-instances-vnet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md index ef4d36fa2..8451b53cd 100644 --- a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md @@ -359,7 +359,7 @@ Now, set `CONTAINER_GROUP_IP` to the IP you retrieved with the `az container sho az container create \ --resource-group $MY_RESOURCE_GROUP_NAME \ --name $MY_COMM_CHECKER_NAME \ - --image alpine:3.4 \ + --image mcr.microsoft.com/azuredocs/aci-helloworld \ --command-line "wget 10.0.0.4" \ --restart-policy never \ --vnet $MY_VNET_NAME \ From 0a6bcdb2b9a52677ebdd0ad602bdaf86e9cfe38a Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 12 Mar 2025 22:36:55 -0700 Subject: [PATCH 41/87] updated doc --- .../container-instances/container-instances-vnet.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md index 8451b53cd..ec4f35ee9 100644 --- a/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-vnet.md @@ -40,6 +40,7 @@ export MY_SUBNET_ID="/subscriptions/$(az account show --query id --output tsv)/r export MY_APP_CONTAINER_NAME="appcontainer" export MY_COMM_CHECKER_NAME="commchecker" export MY_YAML_APP_CONTAINER_NAME="appcontaineryaml" +export MY_REGION="eastus2" ``` ### Create a resource group @@ -47,7 +48,7 @@ export MY_YAML_APP_CONTAINER_NAME="appcontaineryaml" You need a resource group to manage all the resources used in the following examples. To create a resource group, use [az group create][az-group-create]: ```azurecli-interactive -az group create --name $MY_RESOURCE_GROUP_NAME --location eastus +az group create --name $MY_RESOURCE_GROUP_NAME --location $MY_REGION ``` A successful operation should produce output similar to the following JSON: @@ -282,7 +283,7 @@ type: Microsoft.ContainerInstance/containerGroups The following Bash command is for the automated deployment pathway. ```bash -echo -e "apiVersion: '2021-07-01'\nlocation: eastus\nname: $MY_YAML_APP_CONTAINER_NAME\nproperties:\n containers:\n - name: $MY_YAML_APP_CONTAINER_NAME\n properties:\n image: mcr.microsoft.com/azuredocs/aci-helloworld\n ports:\n - port: 80\n protocol: TCP\n resources:\n requests:\n cpu: 1.0\n memoryInGB: 1.5\n ipAddress:\n type: Private\n ports:\n - protocol: tcp\n port: '80'\n osType: Linux\n restartPolicy: Always\n subnetIds:\n - id: $MY_SUBNET_ID\n name: default\ntags: null\ntype: Microsoft.ContainerInstance/containerGroups" > container-instances-vnet.yaml +echo -e "apiVersion: '2021-07-01'\nlocation: $MY_REGION\nname: $MY_YAML_APP_CONTAINER_NAME\nproperties:\n containers:\n - name: $MY_YAML_APP_CONTAINER_NAME\n properties:\n image: mcr.microsoft.com/azuredocs/aci-helloworld\n ports:\n - port: 80\n protocol: TCP\n resources:\n requests:\n cpu: 1.0\n memoryInGB: 1.5\n ipAddress:\n type: Private\n ports:\n - protocol: tcp\n port: '80'\n osType: Linux\n restartPolicy: Always\n subnetIds:\n - id: $MY_SUBNET_ID\n name: default\ntags: null\ntype: Microsoft.ContainerInstance/containerGroups" > container-instances-vnet.yaml ``` Deploy the container group with the [az container create][az-container-create] command, specifying the YAML file name for the `--file` parameter: @@ -359,7 +360,7 @@ Now, set `CONTAINER_GROUP_IP` to the IP you retrieved with the `az container sho az container create \ --resource-group $MY_RESOURCE_GROUP_NAME \ --name $MY_COMM_CHECKER_NAME \ - --image mcr.microsoft.com/azuredocs/aci-helloworld \ + --image mcr.microsoft.com/devcontainers/base:alpine \ --command-line "wget 10.0.0.4" \ --restart-policy never \ --vnet $MY_VNET_NAME \ From 175804f32205c1c318748bf78c6c3579e2eb49d4 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 17 Mar 2025 14:46:22 -0700 Subject: [PATCH 42/87] added support request as next step for support docs --- scenarios/metadata.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scenarios/metadata.json b/scenarios/metadata.json index e3fe2a486..457b0b9f6 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -393,6 +393,12 @@ "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/ObtainPerformanceMetricsLinuxSustem/obtain-performance-metrics-linux-system.md", "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/collect-performance-metrics-from-a-linux-system", + "nextSteps": [ + { + "title": "Create a Support Request for your VM", + "url": "https://portal.azure.com/#view/Microsoft_Azure_Support/HelpAndSupportBlade/~/overview" + } + ], "configurations": { "permissions": [], "configurableParams": [ @@ -928,6 +934,12 @@ "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/FixFstabIssuesRepairVM/fix-fstab-issues-repair-vm.md", "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/linux-virtual-machine-cannot-start-fstab-errors#use-azure-linux-auto-repair-alar", + "nextSteps": [ + { + "title": "Create a Support Request for your VM", + "url": "https://portal.azure.com/#view/Microsoft_Azure_Support/HelpAndSupportBlade/~/overview" + } + ], "configurations": { "permissions": [], "configurableParams": [ @@ -954,6 +966,12 @@ "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/KernelBootIssuesRepairVM/kernel-related-boot-issues-repairvm.md", "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/kernel-related-boot-issues#missing-initramfs-alar", + "nextSteps": [ + { + "title": "Create a Support Request for your VM", + "url": "https://portal.azure.com/#view/Microsoft_Azure_Support/HelpAndSupportBlade/~/overview" + } + ], "configurations": { "permissions": [], "configurableParams": [ @@ -980,6 +998,12 @@ "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/TroubleshootVMGrubError/troubleshoot-vm-grub-error-repairvm.md", "documentationUrl": "https://learn.microsoft.com/en-us/troubleshoot/azure/virtual-machines/linux/troubleshoot-vm-boot-error", + "nextSteps": [ + { + "title": "Create a Support Request for your VM", + "url": "https://portal.azure.com/#view/Microsoft_Azure_Support/HelpAndSupportBlade/~/overview" + } + ], "configurations": { "permissions": [], "configurableParams": [ From e4eb3b45abcbd4baabcd62f9b3dc652ad6b7f362 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Mon, 17 Mar 2025 22:42:39 -0700 Subject: [PATCH 43/87] added flatcar doc in exec docs --- scenarios/metadata.json | 12 ++ .../FlatcarOnAzure/flatcar-on-azure.md | 187 ++++++++++++++++++ tools/flatcar.md | 135 +++++++++++++ tools/flatcar_converted.md | 187 ++++++++++++++++++ 4 files changed, 521 insertions(+) create mode 100644 scenarios/upstream/FlatcarOnAzure/flatcar-on-azure.md create mode 100644 tools/flatcar.md create mode 100644 tools/flatcar_converted.md diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 457b0b9f6..fb4256e09 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1190,5 +1190,17 @@ "configurations": { "permissions": [] } + }, + { + "status": "active", + "key": "upstream/FlatcarOnAzure/flatcar-on-azure.md", + "title": "Running Flatcar Container Linux on Microsoft Azure", + "description": "Deploy Flatcar Container Linux in Microsoft Azure by creating resource groups and using official marketplace images.", + "stackDetails": [ + ], + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/upstream/FlatcarOnAzure/flatcar-on-azure.md", + "documentationUrl": "https://www.flatcar.org/docs/latest/installing/cloud/azure/", + "configurations": { + } } ] diff --git a/scenarios/upstream/FlatcarOnAzure/flatcar-on-azure.md b/scenarios/upstream/FlatcarOnAzure/flatcar-on-azure.md new file mode 100644 index 000000000..aaaf474a2 --- /dev/null +++ b/scenarios/upstream/FlatcarOnAzure/flatcar-on-azure.md @@ -0,0 +1,187 @@ +--- +title: 'Running Flatcar Container Linux on Microsoft Azure' +description: 'Deploy Flatcar Container Linux in Microsoft Azure by creating resource groups and using official marketplace images.' +ms.topic: article +ms.date: 03/17/2025 +author: naman-msft +ms.author: namanparikh +ms.custom: innovation-engine, azure, flatcar +--- + +## Creating resource group via Microsoft Azure CLI + +Follow the [installation and configuration guides][azure-cli] for the Microsoft Azure CLI to set up your local installation. + +Instances on Microsoft Azure must be created within a resource group. Create a new resource group with the following command: + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP_NAME="group-1$RANDOM_SUFFIX" +export REGION="WestUS2" +az group create --name $RESOURCE_GROUP_NAME --location $REGION +``` + +Results: + + +```json +{ + "id": "/subscriptions/xxxxx/resourceGroups/group-1xxx", + "location": "WestUS2", + "managedBy": null, + "name": "group-1xxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +Now that you have a resource group, you can choose a channel of Flatcar Container Linux you would like to install. + +## Using the official image from the Marketplace + +Official Flatcar Container Linux images for all channels are available in the Marketplace. +Flatcar is published by the `kinvolk` publisher on Marketplace. +Flatcar Container Linux is designed to be [updated automatically][update-docs] with different schedules per channel. Updating +can be [disabled][reboot-docs], although it is not recommended to do so. The [release notes][release-notes] contain +information about specific features and bug fixes. + +The following command will query for the latest image URN specifier through the Azure CLI: + +```bash +az vm image list --all -p kinvolk -f flatcar -s stable-gen2 --query '[-1]' +``` + +Results: + + + +```json +{ + "architecture": "x64", + "offer": "flatcar-container-linux-free", + "publisher": "kinvolk", + "sku": "stable-gen2", + "urn": "kinvolk:flatcar-container-linux-free:stable-gen2:3815.2.0", + "version": "3815.2.0" +} +``` + +Use the offer named `flatcar-container-linux-free`; there is also a legacy offer called `flatcar-container-linux` with the same contents. +The SKU, which is the third element of the image URN, relates to one of the release channels and also depends on whether to use Hyper-V Generation 1 or 2 VMs. +Generation 2 instance types use UEFI boot and should be preferred, the SKU matches the pattern `-gen`: `alpha-gen2`, `beta-gen2` or `stable-gen2`. +For Generation 1 instance types drop the `-gen2` from the SKU: `alpha`, `beta` or `stable`. +Note: _`az vm image list -s` flag matches parts of the SKU, which means that `-s stable` will return both the `stable` and `stable-gen2` SKUs._ + +Before being able to use the offers, you may need to accept the legal terms once, which is demonstrated for `flatcar-container-linux-free` and `stable-gen2`: + +```bash +az vm image terms show --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 +az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 +``` + +For quick tests the official Azure CLI also supports an alias for the latest Flatcar stable image: + +```bash +az vm create --name node-1 --resource-group $RESOURCE_GROUP_NAME --admin-username core --image FlatcarLinuxFreeGen2 --generate-ssh-keys +``` + +Results: + + + +```json +{ + "fqdns": null, + "id": "/subscriptions/xxxxx/resourceGroups/group-1xxx/providers/Microsoft.Compute/virtualMachines/node-1", + "location": "WestUS2", + "name": "node-1", + "powerState": "VM running", + "provisioningState": "Succeeded", + "resourceGroup": "group-1xxx", + "zones": null +} +``` + +### CoreVM + +Flatcar images are also published under an offer called `flatcar-container-linux-corevm-amd64`. This offer does not require accepting image terms and does not require specifying plan information when creating instances or building derived images. The content of the images matches the other offers. + +```bash +az vm image list --all -p kinvolk -f flatcar-container-linux-corevm-amd64 -s stable-gen2 --query '[-1]' +``` + +Results: + + + +```json +{ + "architecture": "x64", + "offer": "flatcar-container-linux-corevm-amd64", + "publisher": "kinvolk", + "sku": "stable-gen2", + "urn": "kinvolk:flatcar-container-linux-corevm-amd64:stable-gen2:3815.2.0", + "version": "3815.2.0" +} +``` + +### ARM64 + +Arm64 images are published under the offer called `flatcar-container-linux-corevm`. These are Generation 2 images—the only supported option on Azure for Arm64 instances—so the SKU contains only the release channel name without the `-gen2` suffix: `alpha`, `beta`, or `stable`. This offer has the same properties as the `CoreVM` offer described above. + +```bash +az vm image list --all --architecture arm64 -p kinvolk -f flatcar -s stable --query '[-1]' +``` + +Results: + + + +```json +{ + "architecture": "Arm64", + "offer": "flatcar-container-linux-corevm", + "publisher": "kinvolk", + "sku": "stable", + "urn": "kinvolk:flatcar-container-linux-corevm:stable:3815.2.0", + "version": "3815.2.0" +} +``` + +### Flatcar Pro Images + +Flatcar Pro images were paid marketplace images that came with commercial support and extra features. All the previous features of Flatcar Pro images, such as support for NVIDIA GPUs, are now available to all users in standard Flatcar marketplace images. + +### Plan information for building your image from the Marketplace Image + +When building an image based on the Marketplace image you sometimes need to specify the original plan. The plan name is the image SKU (for example, `stable`), the plan product is the image offer (for example, `flatcar-container-linux-free`), and the plan publisher is the same (`kinvolk`). + +## Community Shared Image Gallery + +While the Marketplace images are recommended, it sometimes might be easier or required to use Shared Image Galleries—for example, when using Packer for Kubernetes CAPI images. + +A public Shared Image Gallery hosts recent Flatcar Stable images for amd64. Here is how to list the image definitions (for now you will only find `flatcar-stable-amd64`) and the image versions they provide: + +```bash +az sig image-definition list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --location westeurope +az sig image-version list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --gallery-image-definition flatcar-stable-amd64 --location westeurope +``` + +A second gallery, `flatcar4capi-742ef0cb-dcaa-4ecb-9cb0-bfd2e43dccc0`, exists for prebuilt Kubernetes CAPI images. It has image definitions for each CAPI version—for example, `flatcar-stable-amd64-capi-v1.26.3` provides recent Flatcar Stable versions. + +[flatcar-user]: https://groups.google.com/forum/#!forum/flatcar-linux-user +[etcd-docs]: https://etcd.io/docs +[quickstart]: ../ +[reboot-docs]: ../../setup/releases/update-strategies +[azure-cli]: https://docs.microsoft.com/en-us/cli/azure/overview +[butane-configs]: ../../provisioning/config-transpiler +[irc]: irc://irc.freenode.org:6667/#flatcar +[docs]: ../../ +[resource-group]: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#naming-rules-and-restrictions +[storage-account]: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#naming-storage-accounts +[azure-flatcar-image-upload]: https://github.com/flatcar/flatcar-cloud-image-uploader +[release-notes]: https://flatcar.org/releases +[update-docs]: ../../setup/releases/update-strategies \ No newline at end of file diff --git a/tools/flatcar.md b/tools/flatcar.md new file mode 100644 index 000000000..dbe355224 --- /dev/null +++ b/tools/flatcar.md @@ -0,0 +1,135 @@ +--- +title: Running Flatcar Container Linux on Microsoft Azure +linktitle: Running on Microsoft Azure +weight: 10 +aliases: + - ../../os/booting-on-azure + - ../../cloud-providers/booting-on-azure +--- + +## Creating resource group via Microsoft Azure CLI + +Follow the [installation and configuration guides][azure-cli] for the Microsoft Azure CLI to set up your local installation. + +Instances on Microsoft Azure must be created within a resource group. Create a new resource group with the following command: + +```bash +az group create --name group-1 --location +``` + +Now that you have a resource group, you can choose a channel of Flatcar Container Linux you would like to install. + +## Using the official image from the Marketplace + +Official Flatcar Container Linux images for all channels are available in the Marketplace. +Flatcar is published by the `kinvolk` publisher on Marketplace. +Flatcar Container Linux is designed to be [updated automatically][update-docs] with different schedules per channel. Updating +can be [disabled][reboot-docs], although it is not recommended to do so. The [release notes][release-notes] contain +information about specific features and bug fixes. + +The following command will create a single instance through the Azure CLI. + +```bash +az vm image list --all -p kinvolk -f flatcar -s stable-gen2 --query '[-1]' # Query the image name urn specifier +``` + +```json +{ + "architecture": "x64", + "offer": "flatcar-container-linux-free", + "publisher": "kinvolk", + "sku": "stable-gen2", + "urn": "kinvolk:flatcar-container-linux-free:stable-gen2:3815.2.0", + "version": "3815.2.0" +} + +Use the offer named `flatcar-container-linux-free`, there is also a legacy offer called `flatcar-container-linux` with the same contents. +The SKU, which is the third element of the image URN, relates to one of the release channels and also depends on whether to use Hyper-V Generation 1 or 2 VM. +Generation 2 instance types use UEFI boot and should be preferred, the SKU matches the pattern `-gen`: `alpha-gen2`, `beta-gen2` or `stable-gen2`. +For Generation 1 instance types drop the `-gen2` from the SKU: `alpha`, `beta` or `stable`. +Note: _`az vm image list -s` flag matches parts of the SKU, which means that `-s stable` will return both the `stable` and `stable-gen2` SKUs._ + +Before being able to use the offers, you may need to accept the legal terms once, here done for `flatcar-container-linux-free` and `stable-gen2`: + +```bash +az vm image terms show --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 +az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 +``` + +For quick tests the official Azure CLI also supports an alias for the latest Flatcar stable image: +```bash +az vm create --name node-1 --resource-group group-1 --admin-username core --user-data config.ign --image FlatcarLinuxFreeGen2 +``` + +### CoreVM + +Flatcar images are also published under an offer called `flatcar-container-linux-corevm-amd64`. This offer does not require accepting image terms and does not require specifying plan information when creating instances or building derived images. The content of the images matches the other offers. +```bash +az vm image list --all -p kinvolk -f flatcar-container-linux-corevm-amd64 -s stable-gen2 --query '[-1]' +``` + +```json +{ + "architecture": "x64", + "offer": "flatcar-container-linux-corevm-amd64", + "publisher": "kinvolk", + "sku": "stable-gen2", + "urn": "kinvolk:flatcar-container-linux-corevm-amd64:stable-gen2:3815.2.0", + "version": "3815.2.0" +} +``` + +### ARM64 +Arm64 images are published under the offer called `flatcar-container-linux-corevm`. These are Generation 2 images, the only supported option on Azure for Arm64 instances, so the SKU contains only the release channel name without the `-gen2` suffix: `alpha`, `beta`, `stable`. This offer has the same properties as the `CoreVM` offer described above. + +```bash +az vm image list --all --architecture arm64 -p kinvolk -f flatcar -s stable --query '[-1]' +``` + +```json +{ + "architecture": "Arm64", + "offer": "flatcar-container-linux-corevm", + "publisher": "kinvolk", + "sku": "stable", + "urn": "kinvolk:flatcar-container-linux-corevm:stable:3815.2.0", + "version": "3815.2.0" +} +``` + + + +### Flatcar Pro Images + +Flatcar Pro images were paid marketplace images that came with commercial support and extra features. All the previous features of Flatcar Pro images, such as support for NVIDIA GPUs, are now available to all users in standard Flatcar marketplace images. + +### Plan information for building your image from the Marketplace Image + +When building an image based on the Marketplace image you sometimes need to specify the original plan. The plan name is the image SKU, e.g., `stable`, the plan product is the image offer, e.g., `flatcar-container-linux-free`, and the plan publisher is the same (`kinvolk`). + +## Community Shared Image Gallery + +While the Marketplace images are recommended, it sometimes might be easier or required to use Shared Image Galleries, e.g., when using Packer for Kubernetes CAPI images. + +A public Shared Image Gallery hosts recent Flatcar Stable images for amd64. Here is how to show the image definitions (for now you will only find `flatcar-stable-amd64`) and the image versions they provide: + +```bash +az sig image-definition list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --location westeurope +az sig image-version list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --gallery-image-definition flatcar-stable-amd64 --location westeurope +``` + +A second gallery `flatcar4capi-742ef0cb-dcaa-4ecb-9cb0-bfd2e43dccc0` exists for prebuilt Kubernetes CAPI images. It has image definitions for each CAPI version, e.g., `flatcar-stable-amd64-capi-v1.26.3` which provides recent Flatcar Stable versions. + +[flatcar-user]: https://groups.google.com/forum/#!forum/flatcar-linux-user +[etcd-docs]: https://etcd.io/docs +[quickstart]: ../ +[reboot-docs]: ../../setup/releases/update-strategies +[azure-cli]: https://docs.microsoft.com/en-us/cli/azure/overview +[butane-configs]: ../../provisioning/config-transpiler +[irc]: irc://irc.freenode.org:6667/#flatcar +[docs]: ../../ +[resource-group]: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#naming-rules-and-restrictions +[storage-account]: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#naming-storage-accounts +[azure-flatcar-image-upload]: https://github.com/flatcar/flatcar-cloud-image-uploader +[release-notes]: https://flatcar.org/releases +[update-docs]: ../../setup/releases/update-strategies \ No newline at end of file diff --git a/tools/flatcar_converted.md b/tools/flatcar_converted.md new file mode 100644 index 000000000..a5797b922 --- /dev/null +++ b/tools/flatcar_converted.md @@ -0,0 +1,187 @@ +--- +title: 'Running Flatcar Container Linux on Microsoft Azure' +description: 'Deploy Flatcar Container Linux in Microsoft Azure by creating resource groups and using official marketplace images.' +ms.topic: article +ms.date: 10/10/2023 +author: naman-msft +ms.author: namanparikh +ms.custom: innovation-engine, azure, flatcar +--- + +## Creating resource group via Microsoft Azure CLI + +Follow the [installation and configuration guides][azure-cli] for the Microsoft Azure CLI to set up your local installation. + +Instances on Microsoft Azure must be created within a resource group. Create a new resource group with the following command: + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP_NAME="group-1$RANDOM_SUFFIX" +export REGION="WestUS2" +az group create --name $RESOURCE_GROUP_NAME --location $REGION +``` + +Results: + + +```json +{ + "id": "/subscriptions/xxxxx/resourceGroups/group-1xxx", + "location": "WestUS2", + "managedBy": null, + "name": "group-1xxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +Now that you have a resource group, you can choose a channel of Flatcar Container Linux you would like to install. + +## Using the official image from the Marketplace + +Official Flatcar Container Linux images for all channels are available in the Marketplace. +Flatcar is published by the `kinvolk` publisher on Marketplace. +Flatcar Container Linux is designed to be [updated automatically][update-docs] with different schedules per channel. Updating +can be [disabled][reboot-docs], although it is not recommended to do so. The [release notes][release-notes] contain +information about specific features and bug fixes. + +The following command will query for the latest image URN specifier through the Azure CLI: + +```bash +az vm image list --all -p kinvolk -f flatcar -s stable-gen2 --query '[-1]' +``` + +Results: + + + +```json +{ + "architecture": "x64", + "offer": "flatcar-container-linux-free", + "publisher": "kinvolk", + "sku": "stable-gen2", + "urn": "kinvolk:flatcar-container-linux-free:stable-gen2:3815.2.0", + "version": "3815.2.0" +} +``` + +Use the offer named `flatcar-container-linux-free`; there is also a legacy offer called `flatcar-container-linux` with the same contents. +The SKU, which is the third element of the image URN, relates to one of the release channels and also depends on whether to use Hyper-V Generation 1 or 2 VMs. +Generation 2 instance types use UEFI boot and should be preferred, the SKU matches the pattern `-gen`: `alpha-gen2`, `beta-gen2` or `stable-gen2`. +For Generation 1 instance types drop the `-gen2` from the SKU: `alpha`, `beta` or `stable`. +Note: _`az vm image list -s` flag matches parts of the SKU, which means that `-s stable` will return both the `stable` and `stable-gen2` SKUs._ + +Before being able to use the offers, you may need to accept the legal terms once, which is demonstrated for `flatcar-container-linux-free` and `stable-gen2`: + +```bash +az vm image terms show --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 +az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 +``` + +For quick tests the official Azure CLI also supports an alias for the latest Flatcar stable image: + +```bash +az vm create --name node-1 --resource-group $RESOURCE_GROUP_NAME --admin-username core --image FlatcarLinuxFreeGen2 --generate-ssh-keys +``` + +Results: + + + +```json +{ + "fqdns": null, + "id": "/subscriptions/xxxxx/resourceGroups/group-1xxx/providers/Microsoft.Compute/virtualMachines/node-1", + "location": "WestUS2", + "name": "node-1", + "powerState": "VM running", + "provisioningState": "Succeeded", + "resourceGroup": "group-1xxx", + "zones": null +} +``` + +### CoreVM + +Flatcar images are also published under an offer called `flatcar-container-linux-corevm-amd64`. This offer does not require accepting image terms and does not require specifying plan information when creating instances or building derived images. The content of the images matches the other offers. + +```bash +az vm image list --all -p kinvolk -f flatcar-container-linux-corevm-amd64 -s stable-gen2 --query '[-1]' +``` + +Results: + + + +```json +{ + "architecture": "x64", + "offer": "flatcar-container-linux-corevm-amd64", + "publisher": "kinvolk", + "sku": "stable-gen2", + "urn": "kinvolk:flatcar-container-linux-corevm-amd64:stable-gen2:3815.2.0", + "version": "3815.2.0" +} +``` + +### ARM64 + +Arm64 images are published under the offer called `flatcar-container-linux-corevm`. These are Generation 2 images—the only supported option on Azure for Arm64 instances—so the SKU contains only the release channel name without the `-gen2` suffix: `alpha`, `beta`, or `stable`. This offer has the same properties as the `CoreVM` offer described above. + +```bash +az vm image list --all --architecture arm64 -p kinvolk -f flatcar -s stable --query '[-1]' +``` + +Results: + + + +```json +{ + "architecture": "Arm64", + "offer": "flatcar-container-linux-corevm", + "publisher": "kinvolk", + "sku": "stable", + "urn": "kinvolk:flatcar-container-linux-corevm:stable:3815.2.0", + "version": "3815.2.0" +} +``` + +### Flatcar Pro Images + +Flatcar Pro images were paid marketplace images that came with commercial support and extra features. All the previous features of Flatcar Pro images, such as support for NVIDIA GPUs, are now available to all users in standard Flatcar marketplace images. + +### Plan information for building your image from the Marketplace Image + +When building an image based on the Marketplace image you sometimes need to specify the original plan. The plan name is the image SKU (for example, `stable`), the plan product is the image offer (for example, `flatcar-container-linux-free`), and the plan publisher is the same (`kinvolk`). + +## Community Shared Image Gallery + +While the Marketplace images are recommended, it sometimes might be easier or required to use Shared Image Galleries—for example, when using Packer for Kubernetes CAPI images. + +A public Shared Image Gallery hosts recent Flatcar Stable images for amd64. Here is how to list the image definitions (for now you will only find `flatcar-stable-amd64`) and the image versions they provide: + +```bash +az sig image-definition list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --location westeurope +az sig image-version list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --gallery-image-definition flatcar-stable-amd64 --location westeurope +``` + +A second gallery, `flatcar4capi-742ef0cb-dcaa-4ecb-9cb0-bfd2e43dccc0`, exists for prebuilt Kubernetes CAPI images. It has image definitions for each CAPI version—for example, `flatcar-stable-amd64-capi-v1.26.3` provides recent Flatcar Stable versions. + +[flatcar-user]: https://groups.google.com/forum/#!forum/flatcar-linux-user +[etcd-docs]: https://etcd.io/docs +[quickstart]: ../ +[reboot-docs]: ../../setup/releases/update-strategies +[azure-cli]: https://docs.microsoft.com/en-us/cli/azure/overview +[butane-configs]: ../../provisioning/config-transpiler +[irc]: irc://irc.freenode.org:6667/#flatcar +[docs]: ../../ +[resource-group]: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#naming-rules-and-restrictions +[storage-account]: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#naming-storage-accounts +[azure-flatcar-image-upload]: https://github.com/flatcar/flatcar-cloud-image-uploader +[release-notes]: https://flatcar.org/releases +[update-docs]: ../../setup/releases/update-strategies \ No newline at end of file From 78d5d24eed7458e36516f9e4d139f07f7cdd8cdc Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 19 Mar 2025 10:30:49 -0700 Subject: [PATCH 44/87] updated doc --- .../articles/batch/quick-create-cli.md | 6 +++--- scenarios/metadata.json | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/scenarios/azure-docs/articles/batch/quick-create-cli.md b/scenarios/azure-docs/articles/batch/quick-create-cli.md index ea6b37499..b0b86f1f4 100644 --- a/scenarios/azure-docs/articles/batch/quick-create-cli.md +++ b/scenarios/azure-docs/articles/batch/quick-create-cli.md @@ -2,10 +2,10 @@ title: 'Quickstart: Use the Azure CLI to create a Batch account and run a job' description: Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool. ms.topic: quickstart -ms.date: 04/12/2023 +ms.date: 03/19/2025 ms.custom: mvc, devx-track-azurecli, mode-api, linux-related-content, innovation-engine -author: (preserved) -ms.author: (preserved) +author: padmalathas +ms.author: padmalathas --- # Quickstart: Use the Azure CLI to create a Batch account and run a job diff --git a/scenarios/metadata.json b/scenarios/metadata.json index fb4256e09..3c75d45ed 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -443,6 +443,13 @@ "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/deploy-postgresql-ha.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/deploy-postgresql-ha", + "nextSteps": [ + { + "title": "Deploy a highly available PostgreSQL database on AKS with Azure CLI", + "url": "https://learn.microsoft.com/en-us/azure/aks/deploy-postgresql-ha?tabs=helm" + } + + ], "configurations": { } }, @@ -487,6 +494,13 @@ "stackDetails": "", "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/DeployHAPGonARO/deploy-ha-pg-aro.md", "documentationUrl": "", + "nextSteps": [ + { + "title": "Deploy a highly available PostgreSQL database on AKS with Azure CLI", + "url": "https://learn.microsoft.com/en-us/azure/aks/deploy-postgresql-ha?tabs=helm" + } + + ], "configurations": { } }, @@ -1203,4 +1217,4 @@ "configurations": { } } -] +] \ No newline at end of file From 5f8c6f30b6a894132a5bfe71019f1153d59312d3 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 20 Mar 2025 16:50:09 -0700 Subject: [PATCH 45/87] updated files and added new docs --- .../tutorial-autoscale-cli.md | 6 +- .../tutorial-modify-scale-sets-cli.md | 22 +- .../disks-enable-performance.md | 160 ++++---------- .../linux/tutorial-manage-vm.md | 8 +- tools/alert-bad-process.yaml | 13 ++ tools/ama-metrics-settings-configmap.yaml | 84 ++++++++ tools/demo.md | 197 ++++++++++++++++++ tools/demo_notes.txt | 71 +++++++ tools/flatcar.md | 135 ------------ tools/flatcar_converted.md | 187 ----------------- 10 files changed, 418 insertions(+), 465 deletions(-) create mode 100644 tools/alert-bad-process.yaml create mode 100644 tools/ama-metrics-settings-configmap.yaml create mode 100644 tools/demo.md create mode 100644 tools/demo_notes.txt delete mode 100644 tools/flatcar.md delete mode 100644 tools/flatcar_converted.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md index 65e4bdc85..f3cc966c0 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md @@ -37,7 +37,7 @@ export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION ``` -Now create a Virtual Machine Scale Set with [az vmss create](/cli/azure/vmss). The following example creates a scale set with an instance count of 2, generates SSH keys if they don't exist, and uses a valid image "Ubuntu2204". +Now create a Virtual Machine Scale Set with [az vmss create](/cli/azure/vmss). The following example creates a scale set with an instance count of 2, generates SSH keys if they don't exist, and uses a valid image *Ubuntu2204*. ```azurecli-interactive export MY_SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" @@ -92,7 +92,7 @@ az monitor autoscale rule create \ ``` ## Simulate CPU load on scale set -To test the autoscale rules, you need to simulate sustained CPU load on the VM instances in the scale set. In this minimalist approach, we avoid installing additional packages by using the built-in "yes" command to generate CPU load. The following command starts 3 background processes that continuously output data to /dev/null for 60 seconds and then terminates them. +To test the autoscale rules, you need to simulate sustained CPU load on the VM instances in the scale set. In this minimalist approach, we avoid installing additional packages by using the built-in `yes` command to generate CPU load. The following command starts 3 background processes that continuously output data to `/dev/null` for 60 seconds and then terminates them. ```bash for i in {1..3}; do @@ -105,7 +105,7 @@ pkill yes This command simulates CPU load without introducing package installation errors. ## Monitor the active autoscale rules -To monitor the number of VM instances in your scale set, use the watch command. It may take up to 5 minutes for the autoscale rules to begin the scale-out process in response to the CPU load. However, once it happens, you can exit watch with Ctrl-c. +To monitor the number of VM instances in your scale set, use the `watch` command. It may take up to 5 minutes for the autoscale rules to begin the scale-out process in response to the CPU load. However, once it happens, you can exit watch with *CTRL + C* keys. By then, the scale set will automatically increase the number of VM instances to meet the demand. The following command shows the list of VM instances in the scale set: diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md index ed996f0ef..6f0bd2656 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md @@ -13,10 +13,10 @@ ms.custom: mimckitt, devx-track-azurecli, linux-related-content, innovation-engi # Tutorial: Modify a Virtual Machine Scale Set using Azure CLI Throughout the lifecycle of your applications, you may need to modify or update your Virtual Machine Scale Set. These updates may include how to update the configuration of the scale set, or change the application configuration. This article describes how to modify an existing scale set using the Azure CLI. -Below, we declare environment variables that will be used throughout this document. A random suffix is appended to resource names that need to be unique for each deployment. The REGION is set to WestUS2. +Below, we declare environment variables that will be used throughout this document. A random suffix is appended to resource names that need to be unique for each deployment. The `REGION` is set to *WestUS2*. ```bash -export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RANDOM_SUFFIX=adcc95 export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" export SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" export NEW_INSTANCE_NAME="myNewInstance$RANDOM_SUFFIX" @@ -46,7 +46,7 @@ az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION ``` ## Create the Virtual Machine Scale Set -To ensure that subsequent update and query commands have a valid resource to work on, create a Virtual Machine Scale Set. In this step, we deploy a basic scale set using a valid image ("Ubuntu2204") and set the instance count to 5 so that instance-specific updates can target an existing instance ID. +To ensure that subsequent update and query commands have a valid resource to work on, create a Virtual Machine Scale Set. In this step, we deploy a basic scale set using a valid image (*Ubuntu2204*) and set the instance count to 5 so that instance-specific updates can target an existing instance ID. ```azurecli-interactive az vmss create \ @@ -164,7 +164,7 @@ The exact presentation of the output depends on the options you provide to the c } ``` -You can use [az vmss update](/cli/azure/vmss#az-vmss-update) to update various properties of your scale set. For example, updating your license type or a VM's instance protection policy. Note that the allowed license type value is "RHEL_BYOS" rather than "Windows_Server." +You can use [az vmss update](/cli/azure/vmss#az-vmss-update) to update various properties of your scale set. For example, updating your license type or a VM's instance protection policy. Note that the allowed license type value is *RHEL_BYOS* rather than *Windows_Server*. ```azurecli-interactive az vmss update --name $SCALE_SET_NAME --resource-group $MY_RESOURCE_GROUP_NAME --license-type RHEL_BYOS @@ -185,7 +185,7 @@ az vmss update \ --protect-from-scale-in ``` -Additionally, if you previously deployed the scale set with the az vmss create command, you can run the az vmss create command again to update the scale set. Make sure that all properties in the az vmss create command are the same as before, except for the properties that you wish to modify. For example, below we're increasing the instance count to five. +Additionally, if you previously deployed the scale set with the `az vmss create` command, you can run the `az vmss create` command again to update the scale set. Make sure that all properties in the `az vmss create` command are the same as before, except for the properties that you wish to modify. For example, below we're increasing the instance count to five. > [!IMPORTANT] >Starting November 2023, VM scale sets created using PowerShell and Azure CLI will default to Flexible Orchestration Mode if no orchestration mode is specified. For more information about this change and what actions you should take, go to [Breaking Change for VMSS PowerShell/CLI Customers - Microsoft Community Hub](https://techcommunity.microsoft.com/t5/azure-compute-blog/breaking-change-for-vmss-powershell-cli-customers/ba-p/3818295) @@ -199,8 +199,7 @@ az vmss create \ --admin-username azureuser \ --generate-ssh-keys \ --instance-count 5 \ - --os-disk-size-gb 64 \ - --admin-username azureuser + --os-disk-size-gb 64 ``` ## Updating individual VM instances in a scale set @@ -288,8 +287,7 @@ These properties describe the configuration of a VM instance within a scale set, You can perform updates to individual VM instances in a scale set just like you would a standalone VM. For example, attaching a new data disk to instance 1: ```azurecli-interactive -export DISK_NAME="disk_name$RANDOM_SUFFIX" -az vm disk attach --resource-group $MY_RESOURCE_GROUP_NAME --vm-name $INSTANCE_NAME --name $DISK_NAME --new +az vm disk attach --resource-group $MY_RESOURCE_GROUP_NAME --vm-name $INSTANCE_NAME --name disk_name1 --new ``` Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the VM instance has the new disk attached. @@ -305,11 +303,11 @@ Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the V "diskSizeGb": 1023, "lun": 0, "managedDisk": { - "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/disks/disk_namexxxx", + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/disks/disk_name1", "resourceGroup": "myResourceGroupxxx", "storageAccountType": "Premium_LRS" }, - "name": "disk_namexxxx", + "name": "disk_name1", "toBeDetached": false } ] @@ -321,7 +319,7 @@ Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the V There are times where you might want to add a new VM to your scale set but want different configuration options than those listed in the scale set model. VMs can be added to a scale set during creation by using the [az vm create](/cli/azure/vmss#az-vmss-create) command and specifying the scale set name you want the instance added to. ```azurecli-interactive -az vm create --name $NEW_INSTANCE_NAME --resource-group $MY_RESOURCE_GROUP_NAME --vmss $SCALE_SET_NAME --image RHELRaw8LVMGen2 --admin-username azureuser +az vm create --name $NEW_INSTANCE_NAME --resource-group $MY_RESOURCE_GROUP_NAME --vmss $SCALE_SET_NAME --image RHELRaw8LVMGen2 ``` ```output diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md b/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md index 12c533f61..cb1c2373e 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md +++ b/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md @@ -6,7 +6,7 @@ ms.service: azure-disk-storage ms.topic: how-to ms.date: 12/09/2024 ms.author: rogarana -ms.custom: devx-track-azurepowershell, innovation-engine +ms.custom: devx-track-azurepowershell --- # Preview - Increase IOPS and throughput limits for Azure Premium SSDs and Standard SSD/HDDs @@ -29,7 +29,7 @@ Either use the Azure Cloud Shell to run your commands or install a version of th ## Enable performance plus -You need to create a new disk to use performance plus. The following scripts show how to create a disk with performance plus enabled and, if desired, attach it to a VM. The commands have been organized into self-contained steps for reliability. +You need to create a new disk to use performance plus. The following script creates a disk that has performance plus enabled and attach it to a VM: # [Azure CLI](#tab/azure-cli) @@ -190,142 +190,52 @@ Results: # [Azure PowerShell](#tab/azure-powershell) -### Create a resource group - -This step creates a resource group with a unique name. +You need to create a new disk to use performance plus. The following script creates a disk that has performance plus enabled and attach it to a VM: ```azurepowershell -$RANDOM_SUFFIX = (New-Guid).Guid.Substring(0,6) -$myRG = "PerfPlusRG$RANDOM_SUFFIX" -$region = "WestUS2" -New-AzResourceGroup -Name $myRG -Location $region -``` - -Results: - - -```JSON -{ - "ResourceGroupName": "PerfPlusRGxxx", - "Location": "WestUS2", - "ProvisioningState": "Succeeded" -} -``` - -### Create a new disk with performance plus enabled +$myRG=yourResourceGroupName +$myDisk=yourDiskName +$myVM=yourVMName +$region=desiredRegion +# Valid values are Premium_LRS, Premium_ZRS, StandardSSD_LRS, StandardSSD_ZRS, or Standard_LRS +$sku=desiredSKU +#Size must be 513 or larger +$size=513 +$lun=desiredLun + +Set-AzContext -SubscriptionName -This step creates a new disk with performance plus enabled using a valid SKU value. - -```azurepowershell -$myDisk = "PerfPlusDisk$RANDOM_SUFFIX" -$sku = "Premium_LRS" -$size = 513 $diskConfig = New-AzDiskConfig -Location $region -CreateOption Empty -DiskSizeGB $size -SkuName $sku -PerformancePlus $true -$dataDisk = New-AzDisk -ResourceGroupName $myRG -DiskName $myDisk -Disk $diskConfig -``` -Results: - - -```JSON -{ - "ResourceGroup": "PerfPlusRGxxx", - "Name": "PerfPlusDiskxxx", - "Location": "WestUS2", - "Sku": "Premium_LRS", - "DiskSizeGB": 513, - "PerformancePlus": true, - "ProvisioningState": "Succeeded" -} -``` - -### Attempt to attach the disk to a VM - -This optional step checks whether the specified VM exists before attempting the disk attachment. - -```azurepowershell -$myVM = "NonExistentVM" -if (Get-AzVM -ResourceGroupName $myRG -Name $myVM -ErrorAction SilentlyContinue) { - Add-AzVMDataDisk -VMName $myVM -ResourceGroupName $myRG -DiskName $myDisk -Lun 0 -CreateOption Empty -ManagedDiskId $dataDisk.Id -} else { - Write-Output "VM $myVM not found. Skipping disk attachment." -} -``` - -Results: +$dataDisk = New-AzDisk -ResourceGroupName $myRG -DiskName $myDisk -Disk $diskConfig - -```text -VM NonExistentVM not found. Skipping disk attachment. +Add-AzVMDataDisk -VMName $myVM -ResourceGroupName $myRG -DiskName $myDisk -Lun $lun -CreateOption Empty -ManagedDiskId $dataDisk.Id ``` -### Create a new disk from an existing disk or snapshot with performance plus enabled - -This series of steps creates a separate resource group and then creates a new disk from an existing disk or snapshot. Replace the $sourceURI with a valid source blob URI that belongs to the same region (WestUS2) as the disk. - -#### Create a resource group for migration +To migrate data from an existing disk or snapshot to a new disk with performance plus enabled, use the following script: ```azurepowershell -$RANDOM_SUFFIX = (New-Guid).Guid.Substring(0,6) -$myMigrRG = "PerfPlusMigrRG$RANDOM_SUFFIX" -$region = "WestUS2" -New-AzResourceGroup -Name $myMigrRG -Location $region -``` - -Results: +$myDisk=yourDiskOrSnapshotName +$myVM=yourVMName +$region=desiredRegion +# Valid values are Premium_LRS, Premium_ZRS, StandardSSD_LRS, StandardSSD_ZRS, or Standard_LRS +$sku=desiredSKU +#Size must be 513 or larger +$size=513 +$sourceURI=diskOrSnapshotURI +$lun=desiredLun + +Set-AzContext -SubscriptionName <> - -```JSON -{ - "ResourceGroupName": "PerfPlusMigrRGxxx", - "Location": "WestUS2", - "ProvisioningState": "Succeeded" -} -``` - -#### Create the disk from an existing snapshot or disk - -```azurepowershell -$myDisk = "PerfPlusMigrDisk$RANDOM_SUFFIX" -$sku = "Premium_LRS" -$size = 513 -$sourceURI = "https://examplestorageaccount.blob.core.windows.net/snapshots/sample-westus2.vhd" # Replace with a valid source blob URI in WestUS2 $diskConfig = New-AzDiskConfig -Location $region -CreateOption Copy -DiskSizeGB $size -SkuName $sku -PerformancePlus $true -SourceResourceID $sourceURI -$dataDisk = New-AzDisk -ResourceGroupName $myMigrRG -DiskName $myDisk -Disk $diskConfig -``` - -Results: - - -```JSON -{ - "ResourceGroup": "PerfPlusMigrRGxxx", - "Name": "PerfPlusMigrDiskxxx", - "Location": "WestUS2", - "Sku": "Premium_LRS", - "DiskSizeGB": 513, - "PerformancePlus": true, - "SourceResourceID": "https://examplestorageaccount.blob.core.windows.net/snapshots/sample-westus2.vhd", - "ProvisioningState": "Succeeded" -} -``` -#### Attempt to attach the migrated disk to a VM - -This optional step verifies the existence of the specified VM before attempting disk attachment. - -```azurepowershell -$myVM = "NonExistentVM" -if (Get-AzVM -ResourceGroupName $myMigrRG -Name $myVM -ErrorAction SilentlyContinue) { - Add-AzVMDataDisk -VMName $myVM -ResourceGroupName $myMigrRG -DiskName $myDisk -Lun 0 -CreateOption Empty -ManagedDiskId $dataDisk.Id -} else { - Write-Output "VM $myVM not found. Skipping disk attachment." -} +$dataDisk = New-AzDisk -ResourceGroupName $myRG -DiskName $myDisk -Disk $diskconfig +Add-AzVMDataDisk -VMName $myVM -ResourceGroupName $myRG -DiskName $myDisk -Lun $lun -CreateOption Empty -ManagedDiskId $dataDisk.Id ``` +--- -Results: +## Next steps - -```text -VM NonExistentVM not found. Skipping disk attachment. -``` \ No newline at end of file +- [Create an incremental snapshot for managed disks](disks-incremental-snapshots.md) +- [Expand virtual hard disks on a Linux VM](linux/expand-disks.md) +- [How to expand virtual hard disks attached to a Windows virtual machine](windows/expand-os-disk.md) \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md index b0830b8af..08dd74bc9 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md @@ -90,11 +90,13 @@ It may take a few minutes to create the VM. Once the VM has been created, the Az ## Connect to VM -The original tutorial includes commands to connect to the VM via SSH. For non-interactive automated execution, the SSH command is not executed. Instead, use the provided public IP address output from VM creation to manually connect if needed. +You can now connect to the VM with SSH in the Azure Cloud Shell or from your local computer. Replace the example IP address with the `publicIpAddress` noted in the previous step. -To connect to the VM, first retrieve the public IP address using the Azure CLI. Execute the following command to store the IP address in a variable: ```export IP_ADDRESS=$(az vm show --show-details --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --query publicIps --output tsv)``` +To connect to the VM, first retrieve the public IP address using the Azure CLI. Execute the following command to store the IP address in a variable: +```export IP_ADDRESS=$(az vm show --show-details --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_VM_NAME --query publicIps --output tsv)``` -Once you have the IP address, use SSH to connect to the VM. The following command connects to the VM using the `azureuser` account and the retrieved IP address: ```ssh -o StrictHostKeyChecking=no azureuser@$IP_ADDRESS``` +Once you have the IP address, use SSH to connect to the VM. The following command connects to the VM using the `azureuser` account and the retrieved IP address: +```ssh -o StrictHostKeyChecking=no azureuser@$IP_ADDRESS``` ## Understand VM images diff --git a/tools/alert-bad-process.yaml b/tools/alert-bad-process.yaml new file mode 100644 index 000000000..c7f812a2f --- /dev/null +++ b/tools/alert-bad-process.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 +kind: instance-spec +image: trace_exec:v0.38.0 +name: alert-bad-process +paramValues: + # monitor all namespaces + operator.KubeManager.all-namespaces: true + # monitor shell executions (only bash on this example) + operator.filter.filter: proc.comm==bash + # name of the metric to export + operator.otel-metrics.otel-metrics-name: 'exec:shell_executions' + # annotate gadget to enable metrics collection + operator.oci.annotate: exec:metrics.collect=true,exec:metrics.implicit-counter.name=shell_executions,exec.k8s.namespace:metrics.type=key,exec.k8s.podname:metrics.type=key,exec.k8s.containername:metrics.type=key diff --git a/tools/ama-metrics-settings-configmap.yaml b/tools/ama-metrics-settings-configmap.yaml new file mode 100644 index 000000000..73481cb01 --- /dev/null +++ b/tools/ama-metrics-settings-configmap.yaml @@ -0,0 +1,84 @@ +kind: ConfigMap +apiVersion: v1 +data: + schema-version: + #string.used by agent to parse config. supported versions are {v1}. Configs with other schema versions will be rejected by the agent. + v1 + config-version: + #string.used by customer to keep track of this config file's version in their source control/repository (max allowed 10 chars, other chars will be truncated) + ver1 + prometheus-collector-settings: |- + cluster_alias = "" + default-scrape-settings-enabled: |- + kubelet = true + coredns = false + cadvisor = true + kubeproxy = false + apiserver = false + kubestate = true + nodeexporter = true + windowsexporter = false + windowskubeproxy = false + kappiebasic = true + networkobservabilityRetina = true + networkobservabilityHubble = true + networkobservabilityCilium = true + prometheuscollectorhealth = false + controlplane-apiserver = true + controlplane-cluster-autoscaler = false + controlplane-kube-scheduler = false + controlplane-kube-controller-manager = false + controlplane-etcd = true + acstor-capacity-provisioner = true + acstor-metrics-exporter = true + # Regex for which namespaces to scrape through pod annotation based scraping. + # This is none by default. + # Ex: Use 'namespace1|namespace2' to scrape the pods in the namespaces 'namespace1' and 'namespace2'. + pod-annotation-based-scraping: |- + podannotationnamespaceregex = "" + default-targets-metrics-keep-list: |- + kubelet = "" + coredns = "" + cadvisor = "" + kubeproxy = "" + apiserver = "" + kubestate = "" + nodeexporter = "" + windowsexporter = "" + windowskubeproxy = "" + podannotations = "" + kappiebasic = "" + networkobservabilityRetina = "" + networkobservabilityHubble = "" + networkobservabilityCilium = "" + controlplane-apiserver = "" + controlplane-cluster-autoscaler = "" + controlplane-kube-scheduler = "" + controlplane-kube-controller-manager = "" + controlplane-etcd = "" + acstor-capacity-provisioner = "" + acstor-metrics-exporter = "" + minimalingestionprofile = true + default-targets-scrape-interval-settings: |- + kubelet = "30s" + coredns = "30s" + cadvisor = "30s" + kubeproxy = "30s" + apiserver = "30s" + kubestate = "30s" + nodeexporter = "30s" + windowsexporter = "30s" + windowskubeproxy = "30s" + kappiebasic = "30s" + networkobservabilityRetina = "30s" + networkobservabilityHubble = "30s" + networkobservabilityCilium = "30s" + prometheuscollectorhealth = "30s" + acstor-capacity-provisioner = "30s" + acstor-metrics-exporter = "30s" + podannotations = "30s" + debug-mode: |- + enabled = false +metadata: + name: ama-metrics-settings-configmap + namespace: kube-system \ No newline at end of file diff --git a/tools/demo.md b/tools/demo.md new file mode 100644 index 000000000..fa9bf045e --- /dev/null +++ b/tools/demo.md @@ -0,0 +1,197 @@ +--- +title: Comprehensive Guide to Using Inspektor Gadget in Kubernetes +description: This Exec Doc provides a detailed walkthrough of a shell script that demonstrates various operations with the Inspektor Gadget in a Kubernetes environment. It explains each functional block, how the gadget plugin is installed, deployed, and used to run examples, export metrics, and verify configurations. +ms.topic: article +ms.date: 03/19/2025 +author: yourgithubusername +ms.author: yourmsalias +ms.custom: innovation-engine, kubernetes, gadget, monitoring +--- + +# Detailed Walkthrough: Inspektor Gadget Shell Script + +This document provides a step-by-step explanation of the provided shell script. The script demonstrates several operations related to the Inspektor Gadget in a Kubernetes environment. Each section below explains the purpose and the functionality of the code blocks that follow. The commands remain unchanged; only the documentation around them has been added for clarity. + +--- + +## Connecting to Your AKS Cluster + +Before running any commands, ensure that your local environment is connected to the desired AKS (Azure Kubernetes Service) cluster. Use the following command to retrieve the cluster credentials and configure `kubectl` to interact with the cluster: + +```bash +# Retrieve AKS cluster credentials: +az aks get-credentials --resource-group "myAKSResourceGroupabcf37" --name "myAKSClusterabcf37" +``` + +After executing this command, `kubectl` will be configured to communicate with the specified AKS cluster. + +--- + +## Viewing AKS Cluster Nodes + +In this section, the script lists the nodes of the current AKS (Azure Kubernetes Service) cluster using the Kubernetes CLI (`kubectl`). This allows you to verify that your cluster is up and running and view the status of the nodes. + +```bash +# Show AKS cluster: + +kubectl get nodes +``` + +After executing this block, the output will display the current nodes in the cluster along with their status, roles, and version information. + +--- + +## Installing the Inspektor Gadget Plugin + +This section installs the Inspektor Gadget plugin using `kubectl krew`. The gadget plugin extends kubectl with additional functionalities, enabling more effective monitoring and tracing within the cluster. + +```bash +# Install kubectl plugin: + +kubectl krew install gadget +``` + +Once installed, the gadget plugin is available for subsequent commands in the script. + +--- + +## Verifying Gadget Plugin Version + +Here, the script verifies the version and server status of the gadget plugin. It checks that the plugin is correctly installed and provides details about its client and server versions. The expected output is a client version (e.g., vX.Y.Z) and a note that the server version is not available. + +```bash +# Verify version and server status: + +kubectl gadget version +# Expected output: +# Client version: vX.Y.Z +# Server version: not available +``` + +This output helps determine that the gadget plugin is operational on your local client. You may compare the shown version with the expected output. + +--- + +## Deploying Inspektor Gadget and Re-Verification + +In this section, the script deploys the Inspektor Gadget in the Kubernetes environment. The command includes options to enable the OpenTelemetry (OTEL) metrics listener on the specified address (0.0.0.0:2223). After deploying, the version command is run again to verify that the gadget deployment is correctly configured, even though the server version remains "not available". + +```bash +# Deploy Inspektor Gadget: + +kubectl gadget deploy --otel-metrics-listen --otel-metrics-listen-address 0.0.0.0:2223 + +# Verify version and server status: + +kubectl gadget version +# Expected output: +# Client version: vX.Y.Z +# Server version: not available +``` + +This deployment sets up the gadget to collect the required metrics, and the follow-up version check confirms that the plugin is still active. + +--- + +## Demonstrating Gadget Usage with trace_exec + +This section illustrates different methods to run the gadget plugin using the `trace_exec` example. The commands include: + +1. Running the gadget with a specific trace_exec version. +2. Creating a test pod running Ubuntu in an interactive session, which is automatically removed after exit. +3. Running the gadget with JSON formatted output. +4. Running the gadget with filtering to display only processes with the command matching "bash". + +These examples show various ways to leverage the gadget for tracing executions in the cluster. + +```bash +# Run simple example with trace_exec with a 10-second timeout to prevent indefinite execution: +timeout 5s kubectl gadget run trace_exec || true + +# Create a background pod that will generate events for us to trace: +kubectl run demo-pod --image=ubuntu -- /bin/bash -c "for i in {1..30}; do echo Running commands...; ls -la /; sleep 1; done" + +# Wait briefly for the pod to start generating events +sleep 5 + +# Run gadget with JSON output and timeout +timeout 5s kubectl gadget run trace_exec --output jsonpretty || true + +# Run gadget with filtering and timeout +timeout 5s kubectl gadget run trace_exec --all-namespaces --filter proc.comm=bash || true +``` + +Each command demonstrates a different facet of the gadget's capabilities, from initiating traces to filtering outputs based on process names. + +--- + +## Creating Metrics Configuration for Alerting + +In this part of the script, a metrics configuration file is edited. The file (alert-bad-process.yaml) is intended to define rules to generate a metric based on certain events in the cluster. The metric, in this context, is used to track shell executions. + +```bash +# Generate a metric based on these events: + +cat alert-bad-process.yaml +``` + +--- + +## Exporting Metrics and Managing Gadget Lifecycle + +This section deploys the gadget manifest using the YAML file created in the previous section. The command includes several annotations to instruct the gadget to collect metrics. The process is detached so that it runs in the background. Subsequently, the script lists the running gadget instances and attaches to the deployed alert for further inspection if necessary. + +```bash +# Clean up any existing instance of the same name +kubectl gadget delete alert-bad-process + +# Run gadget manifest to export metrics: +kubectl gadget run -f alert-bad-process.yaml --annotate exec:metrics.collect=true,exec:metrics.implicit-counter.name=shell_executions,exec.k8s.namespace:metrics.type=key,exec.k8s.podname:metrics.type=key,exec.k8s.containername:metrics.type=key --detach + +# Verify gadget is running in headless mode: + +kubectl gadget list + +timeout 5s kubectl gadget attach alert-bad-process +``` + +These commands ensure that metrics are being collected as defined in the YAML manifest and verify that the gadget is running correctly in headless mode. + +--- + +## Verifying Prometheus Configuration for Metrics Collection + +This section checks the managed Prometheus configuration to ensure that it is set up to scrape metrics from the OTEL listener endpoint exposed on each Inspektor Gadget pod. The first command retrieves the relevant configmap, and the second command displays its full YAML definition with a pager for detailed inspection. Review the output to confirm that the configuration contains the expected annotation for pod-based scraping related to the gadget. + +```bash +# Configure managed Prometheus to collect data from the OTEL listener endpoint we expose on each IG pod? +# Documentation: https://learn.microsoft.com/en-us/azure/azure-monitor/containers/prometheus-metrics-scrape-configuration?tabs=CRDConfig%2CCRDScrapeConfig%2CConfigFileScrapeConfigBasicAuth%2CConfigFileScrapeConfigTLSAuth#configmaps + +kubectl get configmaps -n kube-system ama-metrics-settings-configmap + +# It should contain: pod-annotation-based-scraping: podannotationnamespaceregex = "gadget" +kubectl get configmaps -n kube-system ama-metrics-settings-configmap -o yaml | grep -A 5 "pod-annotation-based-scraping" +``` + +--- + +## Monitoring, Alerting, and Cleanup + +In the final part of the script, the focus shifts to monitoring and alerting: + +1. It provides guidance for viewing the `shell_executions_total` metric in the Grafana dashboard. +2. It suggests creating a Prometheus group alert with a rule that triggers when `shell_executions_total` exceeds 0. +3. Finally, the script undeploys the Inspektor Gadget to clean up resources. + +```bash +# Show shell_executions_total metric in Grafana dashboard: shell_executions_total +# Documentation: https://learn.microsoft.com/en-us/azure/managed-grafana/overview + +# Create a prometheus group alert with the rule "shell_executions_total > 0" +# Documentation: https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-rule-groups + +# Undeploy IG +kubectl gadget undeploy +``` + +These steps ensure that your metrics are visually accessible via Grafana and that alerts are configured for proactive monitoring. The final undeploy command removes the deployed gadget from the cluster, wrapping up the execution workflow. \ No newline at end of file diff --git a/tools/demo_notes.txt b/tools/demo_notes.txt new file mode 100644 index 000000000..2201427b5 --- /dev/null +++ b/tools/demo_notes.txt @@ -0,0 +1,71 @@ +# Show AKS cluster: + +kubectl get nodes + +# Install kubectl plugin: + +kubectl krew install gadget + +# Verify version and server status: + +kubectl gadget version +# Expected output: +# Client version: vX.Y.Z +# Server version: not available + +# Deploy Inspektor Gadget: + +kubectl gadget deploy --otel-metrics-listen --otel-metrics-listen-address 0.0.0.0:2223 + +# Verify version and server status: + +kubectl gadget version +# Expected output: +# Client version: vX.Y.Z +# Server version: not available + +# Run simple example with trace_exec: + +# Run gadget +kubectl gadget run trace_exec:v0.38.0 + +# Run test pod +kubectl run -ti 1p-demo-pod --rm --image=ubuntu -# /bin/bash + +# Run gadget with JSON +kubectl gadget run trace_exec:v0.38.0 --output jsonpretty + +# Run gadget with filtering + +kubectl gadget run trace_exec:v0.38.0 --all-namespaces --filter proc.comm=bash + +# Generate a metric based on these events: + +vi alert-bad-process.yaml + +# Run gadget manifest to export metrics: + +kubectl gadget run -f alert-bad-process.yaml --annotate exec:metrics.collect=true,exec:metrics.implicit-counter.name=shell_executions,exec.k8s.namespace:metrics.type=key,exec.k8s.podname:metrics.type=key,exec.k8s.containername:metrics.type=key --detach + +# Verify gadget is running in headless mode: + +kubectl gadget list + +kubectl gadget attach alert-bad-process + +# Configure managed Prometheus to collect data from the OTEL listener endpoint we expose on each IG pod? +# Documentation: https://learn.microsoft.com/en-us/azure/azure-monitor/containers/prometheus-metrics-scrape-configuration?tabs=CRDConfig%2CCRDScrapeConfig%2CConfigFileScrapeConfigBasicAuth%2CConfigFileScrapeConfigTLSAuth#configmaps + +kubectl get configmaps -n kube-system ama-metrics-settings-configmap + +# It should contain: pod-annotation-based-scraping: podannotationnamespaceregex = "gadget" +kubectl get configmaps -n kube-system ama-metrics-settings-configmap -o yaml | less + +# Show shell_executions_total metric in Grafana dashboard: shell_executions_total +# Documentation: https://learn.microsoft.com/en-us/azure/managed-grafana/overview + +# Create a prometheus group alert with the rule "shell_executions_total > 0" +# Documentation: https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-rule-groups + +# Undeploy IG +kubectl gadget undeploy diff --git a/tools/flatcar.md b/tools/flatcar.md deleted file mode 100644 index dbe355224..000000000 --- a/tools/flatcar.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -title: Running Flatcar Container Linux on Microsoft Azure -linktitle: Running on Microsoft Azure -weight: 10 -aliases: - - ../../os/booting-on-azure - - ../../cloud-providers/booting-on-azure ---- - -## Creating resource group via Microsoft Azure CLI - -Follow the [installation and configuration guides][azure-cli] for the Microsoft Azure CLI to set up your local installation. - -Instances on Microsoft Azure must be created within a resource group. Create a new resource group with the following command: - -```bash -az group create --name group-1 --location -``` - -Now that you have a resource group, you can choose a channel of Flatcar Container Linux you would like to install. - -## Using the official image from the Marketplace - -Official Flatcar Container Linux images for all channels are available in the Marketplace. -Flatcar is published by the `kinvolk` publisher on Marketplace. -Flatcar Container Linux is designed to be [updated automatically][update-docs] with different schedules per channel. Updating -can be [disabled][reboot-docs], although it is not recommended to do so. The [release notes][release-notes] contain -information about specific features and bug fixes. - -The following command will create a single instance through the Azure CLI. - -```bash -az vm image list --all -p kinvolk -f flatcar -s stable-gen2 --query '[-1]' # Query the image name urn specifier -``` - -```json -{ - "architecture": "x64", - "offer": "flatcar-container-linux-free", - "publisher": "kinvolk", - "sku": "stable-gen2", - "urn": "kinvolk:flatcar-container-linux-free:stable-gen2:3815.2.0", - "version": "3815.2.0" -} - -Use the offer named `flatcar-container-linux-free`, there is also a legacy offer called `flatcar-container-linux` with the same contents. -The SKU, which is the third element of the image URN, relates to one of the release channels and also depends on whether to use Hyper-V Generation 1 or 2 VM. -Generation 2 instance types use UEFI boot and should be preferred, the SKU matches the pattern `-gen`: `alpha-gen2`, `beta-gen2` or `stable-gen2`. -For Generation 1 instance types drop the `-gen2` from the SKU: `alpha`, `beta` or `stable`. -Note: _`az vm image list -s` flag matches parts of the SKU, which means that `-s stable` will return both the `stable` and `stable-gen2` SKUs._ - -Before being able to use the offers, you may need to accept the legal terms once, here done for `flatcar-container-linux-free` and `stable-gen2`: - -```bash -az vm image terms show --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 -az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 -``` - -For quick tests the official Azure CLI also supports an alias for the latest Flatcar stable image: -```bash -az vm create --name node-1 --resource-group group-1 --admin-username core --user-data config.ign --image FlatcarLinuxFreeGen2 -``` - -### CoreVM - -Flatcar images are also published under an offer called `flatcar-container-linux-corevm-amd64`. This offer does not require accepting image terms and does not require specifying plan information when creating instances or building derived images. The content of the images matches the other offers. -```bash -az vm image list --all -p kinvolk -f flatcar-container-linux-corevm-amd64 -s stable-gen2 --query '[-1]' -``` - -```json -{ - "architecture": "x64", - "offer": "flatcar-container-linux-corevm-amd64", - "publisher": "kinvolk", - "sku": "stable-gen2", - "urn": "kinvolk:flatcar-container-linux-corevm-amd64:stable-gen2:3815.2.0", - "version": "3815.2.0" -} -``` - -### ARM64 -Arm64 images are published under the offer called `flatcar-container-linux-corevm`. These are Generation 2 images, the only supported option on Azure for Arm64 instances, so the SKU contains only the release channel name without the `-gen2` suffix: `alpha`, `beta`, `stable`. This offer has the same properties as the `CoreVM` offer described above. - -```bash -az vm image list --all --architecture arm64 -p kinvolk -f flatcar -s stable --query '[-1]' -``` - -```json -{ - "architecture": "Arm64", - "offer": "flatcar-container-linux-corevm", - "publisher": "kinvolk", - "sku": "stable", - "urn": "kinvolk:flatcar-container-linux-corevm:stable:3815.2.0", - "version": "3815.2.0" -} -``` - - - -### Flatcar Pro Images - -Flatcar Pro images were paid marketplace images that came with commercial support and extra features. All the previous features of Flatcar Pro images, such as support for NVIDIA GPUs, are now available to all users in standard Flatcar marketplace images. - -### Plan information for building your image from the Marketplace Image - -When building an image based on the Marketplace image you sometimes need to specify the original plan. The plan name is the image SKU, e.g., `stable`, the plan product is the image offer, e.g., `flatcar-container-linux-free`, and the plan publisher is the same (`kinvolk`). - -## Community Shared Image Gallery - -While the Marketplace images are recommended, it sometimes might be easier or required to use Shared Image Galleries, e.g., when using Packer for Kubernetes CAPI images. - -A public Shared Image Gallery hosts recent Flatcar Stable images for amd64. Here is how to show the image definitions (for now you will only find `flatcar-stable-amd64`) and the image versions they provide: - -```bash -az sig image-definition list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --location westeurope -az sig image-version list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --gallery-image-definition flatcar-stable-amd64 --location westeurope -``` - -A second gallery `flatcar4capi-742ef0cb-dcaa-4ecb-9cb0-bfd2e43dccc0` exists for prebuilt Kubernetes CAPI images. It has image definitions for each CAPI version, e.g., `flatcar-stable-amd64-capi-v1.26.3` which provides recent Flatcar Stable versions. - -[flatcar-user]: https://groups.google.com/forum/#!forum/flatcar-linux-user -[etcd-docs]: https://etcd.io/docs -[quickstart]: ../ -[reboot-docs]: ../../setup/releases/update-strategies -[azure-cli]: https://docs.microsoft.com/en-us/cli/azure/overview -[butane-configs]: ../../provisioning/config-transpiler -[irc]: irc://irc.freenode.org:6667/#flatcar -[docs]: ../../ -[resource-group]: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#naming-rules-and-restrictions -[storage-account]: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#naming-storage-accounts -[azure-flatcar-image-upload]: https://github.com/flatcar/flatcar-cloud-image-uploader -[release-notes]: https://flatcar.org/releases -[update-docs]: ../../setup/releases/update-strategies \ No newline at end of file diff --git a/tools/flatcar_converted.md b/tools/flatcar_converted.md deleted file mode 100644 index a5797b922..000000000 --- a/tools/flatcar_converted.md +++ /dev/null @@ -1,187 +0,0 @@ ---- -title: 'Running Flatcar Container Linux on Microsoft Azure' -description: 'Deploy Flatcar Container Linux in Microsoft Azure by creating resource groups and using official marketplace images.' -ms.topic: article -ms.date: 10/10/2023 -author: naman-msft -ms.author: namanparikh -ms.custom: innovation-engine, azure, flatcar ---- - -## Creating resource group via Microsoft Azure CLI - -Follow the [installation and configuration guides][azure-cli] for the Microsoft Azure CLI to set up your local installation. - -Instances on Microsoft Azure must be created within a resource group. Create a new resource group with the following command: - -```bash -export RANDOM_SUFFIX=$(openssl rand -hex 3) -export RESOURCE_GROUP_NAME="group-1$RANDOM_SUFFIX" -export REGION="WestUS2" -az group create --name $RESOURCE_GROUP_NAME --location $REGION -``` - -Results: - - -```json -{ - "id": "/subscriptions/xxxxx/resourceGroups/group-1xxx", - "location": "WestUS2", - "managedBy": null, - "name": "group-1xxx", - "properties": { - "provisioningState": "Succeeded" - }, - "tags": null, - "type": "Microsoft.Resources/resourceGroups" -} -``` - -Now that you have a resource group, you can choose a channel of Flatcar Container Linux you would like to install. - -## Using the official image from the Marketplace - -Official Flatcar Container Linux images for all channels are available in the Marketplace. -Flatcar is published by the `kinvolk` publisher on Marketplace. -Flatcar Container Linux is designed to be [updated automatically][update-docs] with different schedules per channel. Updating -can be [disabled][reboot-docs], although it is not recommended to do so. The [release notes][release-notes] contain -information about specific features and bug fixes. - -The following command will query for the latest image URN specifier through the Azure CLI: - -```bash -az vm image list --all -p kinvolk -f flatcar -s stable-gen2 --query '[-1]' -``` - -Results: - - - -```json -{ - "architecture": "x64", - "offer": "flatcar-container-linux-free", - "publisher": "kinvolk", - "sku": "stable-gen2", - "urn": "kinvolk:flatcar-container-linux-free:stable-gen2:3815.2.0", - "version": "3815.2.0" -} -``` - -Use the offer named `flatcar-container-linux-free`; there is also a legacy offer called `flatcar-container-linux` with the same contents. -The SKU, which is the third element of the image URN, relates to one of the release channels and also depends on whether to use Hyper-V Generation 1 or 2 VMs. -Generation 2 instance types use UEFI boot and should be preferred, the SKU matches the pattern `-gen`: `alpha-gen2`, `beta-gen2` or `stable-gen2`. -For Generation 1 instance types drop the `-gen2` from the SKU: `alpha`, `beta` or `stable`. -Note: _`az vm image list -s` flag matches parts of the SKU, which means that `-s stable` will return both the `stable` and `stable-gen2` SKUs._ - -Before being able to use the offers, you may need to accept the legal terms once, which is demonstrated for `flatcar-container-linux-free` and `stable-gen2`: - -```bash -az vm image terms show --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 -az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 -``` - -For quick tests the official Azure CLI also supports an alias for the latest Flatcar stable image: - -```bash -az vm create --name node-1 --resource-group $RESOURCE_GROUP_NAME --admin-username core --image FlatcarLinuxFreeGen2 --generate-ssh-keys -``` - -Results: - - - -```json -{ - "fqdns": null, - "id": "/subscriptions/xxxxx/resourceGroups/group-1xxx/providers/Microsoft.Compute/virtualMachines/node-1", - "location": "WestUS2", - "name": "node-1", - "powerState": "VM running", - "provisioningState": "Succeeded", - "resourceGroup": "group-1xxx", - "zones": null -} -``` - -### CoreVM - -Flatcar images are also published under an offer called `flatcar-container-linux-corevm-amd64`. This offer does not require accepting image terms and does not require specifying plan information when creating instances or building derived images. The content of the images matches the other offers. - -```bash -az vm image list --all -p kinvolk -f flatcar-container-linux-corevm-amd64 -s stable-gen2 --query '[-1]' -``` - -Results: - - - -```json -{ - "architecture": "x64", - "offer": "flatcar-container-linux-corevm-amd64", - "publisher": "kinvolk", - "sku": "stable-gen2", - "urn": "kinvolk:flatcar-container-linux-corevm-amd64:stable-gen2:3815.2.0", - "version": "3815.2.0" -} -``` - -### ARM64 - -Arm64 images are published under the offer called `flatcar-container-linux-corevm`. These are Generation 2 images—the only supported option on Azure for Arm64 instances—so the SKU contains only the release channel name without the `-gen2` suffix: `alpha`, `beta`, or `stable`. This offer has the same properties as the `CoreVM` offer described above. - -```bash -az vm image list --all --architecture arm64 -p kinvolk -f flatcar -s stable --query '[-1]' -``` - -Results: - - - -```json -{ - "architecture": "Arm64", - "offer": "flatcar-container-linux-corevm", - "publisher": "kinvolk", - "sku": "stable", - "urn": "kinvolk:flatcar-container-linux-corevm:stable:3815.2.0", - "version": "3815.2.0" -} -``` - -### Flatcar Pro Images - -Flatcar Pro images were paid marketplace images that came with commercial support and extra features. All the previous features of Flatcar Pro images, such as support for NVIDIA GPUs, are now available to all users in standard Flatcar marketplace images. - -### Plan information for building your image from the Marketplace Image - -When building an image based on the Marketplace image you sometimes need to specify the original plan. The plan name is the image SKU (for example, `stable`), the plan product is the image offer (for example, `flatcar-container-linux-free`), and the plan publisher is the same (`kinvolk`). - -## Community Shared Image Gallery - -While the Marketplace images are recommended, it sometimes might be easier or required to use Shared Image Galleries—for example, when using Packer for Kubernetes CAPI images. - -A public Shared Image Gallery hosts recent Flatcar Stable images for amd64. Here is how to list the image definitions (for now you will only find `flatcar-stable-amd64`) and the image versions they provide: - -```bash -az sig image-definition list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --location westeurope -az sig image-version list-community --public-gallery-name flatcar-23485951-527a-48d6-9d11-6931ff0afc2e --gallery-image-definition flatcar-stable-amd64 --location westeurope -``` - -A second gallery, `flatcar4capi-742ef0cb-dcaa-4ecb-9cb0-bfd2e43dccc0`, exists for prebuilt Kubernetes CAPI images. It has image definitions for each CAPI version—for example, `flatcar-stable-amd64-capi-v1.26.3` provides recent Flatcar Stable versions. - -[flatcar-user]: https://groups.google.com/forum/#!forum/flatcar-linux-user -[etcd-docs]: https://etcd.io/docs -[quickstart]: ../ -[reboot-docs]: ../../setup/releases/update-strategies -[azure-cli]: https://docs.microsoft.com/en-us/cli/azure/overview -[butane-configs]: ../../provisioning/config-transpiler -[irc]: irc://irc.freenode.org:6667/#flatcar -[docs]: ../../ -[resource-group]: https://docs.microsoft.com/en-us/azure/architecture/best-practices/naming-conventions#naming-rules-and-restrictions -[storage-account]: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#naming-storage-accounts -[azure-flatcar-image-upload]: https://github.com/flatcar/flatcar-cloud-image-uploader -[release-notes]: https://flatcar.org/releases -[update-docs]: ../../setup/releases/update-strategies \ No newline at end of file From e75ea8ce19d6b644cd6cd490a7381638f8213e91 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 20 Mar 2025 16:50:41 -0700 Subject: [PATCH 46/87] updated files and added new docs --- tools/convert.md | 445 ----------------------------------------------- 1 file changed, 445 deletions(-) delete mode 100644 tools/convert.md diff --git a/tools/convert.md b/tools/convert.md deleted file mode 100644 index 2b3bb6b17..000000000 --- a/tools/convert.md +++ /dev/null @@ -1,445 +0,0 @@ ---- -title: Tutorial - Configure dynamic inventories for Azure Virtual Machines using Ansible -description: Learn how to populate your Ansible inventory dynamically from information in Azure -keywords: ansible, azure, devops, bash, cloudshell, dynamic inventory -ms.topic: tutorial -ms.date: 08/14/2024 -ms.custom: devx-track-ansible, devx-track-azurecli, devx-track-azurepowershell, linux-related-content ---- - -# Tutorial: Configure dynamic inventories of your Azure resources using Ansible - -[!INCLUDE [ansible-28-note.md](includes/ansible-28-note.md)] - -The [Ansible dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_dynamic_inventory.html) feature removes the burden of maintaining static inventory files. - -In this tutorial, you use Azure's dynamic-inventory plug-in to populate your Ansible inventory. - -In this article, you learn how to: - -> [!div class="checklist"] -> * Configure two test virtual machines. -> * Add tags to Azure virtual machines -> * Generate a dynamic inventory -> * Use conditional and keyed groups to populate group memberships -> * Run playbooks against groups within the dynamic inventory - -## Prerequisites - -[!INCLUDE [open-source-devops-prereqs-azure-subscription.md](../includes/open-source-devops-prereqs-azure-subscription.md)] -[!INCLUDE [open-source-devops-prereqs-create-service-principal.md](../includes/open-source-devops-prereqs-create-service-principal.md)] -[!INCLUDE [ansible-prereqs-cloudshell-use-or-vm-creation2.md](includes/ansible-prereqs-cloudshell-use-or-vm-creation2.md)] - -## Create Azure VMs - -1. Sign in to the [Azure portal](https://go.microsoft.com/fwlink/p/?LinkID=525040). - -1. Open [Cloud Shell](/azure/cloud-shell/overview). - -1. Create an Azure resource group to hold the virtual machines for this tutorial. - - > [!IMPORTANT] - > The Azure resource group you create in this step must have a name that is entirely lower-case. Otherwise, the generation of the dynamic inventory will fail. - - # [Azure CLI](#tab/azure-cli) - ```azurecli-interactive - az group create --resource-group ansible-inventory-test-rg --location eastus - ``` - # [Azure PowerShell](#tab/azure-powershell) - - ```azurepowershell - New-AzResourceGroup -Name ansible-inventory-test-rg -Location eastus - ``` - --- - -1. Create two Linux virtual machines on Azure using one of the following techniques: - - - **Ansible playbook** - The article, [Create a basic Linux virtual machine in Azure with Ansible](./vm-configure.md) and [Create a basic Windows virtual machine in Azure with Ansible](./vm-configure-windows.md) illustrates how to create a virtual machine from an Ansible playbook. - - - **Azure CLI** - Issue each of the following commands in the Cloud Shell to create the two virtual machines: - - # [Azure CLI](#tab/azure-cli) - ```azurecli-interactive - az vm create \ - --resource-group ansible-inventory-test-rg \ - --name win-vm \ - --image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest \ - --admin-username azureuser \ - --admin-password - - az vm create \ - --resource-group ansible-inventory-test-rg \ - --name linux-vm \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --admin-password - ``` - - - # [Azure PowerShell](#tab/azure-powershell) - - ```azurepowershell - $adminUsername = "azureuser" - $adminPassword = ConvertTo-SecureString -AsPlainText -Force - $credential = New-Object System.Management.Automation.PSCredential ($adminUsername, $adminPassword); - - New-AzVM ` - -ResourceGroupName ansible-inventory-test-rg ` - -Location eastus ` - -Image MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest ` - -Name win-vm ` - -OpenPorts 3389 ` - -Credential $credential - - New-AzVM ` - -ResourceGroupName ansible-inventory-test-rg ` - -Location eastus ` - -Image Ubuntu2204 ` - -Name linux-vm ` - -OpenPorts 22 ` - -Credential $credential - ``` - --- - - Replace the `` your password. - -## Add application role tags - -Tags are used to organize and categorize Azure resources. Assigning the Azure VMs an application role allows you to use the tags as group names within the Azure dynamic inventory. - -Run the following commands to update the VM tags: - -# [Azure CLI](#tab/azure-cli) -```azurecli-interactive -az vm update \ ---resource-group ansible-inventory-test-rg \ ---name linux-vm \ ---set tags.applicationRole='message-broker' - -az vm update \ ---resource-group ansible-inventory-test-rg \ ---name win-vm \ ---set tags.applicationRole='web-server' -``` - -# [Azure PowerShell](#tab/azure-powershell) - -```azurepowershell -Get-AzVM -Name win-vm -ResourceGroupName ansible-inventory-test-rg-pwsh | Update-AzVM -Tag @{"applicationRole"="web-server"} - -Get-AzVM -Name linux-vm -ResourceGroupName ansible-inventory-test-rg-pwsh | Update-AzVM -Tag @{"applicationRole"="message-broker"} -``` - ---- - -Learn more about Azure tagging strategies at [Define your tagging strategy](/azure/cloud-adoption-framework/ready/azure-best-practices/resource-tagging). - -## Generate a dynamic inventory - -Ansible provides an [Azure dynamic-inventory plug-in](https://github.com/ansible/ansible/blob/stable-2.9/lib/ansible/plugins/inventory/azure_rm.py). - -The following steps walk you through using the plug-in: - -1. Create a dynamic inventory named `myazure_rm.yml` - - ```yml - plugin: azure_rm - include_vm_resource_groups: - - ansible-inventory-test-rg - auth_source: auto - ``` - - **Key point:** - * Ansible uses the inventory file name and extension to identify which inventory plug-in to use. To use the Azure dynamic inventory plug-in, the file must end with `azure_rm` and have an extension of either `yml` or `yaml`. - -1. Run the following command to query the VMs within the resource group: - - ```bash - ansible-inventory -i myazure_rm.yml --graph - ``` - -1. When you run the command, you see results similar to the following output: - - ```output - @all: - |--@ungrouped: - | |--linux-vm_cdb4 - | |--win-vm_3211 - ``` - -Both VMs belong to the `ungrouped` group, which is a child of the `all` group in the Ansible inventory. - -**Key point**: -* By default the Azure dynamic inventory plug-in returns globally unique names. For this reason, the VM names may contain extra characters. You can disable that behavior by adding `plain_host_names: yes` to the dynamic inventory. - -## Find Azure VM hostvars - -Run the following command to view all the `hostvars`: - -```bash -ansible-inventory -i myazure_rm.yml --list -``` - -```output -{ - "_meta": { - "hostvars": { - "linux-vm_cdb4": { - "ansible_host": "52.188.118.79", - "availability_zone": null, - "computer_name": "linux-vm", - "default_inventory_hostname": "linux-vm_cdb4", - "id": "/subscriptions//resourceGroups/ansible-inventory-test-rg/providers/Microsoft.Compute/virtualMachines/linux-vm", - "image": { - "offer": "0001-com-ubuntu-server-jammy", - "publisher": "Canonical", - "sku": "22_04-lts-gen2", - "version": "latest" - }, - ..., - "tags": { - "applicationRole": "message-broker" - }, - ... - }, - "win-vm_3211": { - "ansible_host": "52.188.112.110", - "availability_zone": null, - "computer_name": "win-vm", - "default_inventory_hostname": "win-vm_3211", - "id": "/subscriptions//resourceGroups/ansible-inventory-test-rg/providers/Microsoft.Compute/virtualMachines/win-vm", - "image": { - "offer": "WindowsServer", - "publisher": "MicrosoftWindowsServer", - "sku": "2019-Datacenter", - "version": "latest" - }, - ... - "tags": { - "applicationRole": "web-server" - }, - ... - } - } - }, - ... - } -} -``` - -By pulling information from Azure, the dynamic inventory populates the `hostvars` for each Azure VM. Those `hostvars` are then to determine the VM group memberships within the Ansible inventory. - -## Assign group membership with conditional_groups - -Each conditional group is made of two parts. The name of the group and the condition for adding a member to the group. - -Use the property `image.offer` to create conditional group membership for the _linux-vm_. - -Open the `myazure_rm.yml` dynamic inventory and add the following `conditional_group`: - -```yml -plugin: azure_rm -include_vm_resource_groups: - - ansible-inventory-test-rg -auth_source: auto -conditional_groups: - linux: "'ubuntu' in image.offer" - windows: "'WindowsServer' in image.offer" -``` - -Run the `ansible-inventory` with the `--graph` option: - -```bash -ansible-inventory -i myazure_rm.yml --graph -``` - -```output -@all: - |--@linux: - | |--linux-vm_cdb4 - |--@ungrouped: - |--@windows: - | |--win-vm_3211 -``` - -From the output, you can see the VMs are no longer associated with the `ungrouped` group. Instead, each VM is assigned to a new group created by the dynamic inventory. - -**Key point**: -* Conditional groups allow you to name specific groups within your inventory and populate them using `hostvars`. - -## Assign group membership with keyed_groups - -Keyed groups assign group membership the same way conditional groups do, but when using a keyed group the group name is also dynamically populated. - -Add the following keyed_group to the `myazure_rm.yml` dynamic inventory: - -```yml -plugin: azure_rm -include_vm_resource_groups: - - ansible-inventory-test-rg -auth_source: auto -conditional_groups: - linux: "'ubuntu' in image.offer" - windows: "'WindowsServer' in image.offer" -keyed_groups: - - key: tags.applicationRole -``` - -Run the `ansible-inventory` with the `--graph` option: - -```bash -ansible-inventory -i myazure_rm.yml --graph -``` - -```output -@all: - |--@_message_broker: - | |--linux-vm_cdb4 - |--@_web_server: - | |--win-vm_3211 - |--@linux: - | |--linux-vm_cdb4 - |--@ungrouped: - |--@windows: - | |--win-vm_3211 -``` - -From the output, you see two more groups `_message_broker` and `_web_server`. By using a keyed group, the `applicationRole` tag populates the group names and group memberships. - -**Key point**: -* By default, keyed groups include a separator. To remove the separator, add `separator: ""` under the key property. - -## Run playbooks with group name patterns - -Use the groups created by the dynamic inventory to target subgroups. - -1. Create a playbook called `win_ping.yml` with the following contents: - - ```yml - --- - - hosts: windows - gather_facts: false - - vars_prompt: - - name: username - prompt: "Enter local username" - private: false - - name: password - prompt: "Enter password" - - vars: - ansible_user: "{{ username }}" - ansible_password: "{{ password }}" - ansible_connection: winrm - ansible_winrm_transport: ntlm - ansible_winrm_server_cert_validation: ignore - - tasks: - - name: run win_ping - win_ping: - ``` - -1. Run the `win_ping.yml` playbook. - - ```bash - ansible-playbook win_ping.yml -i myazure_rm.yml - ``` - - When prompted, enter the `username` and `password` for the Azure Windows VM. - - ```output - Enter local username: azureuser - Enter password: - - PLAY [windows] ************************************************************************************************************************************** - - TASK [run win_ping] ********************************************************************************************************************************* - ok: [win-vm_3211] - - PLAY RECAP ****************************************************************************************************************************************** - win-vm_3211 : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 - ``` - - > [!IMPORTANT] - > If you get the error `winrm or requests is not installed: No module named 'winrm'`, install pywinrm with the following command: `pip install "pywinrm>=0.3.0"` - -1. Create a second playbook named `ping.yml` with the following contents: - - ```yml - --- - - hosts: all - gather_facts: false - - vars_prompt: - - name: username - prompt: "Enter ssh user" - - name: password - prompt: "Enter password for ssh user" - - vars: - ansible_user: "{{ username }}" - ansible_password: "{{ password }}" - ansible_ssh_common_args: '-o StrictHostKeyChecking=no' - - tasks: - - name: run ping - ping: - ``` - -1. Run the `ping.yml` playbook. - - ```bash - ansible-playbook ping.yml -i myazure_rm.yml - ``` - - When prompted, enter the `username` and `password` for the Azure Linux VM. - - ```output - Enter ssh username: azureuser - Enter password for ssh user: - - PLAY [linux] ******************************************************************************************************* - - TASK [run ping] **************************************************************************************************** - ok: [linux-vm_cdb4] - - PLAY RECAP ********************************************************************************************************* - linux-vm_cdb4 : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 - ``` - -## Clean up resources - -# [Azure CLI](#tab/azure-cli) - -1. Run [az group delete](/cli/azure/group#az-group-delete) to delete the resource group. All resources within the resource group are deleted. - - ```azurecli - az group delete --name - ``` - -1. Verify that the resource group was deleted by using [az group show](/cli/azure/group#az-group-show). - - ```azurecli - az group show --name - ``` - -# [Azure PowerShell](#tab/azure-powershell) - -1. Run [Remove-AzResourceGroup](/powershell/module/az.resources/Remove-AzResourceGroup) to delete the resource group. All resources within the resource group are deleted. - - ```azurepowershell - Remove-AzResourceGroup -Name - ``` - -1. Verify that the resource group was deleted by using [Get-AzResourceGroup](/powershell/module/az.resources/Get-AzResourceGroup). - - ```azurepowershell - Get-AzResourceGroup -Name - ``` - ---- - -## Next steps - -> [!div class="nextstepaction"] -> [Quickstart: Configure Linux virtual machines in Azure using Ansible](./vm-configure.md) \ No newline at end of file From a9a0647c5d14b0beaa66ca811d01778395c37cd2 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 20 Mar 2025 17:19:41 -0700 Subject: [PATCH 47/87] updated docs --- tools/ada.py | 2 +- tools/demo.md | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tools/ada.py b/tools/ada.py index b4116edc7..cd46c17d8 100644 --- a/tools/ada.py +++ b/tools/ada.py @@ -110,7 +110,7 @@ - title = the title of the Exec Doc - description = the description of the Exec Doc - ms.topic = what kind of a doc it is e.g. article, blog, etc. - - ms.date = the date the Exec Doc was last updated by author + - ms.date = the current date in the format MM/DD/YYYY - author = author's GitHub username - ms.author = author's username (e.g. Microsoft Alias) - **ms.custom = comma-separated list of tags to identify the Exec Doc (innovation-engine is the one tag that is mandatory in this list)** diff --git a/tools/demo.md b/tools/demo.md index fa9bf045e..b6d0fca46 100644 --- a/tools/demo.md +++ b/tools/demo.md @@ -108,8 +108,10 @@ These examples show various ways to leverage the gadget for tracing executions i # Run simple example with trace_exec with a 10-second timeout to prevent indefinite execution: timeout 5s kubectl gadget run trace_exec || true +kubectl delete pod demo-pod + # Create a background pod that will generate events for us to trace: -kubectl run demo-pod --image=ubuntu -- /bin/bash -c "for i in {1..30}; do echo Running commands...; ls -la /; sleep 1; done" +kubectl run demo-pod --image=ubuntu -- /bin/bash -c "for i in {1..11}; do echo Running commands...; ls -la /; sleep 1; done" # Wait briefly for the pod to start generating events sleep 5 @@ -118,7 +120,7 @@ sleep 5 timeout 5s kubectl gadget run trace_exec --output jsonpretty || true # Run gadget with filtering and timeout -timeout 5s kubectl gadget run trace_exec --all-namespaces --filter proc.comm=bash || true +timeout 5s kubectl gadget run trace_exec --all-namespaces --filter proc.comm=bash || echo "Attachment timed out, continuing with demo" ``` Each command demonstrates a different facet of the gadget's capabilities, from initiating traces to filtering outputs based on process names. @@ -139,7 +141,7 @@ cat alert-bad-process.yaml ## Exporting Metrics and Managing Gadget Lifecycle -This section deploys the gadget manifest using the YAML file created in the previous section. The command includes several annotations to instruct the gadget to collect metrics. The process is detached so that it runs in the background. Subsequently, the script lists the running gadget instances and attaches to the deployed alert for further inspection if necessary. +This section deploys the gadget manifest using the YAML file created in the previous section. The command includes several annotations to instruct the gadget to collect metrics. The process is detached so that it runs in the background. Subsequently, the script lists the running gadget instances. ```bash # Clean up any existing instance of the same name @@ -147,12 +149,6 @@ kubectl gadget delete alert-bad-process # Run gadget manifest to export metrics: kubectl gadget run -f alert-bad-process.yaml --annotate exec:metrics.collect=true,exec:metrics.implicit-counter.name=shell_executions,exec.k8s.namespace:metrics.type=key,exec.k8s.podname:metrics.type=key,exec.k8s.containername:metrics.type=key --detach - -# Verify gadget is running in headless mode: - -kubectl gadget list - -timeout 5s kubectl gadget attach alert-bad-process ``` These commands ensure that metrics are being collected as defined in the YAML manifest and verify that the gadget is running correctly in headless mode. From e38f6b925dac82f87d367184f7a1836b03508a56 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Wed, 26 Mar 2025 15:24:24 -0700 Subject: [PATCH 48/87] added new docs --- .../aks/azure-cni-powered-by-cilium.md | 229 +++ ...ontainer-instances-quickstart-terraform.md | 93 ++ .../virtual-machines/linux/cloud-init.txt | 41 + .../articles/virtual-machines/linux/main.tf | 124 ++ .../virtual-machines/linux/outputs.tf | 7 + .../virtual-machines/linux/providers.tf | 22 + .../linux/quick-create-terraform.md | 24 +- .../articles/virtual-machines/linux/ssh.tf | 25 + .../linux/tutorial-automate-vm-deployment.md | 193 +++ .../linux/tutorial-lamp-stack.md | 186 +++ .../virtual-machines/linux/variables.tf | 17 + .../articles/ansible/vm-configure.md | 138 ++ .../tutorial-azure-linux-add-nodepool.md | 142 ++ .../tutorial-azure-linux-create-cluster.md | 123 ++ .../tutorial-azure-linux-migration.md | 144 ++ .../tutorial-azure-linux-telemetry-monitor.md | 129 ++ .../tutorial-azure-linux-upgrade.md | 108 ++ .../azure-stack-quick-create-vm-linux-cli.md | 188 +++ .../quickstart-install-connect-docker.md | 1245 +++++++++++++++++ tools/abc.md | 253 ++++ tools/ansible.md | 157 +++ tools/demo_notes.txt | 71 - tools/main.tf | 36 + tools/main.yml | 64 + tools/outputs.tf | 3 + tools/providers.tf | 16 + tools/test.md | 355 +++++ tools/variables.tf | 57 + 28 files changed, 4103 insertions(+), 87 deletions(-) create mode 100644 scenarios/azure-aks-docs/articles/aks/azure-cni-powered-by-cilium.md create mode 100644 scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/cloud-init.txt create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/main.tf create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/outputs.tf create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/providers.tf create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/ssh.tf create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-automate-vm-deployment.md create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lamp-stack.md create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/variables.tf create mode 100644 scenarios/azure-dev-docs/articles/ansible/vm-configure.md create mode 100644 scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md create mode 100644 scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md create mode 100644 scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-migration.md create mode 100644 scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md create mode 100644 scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-upgrade.md create mode 100644 scenarios/azure-stack-docs/azure-stack/user/azure-stack-quick-create-vm-linux-cli.md create mode 100644 scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md create mode 100644 tools/abc.md create mode 100644 tools/ansible.md delete mode 100644 tools/demo_notes.txt create mode 100644 tools/main.tf create mode 100644 tools/main.yml create mode 100644 tools/outputs.tf create mode 100644 tools/providers.tf create mode 100644 tools/test.md create mode 100644 tools/variables.tf diff --git a/scenarios/azure-aks-docs/articles/aks/azure-cni-powered-by-cilium.md b/scenarios/azure-aks-docs/articles/aks/azure-cni-powered-by-cilium.md new file mode 100644 index 000000000..a29bdb9c7 --- /dev/null +++ b/scenarios/azure-aks-docs/articles/aks/azure-cni-powered-by-cilium.md @@ -0,0 +1,229 @@ +--- +title: Configure Azure CNI Powered by Cilium in Azure Kubernetes Service (AKS) +description: Learn how to create an Azure Kubernetes Service (AKS) cluster with Azure CNI Powered by Cilium. +ms.topic: how-to +ms.date: 02/12/2024 +author: asudbring +ms.author: allensu +ms.subservice: aks-networking +ms.custom: references_regions, devx-track-azurecli, build-2023, innovation-engine +--- + +# Configure Azure CNI Powered by Cilium in Azure Kubernetes Service (AKS) + +Azure CNI Powered by Cilium combines the robust control plane of Azure CNI with the data plane of [Cilium](https://cilium.io/) to provide high-performance networking and security. + +By making use of eBPF programs loaded into the Linux kernel and a more efficient API object structure, Azure CNI Powered by Cilium provides the following benefits: + +- Functionality equivalent to existing Azure CNI and Azure CNI Overlay plugins + +- Improved Service routing + +- More efficient network policy enforcement + +- Better observability of cluster traffic + +- Support for larger clusters (more nodes, pods, and services) + +## IP Address Management (IPAM) with Azure CNI Powered by Cilium + +Azure CNI Powered by Cilium can be deployed using two different methods for assigning pod IPs: + +- Assign IP addresses from an overlay network (similar to Azure CNI Overlay mode) + +- Assign IP addresses from a virtual network (similar to existing Azure CNI with Dynamic Pod IP Assignment) + +If you aren't sure which option to select, read ["Choosing a network model to use."](./azure-cni-overlay.md#choosing-a-network-model-to-use) + +## Versions + +| Kubernetes Version | Cilium Version | +|--------------------|----------------| +| 1.27 (LTS) | 1.13.18 | +| 1.28 (End of Life) | 1.13.18 | +| 1.29 | 1.14.19 | +| 1.30 (LTS) | 1.14.19 | +| 1.31 | 1.16.6 | +| 1.32 | 1.17.0 | + +See [Supported Kubernetes Versions](./supported-kubernetes-versions.md) for more information on AKS versioning and release timelines. + +## Network Policy Enforcement + +Cilium enforces [network policies to allow or deny traffic between pods](./operator-best-practices-network.md#control-traffic-flow-with-network-policies). With Cilium, you don't need to install a separate network policy engine such as Azure Network Policy Manager or Calico. + +## Limitations + +Azure CNI powered by Cilium currently has the following limitations: + +* Available only for Linux and not for Windows. + +* Cilium L7 policy enforcement is disabled. + +* Network policies can't use `ipBlock` to allow access to node or pod IPs. See [frequently asked questions](#frequently-asked-questions) for details and recommended workaround. + +* Multiple Kubernetes services can't use the same host port with different protocols (for example, TCP or UDP) ([Cilium issue #14287](https://github.com/cilium/cilium/issues/14287)). + +* Network policies may be enforced on reply packets when a pod connects to itself via service cluster IP ([Cilium issue #19406](https://github.com/cilium/cilium/issues/19406)). + +* Network policies aren't applied to pods using host networking (`spec.hostNetwork: true`) because these pods use the host identity instead of having individual identities. + +## Prerequisites + +* Azure CLI version 2.48.1 or later. Run `az --version` to see the currently installed version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). + +* If using ARM templates or the REST API, the AKS API version must be 2022-09-02-preview or later. + +> [!NOTE] +> Previous AKS API versions (2022-09-02preview to 2023-01-02preview) used the field [`networkProfile.ebpfDataplane=cilium`](https://github.com/Azure/azure-rest-api-specs/blob/06dbe269f7d9c709cc225c92358b38c3c2b74d60/specification/containerservice/resource-manager/Microsoft.ContainerService/aks/preview/2022-09-02-preview/managedClusters.json#L6939-L6955). AKS API versions since 2023-02-02preview use the field [`networkProfile.networkDataplane=cilium`](https://github.com/Azure/azure-rest-api-specs/blob/06dbe269f7d9c709cc225c92358b38c3c2b74d60/specification/containerservice/resource-manager/Microsoft.ContainerService/aks/preview/2023-02-02-preview/managedClusters.json#L7152-L7173) to enable Azure CNI Powered by Cilium. + +## Create a new AKS Cluster with Azure CNI Powered by Cilium + +### Create a Resource Group + +Use the following command to create a resource group. Environment variables are declared and used below to replace placeholders. + +```azurecli-interactive +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP="myResourceGroup$RANDOM_SUFFIX" +export LOCATION="EastUS2" + +az group create \ + --name $RESOURCE_GROUP \ + --location $LOCATION +``` + +Result: + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupxxx", + "location": "WestUS2", + "name": "myResourceGroupxxx", + "provisioningState": "Succeeded" +} +``` + +### Assign IP addresses from an overlay network + +Use the following commands to create a cluster with an overlay network and Cilium. Environment variables are declared and used below to replace placeholders. + +```azurecli-interactive +export CLUSTER_NAME="myAKSCluster$RANDOM_SUFFIX" + +az aks create \ + --name $CLUSTER_NAME \ + --resource-group $RESOURCE_GROUP \ + --location $LOCATION \ + --network-plugin azure \ + --network-plugin-mode overlay \ + --pod-cidr 192.168.0.0/16 \ + --network-dataplane cilium \ + --generate-ssh-keys +``` + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.ContainerService/managedClusters/myAKSClusterxxx", + "location": "WestUS2", + "name": "myAKSClusterxxx", + "provisioningState": "Succeeded" +} +``` + +> [!NOTE] +> The `--network-dataplane cilium` flag replaces the deprecated `--enable-ebpf-dataplane` flag used in earlier versions of the aks-preview CLI extension. + +## Frequently asked questions + +- **Can I customize Cilium configuration?** + + No, AKS manages the Cilium configuration and it can't be modified. We recommend that customers who require more control use [AKS BYO CNI](./use-byo-cni.md) and install Cilium manually. + +- **Can I use `CiliumNetworkPolicy` custom resources instead of Kubernetes `NetworkPolicy` resources?** + + `CiliumNetworkPolicy` custom resources are partially supported. Customers may use FQDN filtering as part of the [Advanced Container Networking Services](./advanced-container-networking-services-overview.md) feature bundle. + + This `CiliumNetworkPolicy` example demonstrates a sample matching pattern for services that match the specified label. + + ```yaml + apiVersion: "cilium.io/v2" + kind: CiliumNetworkPolicy + metadata: + name: "example-fqdn" + spec: + endpointSelector: + matchLabels: + foo: bar + egress: + - toFQDNs: + - matchPattern: "*.example.com" + ``` + +- **Why is traffic being blocked when the `NetworkPolicy` has an `ipBlock` that allows the IP address?** + + A limitation of Azure CNI Powered by Cilium is that a `NetworkPolicy`'s `ipBlock` can't select pod or node IPs. + + For example, this `NetworkPolicy` has an `ipBlock` that allows all egress to `0.0.0.0/0`: + ```yaml + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: example-ipblock + spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 # This will still block pod and node IPs. + ``` + + However, when this `NetworkPolicy` is applied, Cilium blocks egress to pod and node IPs even though the IPs are within the `ipBlock` CIDR. + + As a workaround, you can add `namespaceSelector` and `podSelector` to select pods. This example selects all pods in all namespaces: + ```yaml + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: example-ipblock + spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + - namespaceSelector: {} + - podSelector: {} + ``` + + > [!NOTE] + > It isn't currently possible to specify a `NetworkPolicy` with an `ipBlock` to allow traffic to node IPs. +- **Does AKS configure CPU or memory limits on the Cilium `daemonset`?** + + No, AKS doesn't configure CPU or memory limits on the Cilium `daemonset` because Cilium is a critical system component for pod networking and network policy enforcement. + +- **Does Azure CNI powered by Cilium use Kube-Proxy?** + + No, AKS clusters created with network dataplane as Cilium don't use Kube-Proxy. + If the AKS clusters are on [Azure CNI Overlay](./azure-cni-overlay.md) or [Azure CNI with dynamic IP allocation](./configure-azure-cni-dynamic-ip-allocation.md) and are upgraded to AKS clusters running Azure CNI powered by Cilium, new nodes workloads are created without kube-proxy. Older workloads are also migrated to run without kube-proxy as a part of this upgrade process. + +## Next steps + +Learn more about networking in AKS in the following articles: + +* [Upgrade Azure CNI IPAM modes and Dataplane Technology](upgrade-azure-cni.md). + +* [Use a static IP address with the Azure Kubernetes Service (AKS) load balancer](static-ip.md) + +* [Use an internal load balancer with Azure Container Service (AKS)](internal-lb.md) + +* [Create a basic ingress controller with external network connectivity][aks-ingress-basic] + + +[aks-ingress-basic]: ingress-basic.md \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md new file mode 100644 index 000000000..3c7b39cab --- /dev/null +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md @@ -0,0 +1,93 @@ +--- +title: 'Quickstart: Create an Azure Container Instance with a public IP address using Terraform' +description: 'In this article, you create an Azure Container Instance with a public IP address using Terraform' +ms.topic: quickstart +ms.service: azure-container-instances +ms.date: 08/29/2024 +ms.custom: devx-track-terraform, linux-related-content +author: TomArcherMsft +ms.author: tarcher +content_well_notification: + - AI-contribution +ai-usage: ai-assisted +--- + +# Quickstart: Create an Azure Container Instance with a public IP address using Terraform + +Use Azure Container Instances to run serverless Docker containers in Azure with simplicity and speed. Deploy an application to a container instance on-demand when you don't need a full container orchestration platform like Azure Kubernetes Service. In this article, you use [Terraform](/azure/terraform) to deploy an isolated Docker container and make its web application available with a public IP address. + +[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] + +In this article, you learn how to: + +> [!div class="checklist"] +> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/resource_group/pet) +> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) +> * Create a random value for the container name using [random_string](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) +> * Create an Azure container group using [azurerm_container_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_group) + +## Prerequisites + +- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) + +## Implement the Terraform code + +> [!NOTE] +> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip/TestRecord.md). +> +> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) + +1. Create a directory in which to test and run the sample Terraform code and make it the current directory. + +1. Create a file named `main.tf` and insert the following code: + + [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/main.tf)] + +1. Create a file named `outputs.tf` and insert the following code: + + [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/outputs.tf)] + +1. Create a file named `providers.tf` and insert the following code: + + [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/providers.tf)] + +1. Create a file named `variables.tf` and insert the following code: + + [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/variables.tf)] + +## Initialize Terraform + +[!INCLUDE [terraform-init.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-init.md)] + +## Create a Terraform execution plan + +[!INCLUDE [terraform-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan.md)] + +## Apply a Terraform execution plan + +[!INCLUDE [terraform-apply-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-apply-plan.md)] + +## Verify the results + +1. When you apply the execution plan, Terraform outputs the public IP address. To display the IP address again, run [terraform output](https://developer.hashicorp.com/terraform/cli/commands/output). + + ```console + terraform output -raw container_ipv4_address + ``` + +1. Enter the sample's public IP address in your browser's address bar. + + :::image type="content" source="./media/container-instances-quickstart-terraform/azure-container-instances-demo.png" alt-text="Screenshot of the Azure Container Instances sample page"::: + +## Clean up resources + +[!INCLUDE [terraform-plan-destroy.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan-destroy.md)] + +## Troubleshoot Terraform on Azure + +[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) + +## Next steps + +> [!div class="nextstepaction"] +> [Tutorial: Create a container image for deployment to Azure Container Instances](./container-instances-tutorial-prepare-app.md) \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/cloud-init.txt b/scenarios/azure-compute-docs/articles/virtual-machines/linux/cloud-init.txt new file mode 100644 index 000000000..6f0566319 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/cloud-init.txt @@ -0,0 +1,41 @@ +#cloud-config +package_upgrade: true +packages: + - nginx + - nodejs + - npm +write_files: + - owner: www-data:www-data + path: /etc/nginx/sites-available/default + defer: true + content: | + server { + listen 80; + location / { + proxy_pass http://localhost:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + - owner: azureuser:azureuser + path: /home/azureuser/myapp/index.js + defer: true + content: | + var express = require('express') + var app = express() + var os = require('os'); + app.get('/', function (req, res) { + res.send('Hello World from host ' + os.hostname() + '!') + }) + app.listen(3000, function () { + console.log('Hello world app listening on port 3000!') + }) +runcmd: + - service nginx restart + - cd "/home/azureuser/myapp" + - npm init + - npm install express -y + - nodejs index.js \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/main.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/main.tf new file mode 100644 index 000000000..9482a95fa --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/main.tf @@ -0,0 +1,124 @@ +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +resource "azurerm_resource_group" "rg" { + location = var.resource_group_location + name = random_pet.rg_name.id +} + +# Create virtual network +resource "azurerm_virtual_network" "my_terraform_network" { + name = "myVnet" + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name +} + +# Create subnet +resource "azurerm_subnet" "my_terraform_subnet" { + name = "mySubnet" + resource_group_name = azurerm_resource_group.rg.name + virtual_network_name = azurerm_virtual_network.my_terraform_network.name + address_prefixes = ["10.0.1.0/24"] +} + +# Create public IPs +resource "azurerm_public_ip" "my_terraform_public_ip" { + name = "myPublicIP" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + allocation_method = "Dynamic" +} + +# Create Network Security Group and rule +resource "azurerm_network_security_group" "my_terraform_nsg" { + name = "myNetworkSecurityGroup" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + security_rule { + name = "SSH" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefix = "*" + destination_address_prefix = "*" + } +} + +# Create network interface +resource "azurerm_network_interface" "my_terraform_nic" { + name = "myNIC" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + + ip_configuration { + name = "my_nic_configuration" + subnet_id = azurerm_subnet.my_terraform_subnet.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = azurerm_public_ip.my_terraform_public_ip.id + } +} + +# Connect the security group to the network interface +resource "azurerm_network_interface_security_group_association" "example" { + network_interface_id = azurerm_network_interface.my_terraform_nic.id + network_security_group_id = azurerm_network_security_group.my_terraform_nsg.id +} + +# Generate random text for a unique storage account name +resource "random_id" "random_id" { + keepers = { + # Generate a new ID only when a new resource group is defined + resource_group = azurerm_resource_group.rg.name + } + + byte_length = 8 +} + +# Create storage account for boot diagnostics +resource "azurerm_storage_account" "my_storage_account" { + name = "diag${random_id.random_id.hex}" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + account_tier = "Standard" + account_replication_type = "LRS" +} + +# Create virtual machine +resource "azurerm_linux_virtual_machine" "my_terraform_vm" { + name = "myVM" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + network_interface_ids = [azurerm_network_interface.my_terraform_nic.id] + size = "Standard_DS1_v2" + + os_disk { + name = "myOsDisk" + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts-gen2" + version = "latest" + } + + computer_name = "hostname" + admin_username = var.username + + admin_ssh_key { + username = var.username + public_key = azapi_resource_action.ssh_public_key_gen.output.publicKey + } + + boot_diagnostics { + storage_account_uri = azurerm_storage_account.my_storage_account.primary_blob_endpoint + } +} \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/outputs.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/outputs.tf new file mode 100644 index 000000000..f7d0c3184 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/outputs.tf @@ -0,0 +1,7 @@ +output "resource_group_name" { + value = azurerm_resource_group.rg.name +} + +output "public_ip_address" { + value = azurerm_linux_virtual_machine.my_terraform_vm.public_ip_address +} \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/providers.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/providers.tf new file mode 100644 index 000000000..158b40408 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/providers.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">=0.12" + + required_providers { + azapi = { + source = "azure/azapi" + version = "~>1.5" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} + +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md index 9ef176aa1..d6e92dc62 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md @@ -52,8 +52,7 @@ In this article, you learn how to: 1. Create a file named providers.tf and insert the following code: -```bash -cat <<'EOF' > providers.tf +```text terraform { required_version = ">=0.12" @@ -76,13 +75,11 @@ terraform { provider "azurerm" { features {} } -EOF ``` 1. Create a file named ssh.tf and insert the following code: -```bash -cat <<'EOF' > ssh.tf +```text resource "random_pet" "ssh_key_name" { prefix = "ssh" separator = "" @@ -107,13 +104,11 @@ resource "azapi_resource" "ssh_public_key" { output "key_data" { value = azapi_resource_action.ssh_public_key_gen.output.publicKey } -EOF ``` 1. Create a file named main.tf and insert the following code: -```bash -cat <<'EOF' > main.tf +```text resource "random_pet" "rg_name" { prefix = var.resource_group_name_prefix } @@ -238,16 +233,14 @@ resource "azurerm_linux_virtual_machine" "my_terraform_vm" { storage_account_uri = azurerm_storage_account.my_storage_account.primary_blob_endpoint } } -EOF ``` 1. Create a file named variables.tf and insert the following code: -```bash -cat <<'EOF' > variables.tf +```text variable "resource_group_location" { type = string - default = "eastus" + default = "eastus2" description = "Location of the resource group." } @@ -262,13 +255,11 @@ variable "username" { description = "The username for the local account that will be created on the new VM." default = "azureadmin" } -EOF ``` 1. Create a file named outputs.tf and insert the following code: -```bash -cat <<'EOF' > outputs.tf +```text output "resource_group_name" { value = azurerm_resource_group.rg.name } @@ -276,7 +267,6 @@ output "resource_group_name" { output "public_ip_address" { value = azurerm_linux_virtual_machine.my_terraform_vm.public_ip_address } -EOF ``` ## Initialize Terraform @@ -284,6 +274,8 @@ EOF In this section, Terraform is initialized; this command downloads the Azure provider required to manage your Azure resources. Before running the command, ensure you are in the directory where you created the Terraform files. You can set any necessary environment variables here. ```bash +# Set your preferred Azure region (defaults to eastus2 if not specified) +export TF_VAR_resource_group_location="eastus2" export TERRAFORM_DIR=$(pwd) terraform init -upgrade ``` diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/ssh.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/ssh.tf new file mode 100644 index 000000000..11de7c0a4 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/ssh.tf @@ -0,0 +1,25 @@ +resource "random_pet" "ssh_key_name" { + prefix = "ssh" + separator = "" +} + +resource "azapi_resource_action" "ssh_public_key_gen" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + resource_id = azapi_resource.ssh_public_key.id + action = "generateKeyPair" + method = "POST" + + response_export_values = ["publicKey", "privateKey"] +} + +resource "azapi_resource" "ssh_public_key" { + type = "Microsoft.Compute/sshPublicKeys@2022-11-01" + name = random_pet.ssh_key_name.id + location = azurerm_resource_group.rg.location + parent_id = azurerm_resource_group.rg.id +} + +output "key_data" { + value = azapi_resource_action.ssh_public_key_gen.output.publicKey + sensitive = true +} \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-automate-vm-deployment.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-automate-vm-deployment.md new file mode 100644 index 000000000..5b46a9fd8 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-automate-vm-deployment.md @@ -0,0 +1,193 @@ +--- +title: Tutorial - Customize a Linux VM with cloud-init in Azure +description: In this tutorial, you learn how to use cloud-init and Key Vault to customize Linux VMs the first time they boot in Azure +author: ju-shim +ms.service: azure-virtual-machines +ms.collection: linux +ms.topic: tutorial +ms.date: 10/18/2023 +ms.author: jushiman +ms.reviewer: mattmcinnes +ms.custom: mvc, devx-track-azurecli, linux-related-content, innovation-engine +--- + +# Tutorial - How to use cloud-init to customize a Linux virtual machine in Azure on first boot + +**Applies to:** :heavy_check_mark: Linux VMs :heavy_check_mark: Flexible scale sets + +In a previous tutorial, you learned how to SSH to a virtual machine (VM) and manually install NGINX. To create VMs in a quick and consistent manner, some form of automation is typically desired. A common approach to customize a VM on first boot is to use [cloud-init](https://cloudinit.readthedocs.io). In this tutorial you learn how to: + +> [!div class="checklist"] +> * Create a cloud-init config file +> * Create a VM that uses a cloud-init file +> * View a running Node.js app after the VM is created +> * Use Key Vault to securely store certificates +> * Automate secure deployments of NGINX with cloud-init + +If you choose to install and use the CLI locally, this tutorial requires that you are running the Azure CLI version 2.0.30 or later. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI]( /cli/azure/install-azure-cli). + +## Cloud-init overview + +[Cloud-init](https://cloudinit.readthedocs.io) is a widely used approach to customize a Linux VM as it boots for the first time. You can use cloud-init to install packages and write files, or to configure users and security. As cloud-init runs during the initial boot process, there are no additional steps or required agents to apply your configuration. + +Cloud-init also works across distributions. For example, you don't use **apt-get install** or **yum install** to install a package. Instead you can define a list of packages to install. Cloud-init automatically uses the native package management tool for the distro you select. + +We are working with our partners to get cloud-init included and working in the images that they provide to Azure. For detailed information cloud-init support for each distribution, see [Cloud-init support for VMs in Azure](using-cloud-init.md). + +## Create cloud-init config file + +To see cloud-init in action, create a VM that installs NGINX and runs a simple 'Hello World' Node.js app. The following cloud-init configuration installs the required packages, creates a Node.js app, then initializes and starts the app. + +At your bash prompt or in the Cloud Shell, create a file named *cloud-init.txt* and paste the following configuration. For example, type `sensible-editor cloud-init.txt` to create the file and see a list of available editors. Make sure that the whole cloud-init file is copied correctly, especially the first line: + +```yaml +#cloud-config +package_upgrade: true +packages: + - nginx + - nodejs + - npm +write_files: + - owner: www-data:www-data + path: /etc/nginx/sites-available/default + defer: true + content: | + server { + listen 80; + location / { + proxy_pass http://localhost:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + } + - owner: azureuser:azureuser + path: /home/azureuser/myapp/index.js + defer: true + content: | + var express = require('express') + var app = express() + var os = require('os'); + app.get('/', function (req, res) { + res.send('Hello World from host ' + os.hostname() + '!') + }) + app.listen(3000, function () { + console.log('Hello world app listening on port 3000!') + }) +runcmd: + - service nginx restart + - cd "/home/azureuser/myapp" + - npm init + - npm install express -y + - nodejs index.js +``` + +For more information about cloud-init configuration options, see [cloud-init config examples](https://cloudinit.readthedocs.io/en/latest/topics/examples.html). + +## Create virtual machine + +Before you can create a VM, create a resource group with [az group create](/cli/azure/group#az-group-create). The following example creates a resource group. In these commands, a random suffix is appended to the resource group and VM names to prevent name collisions during repeated deployments. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP="myResourceGroupAutomate$RANDOM_SUFFIX" +export REGION="eastus2" +az group create --name $RESOURCE_GROUP --location $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupAutomatexxx", + "location": "eastus", + "managedBy": null, + "name": "myResourceGroupAutomatexxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +Now create a VM with [az vm create](/cli/azure/vm#az-vm-create). Use the `--custom-data` parameter to pass in your cloud-init config file. Provide the full path to the *cloud-init.txt* config if you saved the file outside of your present working directory. The following example creates a VM; note that the VM name is also appended with the random suffix. + +```bash +export VM_NAME="myAutomatedVM$RANDOM_SUFFIX" +az vm create \ + --resource-group $RESOURCE_GROUP \ + --name $VM_NAME \ + --image Ubuntu2204 \ + --admin-username azureuser \ + --generate-ssh-keys \ + --custom-data cloud-init.txt +``` + +Results: + + +```JSON +{ + "fqdns": "", + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupAutomatexxx/providers/Microsoft.Compute/virtualMachines/myAutomatedVMxxx", + "location": "eastus", + "name": "myAutomatedVMxxx", + "powerState": "VM running", + "publicIpAddress": "x.x.x.x", + "resourceGroup": "myResourceGroupAutomatexxx", + "zones": "" +} +``` + +It takes a few minutes for the VM to be created, the packages to install, and the app to start. There are background tasks that continue to run after the Azure CLI returns you to the prompt. It may be another couple of minutes before you can access the app. When the VM has been created, take note of the `publicIpAddress` displayed by the Azure CLI. This address is used to access the Node.js app via a web browser. + +To allow web traffic to reach your VM, open port 80 from the Internet with [az vm open-port](/cli/azure/vm#az-vm-open-port): + +```bash +az vm open-port --port 80 --resource-group $RESOURCE_GROUP --name $VM_NAME +``` + +Results: + + +```JSON +{ + "endpoints": [ + { + "name": "80", + "protocol": "tcp", + "publicPort": 80, + "privatePort": 80 + } + ], + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupAutomatexxx/providers/Microsoft.Compute/virtualMachines/myAutomatedVMxxx", + "location": "eastus", + "name": "myAutomatedVMxxx" +} +``` + +## Test web app + +Now you can open a web browser and enter *http://* in the address bar. Provide your own public IP address from the VM create process. Your Node.js app is displayed as shown in the following example: + +![View running NGINX site](./media/tutorial-automate-vm-deployment/nginx.png) + +## Next steps + +In this tutorial, you configured VMs on first boot with cloud-init. You learned how to: + +> [!div class="checklist"] +> * Create a cloud-init config file +> * Create a VM that uses a cloud-init file +> * View a running Node.js app after the VM is created +> * Use Key Vault to securely store certificates +> * Automate secure deployments of NGINX with cloud-init + +Advance to the next tutorial to learn how to create custom VM images. + +> [!div class="nextstepaction"] +> [Create custom VM images](./tutorial-custom-images.md) \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lamp-stack.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lamp-stack.md new file mode 100644 index 000000000..ae437b898 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lamp-stack.md @@ -0,0 +1,186 @@ +--- +title: Tutorial - Deploy LAMP and WordPress on a VM +description: In this tutorial, you learn how to install the LAMP stack, and WordPress, on a Linux virtual machine in Azure. +author: ju-shim +ms.collection: linux +ms.service: azure-virtual-machines +ms.devlang: azurecli +ms.custom: linux-related-content, innovation-engine +ms.topic: tutorial +ms.date: 4/4/2023 +ms.author: mattmcinnes +ms.reviewer: cynthn +#Customer intent: As an IT administrator, I want to learn how to install the LAMP stack so that I can quickly prepare a Linux VM to run web applications. +--- + +# Tutorial: Install a LAMP stack on an Azure Linux VM + +**Applies to:** :heavy_check_mark: Linux VMs + +This article walks you through how to deploy an Apache web server, MySQL, and PHP (the LAMP stack) on an Ubuntu VM in Azure. To see the LAMP server in action, you can optionally install and configure a WordPress site. In this tutorial you learn how to: + +> [!div class="checklist"] +> * Create an Ubuntu VM +> * Open port 80 for web traffic +> * Install Apache, MySQL, and PHP +> * Verify installation and configuration +> * Install WordPress + +This setup is for quick tests or proof of concept. For more on the LAMP stack, including recommendations for a production environment, see the [Ubuntu documentation](https://help.ubuntu.com/community/ApacheMySQLPHP). + +This tutorial uses the CLI within the [Azure Cloud Shell](/azure/cloud-shell/overview), which is constantly updated to the latest version. To open the Cloud Shell, select **Try it** from the top of any code block. + +If you choose to install and use the CLI locally, this tutorial requires that you're running the Azure CLI version 2.0.30 or later. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI]( /cli/azure/install-azure-cli). + +## Create a resource group + +Create a resource group with the [az group create](/cli/azure/group) command. An Azure resource group is a logical container into which Azure resources are deployed and managed. + +The following example creates a resource group using environment variables and appends a random suffix to ensure uniqueness. + +```azurecli-interactive +export REGION="eastus2" +export RANDOM_SUFFIX="$(openssl rand -hex 3)" +export MY_RESOURCE_GROUP_NAME="myResourceGroup${RANDOM_SUFFIX}" +az group create --name "${MY_RESOURCE_GROUP_NAME}" --location $REGION +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupxxxxx", + "location": "eastus", + "name": "myResourceGroupxxxxx", + "properties": { + "provisioningState": "Succeeded" + } +} +``` + +## Create a virtual machine + +Create a VM with the [az vm create](/cli/azure/vm) command. + +The following example creates a VM using environment variables. It creates a VM named *myVM* and creates SSH keys if they don't already exist in a default key location. To use a specific set of keys, use the `--ssh-key-value` option. The command also sets *azureuser* as an administrator user name. You use this name later to connect to the VM. + +```azurecli-interactive +export MY_VM_NAME="myVM${RANDOM_SUFFIX}" +export IMAGE="Ubuntu2204" +export ADMIN_USERNAME="azureuser" +az vm create \ + --resource-group "${MY_RESOURCE_GROUP_NAME}" \ + --name $MY_VM_NAME \ + --image $IMAGE \ + --admin-username $ADMIN_USERNAME \ + --generate-ssh-keys +``` + +When the VM has been created, the Azure CLI shows information similar to the following example. Take note of the `publicIpAddress`. This address is used to access the VM in later steps. + +```output +{ + "fqdns": "", + "id": "/subscriptions//resourceGroups/myResourceGroup/providers/Microsoft.Compute/virtualMachines/myVM", + "location": "eastus", + "macAddress": "00-0D-3A-23-9A-49", + "powerState": "VM running", + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "40.68.254.142", + "resourceGroup": "myResourceGroup" +} +``` + +## Open port 80 for web traffic + +By default, only SSH connections are allowed into Linux VMs deployed in Azure. Because this VM is going to be a web server, you need to open port 80 from the internet. Use the [az vm open-port](/cli/azure/vm) command to open the desired port. + +```azurecli-interactive +az vm open-port --port 80 --resource-group "${MY_RESOURCE_GROUP_NAME}" --name $MY_VM_NAME +``` + +For more information about opening ports to your VM, see [Open ports](nsg-quickstart.md). + +## SSH into your VM + +If you don't already know the public IP address of your VM, run the [az network public-ip list](/cli/azure/network/public-ip) command. You need this IP address for several later steps. + +```azurecli-interactive +export PUBLIC_IP=$(az network public-ip list --resource-group "${MY_RESOURCE_GROUP_NAME}" --query [].ipAddress -o tsv) +``` + +Use the `ssh` command to create an SSH session with the virtual machine. Substitute the correct public IP address of your virtual machine. + +## Install Apache, MySQL, and PHP + +Run the following command to update Ubuntu package sources and install Apache, MySQL, and PHP. Note the caret (^) at the end of the command, which is part of the `lamp-server^` package name. + +```bash +ssh -o StrictHostKeyChecking=no azureuser@$PUBLIC_IP "sudo DEBIAN_FRONTEND=noninteractive apt-get -y install lamp-server^" +``` + +You're prompted to install the packages and other dependencies. This process installs the minimum required PHP extensions needed to use PHP with MySQL. + +## Verify Apache + +Check the version of Apache with the following command: +```bash +ssh -o StrictHostKeyChecking=no azureuser@$PUBLIC_IP "apache2 -v" +``` + +With Apache installed, and port 80 open to your VM, the web server can now be accessed from the internet. To view the Apache2 Ubuntu Default Page, open a web browser, and enter the public IP address of the VM. Use the public IP address you used to SSH to the VM: + +![Apache default page][3] + +## Verify and secure MySQL + +Check the version of MySQL with the following command (note the capital `V` parameter): + +```bash +ssh -o StrictHostKeyChecking=no azureuser@$PUBLIC_IP "mysql -V" +``` + +To help secure the installation of MySQL, including setting a root password, you can run the `sudo mysql_secure_installation` command. This command prompts you to answer several questions to help secure your MySQL installation. + +You can optionally set up the Validate Password Plugin (recommended). Then, set a password for the MySQL root user, and configure the remaining security settings for your environment. We recommend that you answer "Y" (yes) to all questions. + +If you want to try MySQL features (create a MySQL database, add users, or change configuration settings), login to MySQL. This step isn't required to complete this tutorial. For doing this, you can use the `sudo mysql -u root -p` command in your VM and then enter your root password when prompted. This command connects to your VM via SSH and launches the MySQL command line client as the root user. + +When done, exit the mysql prompt by typing `\q`. + +## Verify PHP + +Check the version of PHP with the following command: + +```bash +ssh -o StrictHostKeyChecking=no azureuser@$PUBLIC_IP "php -v" +``` + +If you want to test further, you can create a quick PHP info page to view in a browser. The following command creates the PHP info page `sudo sh -c 'echo \"\" > /var/www/html/info.php` + +Now you can check the PHP info page you created. Open a browser and go to `http://yourPublicIPAddress/info.php`. Substitute the public IP address of your VM. It should look similar to this image. + +![PHP info page][2] + +[!INCLUDE [virtual-machines-linux-tutorial-wordpress.md](../includes/virtual-machines-linux-tutorial-wordpress.md)] + +## Next steps + +In this tutorial, you deployed a LAMP server in Azure. You learned how to: + +> [!div class="checklist"] +> * Create an Ubuntu VM +> * Open port 80 for web traffic +> * Install Apache, MySQL, and PHP +> * Verify installation and configuration +> * Install WordPress on the LAMP server + +Advance to the next tutorial to learn how to secure web servers with TLS/SSL certificates. + +> [!div class="nextstepaction"] +> [Secure web server with TLS](tutorial-secure-web-server.md) + +[2]: ./media/tutorial-lamp-stack/phpsuccesspage.png +[3]: ./media/tutorial-lamp-stack/apachesuccesspage.png \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/variables.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/variables.tf new file mode 100644 index 000000000..37a12b1f4 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/variables.tf @@ -0,0 +1,17 @@ +variable "resource_group_location" { + type = string + default = "eastus2" + description = "Location of the resource group." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random ID so name is unique in your Azure subscription." +} + +variable "username" { + type = string + description = "The username for the local account that will be created on the new VM." + default = "azureadmin" +} \ No newline at end of file diff --git a/scenarios/azure-dev-docs/articles/ansible/vm-configure.md b/scenarios/azure-dev-docs/articles/ansible/vm-configure.md new file mode 100644 index 000000000..e785a1230 --- /dev/null +++ b/scenarios/azure-dev-docs/articles/ansible/vm-configure.md @@ -0,0 +1,138 @@ +--- +title: Create a Linux virtual machines in Azure using Ansible +description: Learn how to create a Linux virtual machine in Azure using Ansible +keywords: ansible, azure, devops, virtual machine +ms.topic: tutorial +ms.date: 08/14/2024 +ms.custom: devx-track-ansible, linux-related-content +--- + +# Create a Linux virtual machines in Azure using Ansible + +This article presents a sample Ansible playbook for configuring a Linux virtual machine. + +In this article, you learn how to: + +> [!div class="checklist"] +> * Create a resource group +> * Create a virtual network +> * Create a public IP address +> * Create a network security group +> * Create a virtual network interface card +> * Create a virtual machine + +## 1. Configure your environment + +[!INCLUDE [open-source-devops-prereqs-azure-sub.md](../includes/open-source-devops-prereqs-azure-subscription.md)] +[!INCLUDE [ansible-prereqs-cloudshell-use-or-vm-creation1.md](includes/ansible-prereqs-cloudshell-use-or-vm-creation1.md)] + +## 2. Create an SSH key pair + +1. Run the following command. When prompted, specify the files to be created in the following directory: `/home/azureuser/.ssh/authorized_keys`. + + ```bash + ssh-keygen -m PEM -t rsa -b 4096 + ``` + +1. Copy the contents of the public key file. By default, the public key file is named `id_rsa.pub`. The value is a long string starting with "ssh-rsa ". You'll need this value in the next step. + +## 3. Implement the Ansible playbook + +1. Create a directory in which to test and run the sample Ansible code and make it the current directory. + +1. Create a file named `main.yml` and insert the following code. Replace the `` placeholder with the public key value from the previous step. + + ```yaml + - name: Create Azure VM + hosts: localhost + connection: local + tasks: + - name: Create resource group + azure_rm_resourcegroup: + name: myResourceGroup + location: eastus + - name: Create virtual network + azure_rm_virtualnetwork: + resource_group: myResourceGroup + name: myVnet + address_prefixes: "10.0.0.0/16" + - name: Add subnet + azure_rm_subnet: + resource_group: myResourceGroup + name: mySubnet + address_prefix: "10.0.1.0/24" + virtual_network: myVnet + - name: Create public IP address + azure_rm_publicipaddress: + resource_group: myResourceGroup + allocation_method: Static + name: myPublicIP + register: output_ip_address + - name: Public IP of VM + debug: + msg: "The public IP is {{ output_ip_address.state.ip_address }}." + - name: Create Network Security Group that allows SSH + azure_rm_securitygroup: + resource_group: myResourceGroup + name: myNetworkSecurityGroup + rules: + - name: SSH + protocol: Tcp + destination_port_range: 22 + access: Allow + priority: 1001 + direction: Inbound + - name: Create virtual network interface card + azure_rm_networkinterface: + resource_group: myResourceGroup + name: myNIC + virtual_network: myVnet + subnet: mySubnet + public_ip_name: myPublicIP + security_group: myNetworkSecurityGroup + - name: Create VM + azure_rm_virtualmachine: + resource_group: myResourceGroup + name: myVM + vm_size: Standard_DS1_v2 + admin_username: azureuser + ssh_password_enabled: false + ssh_public_keys: + - path: /home/azureuser/.ssh/authorized_keys + key_data: "" + network_interfaces: myNIC + image: + offer: 0001-com-ubuntu-server-jammy + publisher: Canonical + sku: 22_04-lts + version: latest + ``` + +## 4. Run the playbook + +[!INCLUDE [ansible-playbook.md](includes/ansible-playbook.md)] + +## 5. Verify the results + +Run [az vm list](/cli/azure/vm#az-vm-list) to verify the VM was created. + + ```azurecli + az vm list -d -o table --query "[?name=='myVM']" + ``` + +## 6. Connect to the VM + +Run the SSH command to connect to your new Linux VM. Replace the <ip-address> placeholder with the IP address from the previous step. + +```bash +ssh azureuser@ -i /home/azureuser/.ssh/authorized_keys/id_rsa +``` + +## Clean up resources + +[!INCLUDE [ansible-delete-resource-group.md](includes/ansible-delete-resource-group.md)] + +## Next steps + +> [!div class="nextstepaction"] +> [Manage a Linux virtual machine in Azure using Ansible](./vm-manage.md) \ No newline at end of file diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md new file mode 100644 index 000000000..91ed7b58e --- /dev/null +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md @@ -0,0 +1,142 @@ +--- +title: Azure Linux Container Host for AKS tutorial - Add an Azure Linux node pool to your existing AKS cluster +description: In this Azure Linux Container Host for AKS tutorial, you learn how to add an Azure Linux node pool to your existing cluster. +author: suhuruli +ms.author: suhuruli +ms.service: microsoft-linux +ms.custom: linux-related-content, innovation-engine +ms.topic: tutorial +ms.date: 06/06/2023 +--- + +# Tutorial: Add an Azure Linux node pool to your existing AKS cluster + +In AKS, nodes with the same configurations are grouped together into node pools. Each pool contains the VMs that run your applications. In the previous tutorial, you created an Azure Linux Container Host cluster with a single node pool. To meet the varying compute or storage requirements of your applications, you can create additional user node pools. + +In this tutorial, part two of five, you learn how to: + +> [!div class="checklist"] +> +> * Add an Azure Linux node pool. +> * Check the status of your node pools. + +In later tutorials, you learn how to migrate nodes to Azure Linux and enable telemetry to monitor your clusters. + +## Prerequisites + +* In the previous tutorial, you created and deployed an Azure Linux Container Host cluster. If you haven't done these steps and would like to follow along, start with [Tutorial 1: Create a cluster with the Azure Linux Container Host for AKS](./tutorial-azure-linux-create-cluster.md). +* You need the latest version of Azure CLI. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). + +## Add an Azure Linux node pool + +To add an Azure Linux node pool into your existing cluster, use the `az aks nodepool add` command and specify `--os-sku AzureLinux`. The following example creates a node pool named *ALnodepool* that runs three nodes in the *testAzureLinuxCluster* cluster in the *testAzureLinuxResourceGroup* resource group. Environment variables are declared below and a random suffix is appended to the resource group and cluster names to ensure uniqueness. + +```azurecli-interactive +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export NODEPOOL_NAME="np$RANDOM_SUFFIX" + +az aks nodepool add \ + --resource-group $RESOURCE_GROUP \ + --cluster-name $CLUSTER_NAME \ + --name $NODEPOOL_NAME \ + --node-count 3 \ + --os-sku AzureLinux +``` + + +```JSON +{ + "agentPoolType": "VirtualMachineScaleSets", + "count": 3, + "name": "alnodepool", + "osType": "Linux", + "provisioningState": "Succeeded", + "resourceGroup": "testAzureLinuxResourceGroupxxxxx", + "type": "Microsoft.ContainerService/managedClusters/agentPools" +} +``` + +> [!NOTE] +> The name of a node pool must start with a lowercase letter and can only contain alphanumeric characters. For Linux node pools the length must be between one and 12 characters. + +## Check the node pool status + +To see the status of your node pools, use the `az aks nodepool list` command and specify your resource group and cluster name. The same environment variable values declared earlier are used here. + +```azurecli-interactive +export CLUSTER_NAME="myAKSClusterabcf37" +export RESOURCE_GROUP="myAKSResourceGroupabcf37" +az aks nodepool list --resource-group $RESOURCE_GROUP --cluster-name $CLUSTER_NAME +``` + + +```output +[ + { + "agentPoolType": "VirtualMachineScaleSets", + "availabilityZones": null, + "count": 1, + "enableAutoScaling": false, + "enableEncryptionAtHost": false, + "enableFips": false, + "enableNodePublicIp": false, + "id": "/subscriptions/REDACTED/resourcegroups/myAKSResourceGroupxxxxx/providers/Microsoft.ContainerService/managedClusters/myAKSClusterxxxxx/agentPools/nodepoolx", + "maxPods": 110, + "mode": "System", + "name": "nodepoolx", + "nodeImageVersion": "AKSUbuntu-1804gen2containerd-2023.06.06", + "orchestratorVersion": "1.25.6", + "osDiskSizeGb": 128, + "osDiskType": "Managed", + "osSku": "Ubuntu", + "osType": "Linux", + "powerState": { + "code": "Running" + }, + "provisioningState": "Succeeded", + "resourceGroup": "myAKSResourceGroupxxxxx", + "type": "Microsoft.ContainerService/managedClusters/agentPools", + "vmSize": "Standard_DS2_v2" + }, + { + "agentPoolType": "VirtualMachineScaleSets", + "availabilityZones": null, + "count": 3, + "enableAutoScaling": false, + "enableEncryptionAtHost": false, + "enableFips": false, + "enableNodePublicIp": false, + "id": "/subscriptions/REDACTED/resourcegroups/myAKSResourceGroupxxxxx/providers/Microsoft.ContainerService/managedClusters/myAKSClusterxxxxx/agentPools/npxxxxxx", + "maxPods": 110, + "mode": "User", + "name": "npxxxxxx", + "nodeImageVersion": "AzureLinuxContainerHost-2023.06.06", + "orchestratorVersion": "1.25.6", + "osDiskSizeGb": 128, + "osDiskType": "Managed", + "osSku": "AzureLinux", + "osType": "Linux", + "powerState": { + "code": "Running" + }, + "provisioningState": "Succeeded", + "resourceGroup": "myAKSResourceGroupxxxxx", + "type": "Microsoft.ContainerService/managedClusters/agentPools", + "vmSize": "Standard_DS2_v2" + } +] +``` + +## Next steps + +In this tutorial, you added an Azure Linux node pool to your existing cluster. You learned how to: + +> [!div class="checklist"] +> +> * Add an Azure Linux node pool. +> * Check the status of your node pools. + +In the next tutorial, you learn how to migrate existing nodes to Azure Linux. + +> [!div class="nextstepaction"] +> [Migrating to Azure Linux](./tutorial-azure-linux-migration.md) \ No newline at end of file diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md new file mode 100644 index 000000000..e4bb92c5c --- /dev/null +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md @@ -0,0 +1,123 @@ +--- +title: Azure Linux Container Host for AKS tutorial - Create a cluster +description: In this Azure Linux Container Host for AKS tutorial, you will learn how to create an AKS cluster with Azure Linux. +author: suhuruli +ms.author: suhuruli +ms.service: microsoft-linux +ms.custom: linux-related-content, innovation-engine +ms.topic: tutorial +ms.date: 04/18/2023 +--- + +# Tutorial: Create a cluster with the Azure Linux Container Host for AKS + +To create a cluster with the Azure Linux Container Host, you will use: +1. Azure resource groups, a logical container into which Azure resources are deployed and managed. +1. [Azure Kubernetes Service (AKS)](/azure/aks/intro-kubernetes), a hosted Kubernetes service that allows you to quickly create a production ready Kubernetes cluster. + +In this tutorial, part one of five, you will learn how to: + +> [!div class="checklist"] +> * Install the Kubernetes CLI, `kubectl`. +> * Create an Azure resource group. +> * Create and deploy an Azure Linux Container Host cluster. +> * Configure `kubectl` to connect to your Azure Linux Container Host cluster. + +In later tutorials, you'll learn how to add an Azure Linux node pool to an existing cluster and migrate existing nodes to Azure Linux. + +## Prerequisites + +- You need the latest version of Azure CLI. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). + +## Create a resource group + +When creating a resource group, it is required to specify a location. This location is: +- The storage location of your resource group metadata. +- Where your resources will run in Azure if you don't specify another region when creating a resource. + +Before running the command, environment variables are declared to ensure unique resource names for each deployment. + +```bash +export RANDOM_SUFFIX=c97736 +export RESOURCE_GROUP_NAME="testAzureLinuxResourceGroup${RANDOM_SUFFIX}" +export REGION="EastUS2" +az group create --name $RESOURCE_GROUP_NAME --location $REGION +``` + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/testAzureLinuxResourceGroupxxxxx", + "location": "EastUS2", + "managedBy": null, + "name": "testAzureLinuxResourceGroupxxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +> [!NOTE] +> The above example uses *WestUS2*, but Azure Linux Container Host clusters are available in all regions. + +## Create an Azure Linux Container Host cluster + +Create an AKS cluster using the `az aks create` command with the `--os-sku` parameter to provision the Azure Linux Container Host with an Azure Linux image. The following example creates an Azure Linux Container Host cluster. A unique cluster name is generated using the same RANDOM_SUFFIX used when creating the resource group. + +```bash +export CLUSTER_NAME="testAzureLinuxCluster${RANDOM_SUFFIX}" +az aks create --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME --os-sku AzureLinux +``` + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/testAzureLinuxResourceGroupxxxxx/providers/Microsoft.ContainerService/managedClusters/testAzureLinuxClusterxxxxx", + "location": "WestUS2", + "name": "testAzureLinuxClusterxxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "type": "Microsoft.ContainerService/managedClusters" +} +``` + +After a few minutes, the command completes and returns JSON-formatted information about the cluster. + +## Connect to the cluster using kubectl + +To configure `kubectl` to connect to your Kubernetes cluster, use the `az aks get-credentials` command. The following example gets credentials for the Azure Linux Container Host cluster using the resource group and cluster name created earlier: + +```azurecli +az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $CLUSTER_NAME +``` + +To verify the connection to your cluster, run the [kubectl get nodes](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#get) command to return a list of the cluster nodes: + +```azurecli-interactive +kubectl get nodes +``` + + +```text +NAME STATUS ROLES AGE VERSION +aks-nodepool1-00000000-0 Ready agent 10m v1.20.7 +aks-nodepool1-00000000-1 Ready agent 10m v1.20.7 +``` + +## Next steps + +In this tutorial, you created and deployed an Azure Linux Container Host cluster. You learned how to: + +> [!div class="checklist"] +> * Install the Kubernetes CLI, `kubectl`. +> * Create an Azure resource group. +> * Create and deploy an Azure Linux Container Host cluster. +> * Configure `kubectl` to connect to your Azure Linux Container Host cluster. + +In the next tutorial, you'll learn how to add an Azure Linux node pool to an existing cluster. + +> [!div class="nextstepaction"] +> [Add an Azure Linux node pool](./tutorial-azure-linux-add-nodepool.md) \ No newline at end of file diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-migration.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-migration.md new file mode 100644 index 000000000..adc85d4a0 --- /dev/null +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-migration.md @@ -0,0 +1,144 @@ +--- +title: Azure Linux Container Host for AKS tutorial - Migrating to Azure Linux +description: In this Azure Linux Container Host for AKS tutorial, you learn how to migrate your nodes to Azure Linux nodes. +author: suhuruli +ms.author: suhuruli +ms.reviewer: schaffererin +ms.service: microsoft-linux +ms.custom: devx-track-azurecli, linux-related-content, innovation-engine +ms.topic: tutorial +ms.date: 01/19/2024 +--- + +# Tutorial: Migrate nodes to Azure Linux + +In this tutorial, part three of five, you migrate your existing nodes to Azure Linux. You can migrate your existing nodes to Azure Linux using one of the following methods: + +* Remove existing node pools and add new Azure Linux node pools. +* In-place OS SKU migration. + +If you don't have any existing nodes to migrate to Azure Linux, skip to the [next tutorial](./tutorial-azure-linux-telemetry-monitor.md). In later tutorials, you learn how to enable telemetry and monitoring in your clusters and upgrade Azure Linux nodes. + +## Prerequisites + +* In previous tutorials, you created and deployed an Azure Linux Container Host for AKS cluster. To complete this tutorial, you need to add an Azure Linux node pool to your existing cluster. If you haven't done this step and would like to follow along, start with [Tutorial 2: Add an Azure Linux node pool to your existing AKS cluster](./tutorial-azure-linux-add-nodepool.md). + + > [!NOTE] + > When adding a new Azure Linux node pool, you need to add at least one as `--mode System`. Otherwise, AKS won't allow you to delete your existing node pool. + +* You need the latest version of Azure CLI. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). + +## Add Azure Linux node pools and remove existing node pools + +1. Add a new Azure Linux node pool using the `az aks nodepool add` command. This command adds a new node pool to your cluster with the `--mode System` flag, which makes it a system node pool. System node pools are required for Azure Linux clusters. + +```azurecli-interactive +# Declare environment variables with a random suffix for uniqueness +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export NODE_POOL_NAME="np$RANDOM_SUFFIX" +az aks nodepool add --resource-group $RESOURCE_GROUP --cluster-name $CLUSTER_NAME --name $NODE_POOL_NAME --mode System --os-sku AzureLinux +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.ContainerService/managedClusters/myAKSCluster/nodePools/systempool", + "name": "systempool", + "provisioningState": "Succeeded" +} +``` + +2. Remove your existing nodes using the `az aks nodepool delete` command. + +## In-place OS SKU migration + +You can now migrate your existing Ubuntu node pools to Azure Linux by changing the OS SKU of the node pool, which rolls the cluster through the standard node image upgrade process. This new feature doesn't require the creation of new node pools. + +### Limitations + +There are several settings that can block the OS SKU migration request. To ensure a successful migration, review the following guidelines and limitations: + +* The OS SKU migration feature isn't available through PowerShell or the Azure portal. +* The OS SKU migration feature isn't able to rename existing node pools. +* Ubuntu and Azure Linux are the only supported Linux OS SKU migration targets. +* An Ubuntu OS SKU with `UseGPUDedicatedVHD` enabled can't perform an OS SKU migration. +* An Ubuntu OS SKU with CVM 20.04 enabled can't perform an OS SKU migration. +* Node pools with Kata enabled can't perform an OS SKU migration. +* Windows OS SKU migration isn't supported. +* OS SKU migration from Mariner to Azure Linux is supported, but rolling back to Mariner is not supported. + +### Prerequisites + +* An existing AKS cluster with at least one Ubuntu node pool. +* We recommend that you ensure your workloads configure and run successfully on the Azure Linux container host before attempting to use the OS SKU migration feature by [deploying an Azure Linux cluster](./quickstart-azure-cli.md) in dev/prod and verifying your service remains healthy. +* Ensure the migration feature is working for you in test/dev before using the process on a production cluster. +* Ensure that your pods have enough [Pod Disruption Budget](/azure/aks/operator-best-practices-scheduler#plan-for-availability-using-pod-disruption-budgets) to allow AKS to move pods between VMs during the upgrade. +* You need Azure CLI version [2.61.0](/cli/azure/release-notes-azure-cli#may-21-2024) or higher. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). +* If you are using Terraform, you must have [v3.111.0](https://github.com/hashicorp/terraform-provider-azurerm/releases/tag/v3.111.0) or greater of the AzureRM Terraform module. + +### [Azure CLI](#tab/azure-cli) + +#### Migrate the OS SKU of your Ubuntu node pool + +* Migrate the OS SKU of your node pool to Azure Linux using the `az aks nodepool update` command. This command updates the OS SKU for your node pool from Ubuntu to Azure Linux. The OS SKU change triggers an immediate upgrade operation, which takes several minutes to complete. + +```azurecli-interactive +az aks nodepool update --resource-group $RESOURCE_GROUP --cluster-name $CLUSTER_NAME --name $NODE_POOL_NAME --os-sku AzureLinux +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.ContainerService/managedClusters/myAKSCluster/nodePools/nodepool1", + "name": "nodepool1", + "osSku": "AzureLinux", + "provisioningState": "Succeeded" +} +``` + +> [!NOTE] +> If you experience issues during the OS SKU migration, you can [roll back to your previous OS SKU](#rollback). + +### Verify the OS SKU migration + +Once the migration is complete on your test clusters, you should verify the following to ensure a successful migration: + +* If your migration target is Azure Linux, run the `kubectl get nodes -o wide` command. The output should show `CBL-Mariner/Linux` as your OS image and `.cm2` at the end of your kernel version. +* Run the `kubectl get pods -o wide -A` command to verify that all of your pods and daemonsets are running on the new node pool. +* Run the `kubectl get nodes --show-labels` command to verify that all of the node labels in your upgraded node pool are what you expect. + +> [!TIP] +> We recommend monitoring the health of your service for a couple weeks before migrating your production clusters. + +### Run the OS SKU migration on your production clusters + +1. Update your existing templates to set `OSSKU=AzureLinux`. In ARM templates, you use `"OSSKU": "AzureLinux"` in the `agentPoolProfile` section. In Bicep, you use `osSku: "AzureLinux"` in the `agentPoolProfile` section. Lastly, for Terraform, you use `os_sku = "AzureLinux"` in the `default_node_pool` section. Make sure that your `apiVersion` is set to `2023-07-01` or later. +2. Redeploy your ARM, Bicep, or Terraform template for the cluster to apply the new `OSSKU` setting. During this deploy, your cluster behaves as if it's taking a node image upgrade. Your cluster surges capacity, and then reboots your existing nodes one by one into the latest AKS image from your new OS SKU. + +### Rollback + +If you experience issues during the OS SKU migration, you can roll back to your previous OS SKU. To do this, you need to change the OS SKU field in your template and resubmit the deployment, which triggers another upgrade operation and restores the node pool to its previous OS SKU. + + > [!NOTE] + > + > OS SKU migration does not support rolling back to OS SKU Mariner. + +* Roll back to your previous OS SKU using the `az aks nodepool update` command. This command updates the OS SKU for your node pool from Azure Linux back to Ubuntu. + +## Next steps + +In this tutorial, you migrated existing nodes to Azure Linux using one of the following methods: + +* Remove existing node pools and add new Azure Linux node pools. +* In-place OS SKU migration. + +In the next tutorial, you learn how to enable telemetry to monitor your clusters. + +> [!div class="nextstepaction"] +> [Enable telemetry and monitoring](./tutorial-azure-linux-telemetry-monitor.md) \ No newline at end of file diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md new file mode 100644 index 000000000..272c48050 --- /dev/null +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md @@ -0,0 +1,129 @@ +--- +title: Azure Linux Container Host for AKS tutorial - Enable telemetry and monitoring for the Azure Linux Container Host +description: In this Azure Linux Container Host for AKS tutorial, you'll learn how to enable telemetry and monitoring for the Azure Linux Container Host. +author: suhuruli +ms.author: suhuruli +ms.service: microsoft-linux +ms.custom: linux-related-content, innovation-engine +ms.topic: tutorial +ms.date: 03/26/2025 +--- + +# Tutorial: Enable telemetry and monitoring for your Azure Linux Container Host cluster + +In this tutorial, part four of five, you'll set up Container Insights to monitor an Azure Linux Container Host cluster. You'll learn how to: + +> [!div class="checklist"] +> * Enable monitoring for an existing cluster. +> * Verify that the agent is deployed successfully. +> * Verify that the solution is enabled. + +In the next and last tutorial, you'll learn how to upgrade your Azure Linux nodes. + +## Prerequisites + +- In previous tutorials, you created and deployed an Azure Linux Container Host cluster. To complete this tutorial, you need an existing cluster. If you haven't done this step and would like to follow along, start with [Tutorial 1: Create a cluster with the Azure Linux Container Host for AKS](./tutorial-azure-linux-create-cluster.md). +- If you're connecting an existing AKS cluster to a Log Analytics workspace in another subscription, the Microsoft.ContainerService resource provider must be registered in the subscription with the Log Analytics workspace. For more information, see [Register resource provider](/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider). +- You need the latest version of Azure CLI. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). + +## Environment Variables + +To ensure unique resource names for each deployment, we declare the following environment variables. These variables will be used throughout the tutorial. + +```bash +export RESOURCE_GROUP="myAKSResourceGroupabcf37" +export CLUSTER_NAME="myAKSClusterabcf37" +``` + +## Enable monitoring + +## Connect to your cluster + +Before enabling monitoring, it's important to ensure you're connected to the correct cluster. The following command retrieves the credentials for your Azure Linux Container Host cluster and configures kubectl to use them: + +```azurecli +az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME +``` + +### Use a default Log Analytics workspace + +The following step enables monitoring for your Azure Linux Container Host cluster using Azure CLI. In this example, you aren't required to precreate or specify an existing workspace. This command simplifies the process for you by creating a default workspace in the default resource group of the AKS cluster subscription. If one doesn't already exist in the region, the default workspace created will resemble the format *DefaultWorkspace-< GUID >-< Region >*. + +```azurecli +# Check if monitoring addon is already enabled +MONITORING_ENABLED=$(az aks show -g $RESOURCE_GROUP -n $CLUSTER_NAME --query "addonProfiles.omsagent.enabled" -o tsv) + +if [ "$MONITORING_ENABLED" != "true" ]; then + az aks enable-addons -a monitoring -n $CLUSTER_NAME -g $RESOURCE_GROUP +fi +``` + +### Option 2: Specify a Log Analytics workspace + +In this example, you can specify a Log Analytics workspace to enable monitoring of your Azure Linux Container Host cluster. The resource ID of the workspace will be in the form `"/subscriptions//resourceGroups//providers/Microsoft.OperationalInsights/workspaces/"`. The command to enable monitoring with a specified workspace is as follows: ```az aks enable-addons -a monitoring -n $CLUSTER_NAME -g $RESOURCE_GROUP --workspace-resource-id ``` + +## Verify agent and solution deployment + +Run the following command to verify that the agent is deployed successfully. + +```bash +kubectl get ds ama-logs --namespace=kube-system +``` + +The output should resemble the following example, which indicates that it was deployed properly: + + +```text +User@aksuser:~$ kubectl get ds ama-logs --namespace=kube-system +NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE +ama-logs 3 3 3 3 3 3m22s +``` + +To verify deployment of the solution, run the following command: + +```bash +kubectl get deployment ama-logs-rs -n=kube-system +``` + +The output should resemble the following example, which indicates that it was deployed properly: + + +```text +User@aksuser:~$ kubectl get deployment ama-logs-rs -n=kube-system +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +ama-logs-rs 1 1 1 1 3h +``` + +## Verify solution configuration + +Use the `aks show` command to find out whether the solution is enabled or not, what the Log Analytics workspace resource ID is, and summary information about the cluster. + +```azurecli +az aks show -g $RESOURCE_GROUP -n $CLUSTER_NAME --query "addonProfiles.omsagent" +``` + +After a few minutes, the command completes and returns JSON-formatted information about the solution. The results of the command should show the monitoring add-on profile and resemble the following example output: + + +```JSON +{ + "config": { + "logAnalyticsWorkspaceResourceID": "/subscriptions/xxxxx/resourceGroups/xxxxx/providers/Microsoft.OperationalInsights/workspaces/xxxxx" + }, + "enabled": true +} +``` + +## Next steps + +In this tutorial, you enabled telemetry and monitoring for your Azure Linux Container Host cluster. You learned how to: + +> [!div class="checklist"] +> * Enable monitoring for an existing cluster. +> * Verify that the agent is deployed successfully. +> * Verify that the solution is enabled. + +In the next tutorial, you'll learn how to upgrade your Azure Linux nodes. + +> [!div class="nextstepaction"] +> [Upgrade Azure Linux nodes](./tutorial-azure-linux-upgrade.md) \ No newline at end of file diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-upgrade.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-upgrade.md new file mode 100644 index 000000000..a0373ff2c --- /dev/null +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-upgrade.md @@ -0,0 +1,108 @@ +--- +title: Azure Linux Container Host for AKS tutorial - Upgrade Azure Linux Container Host nodes +description: In this Azure Linux Container Host for AKS tutorial, you learn how to upgrade Azure Linux Container Host nodes. +author: suhuruli +ms.author: suhuruli +ms.service: microsoft-linux +ms.custom: linux-related-content, innovation-engine +ms.topic: tutorial +ms.date: 08/18/2024 +--- + +# Tutorial: Upgrade Azure Linux Container Host nodes + +The Azure Linux Container Host ships updates through two mechanisms: updated Azure Linux node images and automatic package updates. + +As part of the application and cluster lifecycle, we recommend keeping your clusters up to date and secured by enabling upgrades for your cluster. You can enable automatic node-image upgrades to ensure your clusters use the latest Azure Linux Container Host image when it scales up. You can also manually upgrade the node-image on a cluster. + +In this tutorial, part five of five, you learn how to: + +> [!div class="checklist"] +> +> * Manually upgrade the node-image on a cluster. +> * Automatically upgrade an Azure Linux Container Host cluster. +> * Deploy Kured in an Azure Linux Container Host cluster. + +> [!NOTE] +> Any upgrade operation, whether performed manually or automatically, upgrades the node image version if not already on the latest. The latest version is contingent on a full AKS release, and can be determined by visiting the [AKS release tracker](/azure/aks/release-tracker). + +## Prerequisites + +* In previous tutorials, you created and deployed an Azure Linux Container Host cluster. To complete this tutorial, you need an existing cluster. If you haven't done this step and would like to follow along, start with [Tutorial 1: Create a cluster with the Azure Linux Container Host for AKS](./tutorial-azure-linux-create-cluster.md). +* You need the latest version of Azure CLI. Find the version using the `az --version` command. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). + +## Manually upgrade your cluster + +In order to manually upgrade the node-image on a cluster, you can run the `az aks nodepool upgrade. + +## Automatically upgrade your cluster + +Auto-upgrade provides a set once and forget mechanism that yields tangible time and operational cost benefits. By enabling auto-upgrade, you can ensure your clusters are up to date and don't miss the latest Azure Linux Container Host features or patches from AKS and upstream Kubernetes. + +Automatically completed upgrades are functionally the same as manual upgrades. The selected channel determines the timing of upgrades. When making changes to auto-upgrade, allow 24 hours for the changes to take effect. + +To set the auto-upgrade channel on an existing cluster, update the --auto-upgrade-channel parameter: + +```bash +az aks update --resource-group $AZ_LINUX_RG --name $AZ_LINUX_CLUSTER --auto-upgrade-channel stable +``` + + +```json +{ + "id": "/subscriptions/xxxxx/resourceGroups/testAzureLinuxResourceGroup", + "location": "WestUS2", + "name": "testAzureLinuxCluster", + "properties": { + "autoUpgradeChannel": "stable", + "provisioningState": "Succeeded" + } +} +``` + +For more information on upgrade channels, see [Using cluster auto-upgrade](/azure/aks/auto-upgrade-cluster). + +## Enable automatic package upgrades + +Similar to setting your clusters to auto-upgrade, you can use the same set once and forget mechanism for package upgrades by enabling the node-os upgrade channel. If automatic package upgrades are enabled, the dnf-automatic systemd service runs daily and installs any updated packages that have been published. + +To set the node-os upgrade channel on an existing cluster, update the --node-os-upgrade-channel parameter: + +```bash +az aks update --resource-group $AZ_LINUX_RG --name $AZ_LINUX_CLUSTER --node-os-upgrade-channel Unmanaged +``` + + +```json +{ + "id": "/subscriptions/xxxxx/resourceGroups/testAzureLinuxResourceGroup", + "location": "WestUS2", + "name": "testAzureLinuxCluster", + "properties": { + "nodeOsUpgradeChannel": "Unmanaged", + "provisioningState": "Succeeded" + } +} +``` + +## Enable an automatic reboot daemon + +To protect your clusters, security updates are automatically applied to Azure Linux nodes. These updates include OS security fixes, kernel updates, and package upgrades. Some of these updates require a node reboot to complete the process. AKS doesn't automatically reboot these nodes to complete the update process. + +We recommend enabling an automatic reboot daemon, such as [Kured](https://kured.dev/docs/), so that your cluster can reboot nodes that have taken kernel updates. To deploy the Kured DaemonSet in an Azure Linux Container Host cluster, see [Deploy Kured in an AKS cluster](/azure/aks/node-updates-kured#deploy-kured-in-an-aks-cluster). + +## Clean up resources + +As this tutorial is the last part of the series, you may want to delete your Azure Linux Container Host cluster. The Kubernetes nodes run on Azure virtual machines and continue incurring charges even if you don't use the cluster. + +## Next steps + +In this tutorial, you upgraded your Azure Linux Container Host cluster. You learned how to: + +> [!div class="checklist"] +> +> * Manually upgrade the node-image on a cluster. +> * Automatically upgrade an Azure Linux Container Host cluster. +> * Deploy kured in an Azure Linux Container Host cluster. + +For more information on the Azure Linux Container Host, see the [Azure Linux Container Host overview](./intro-azure-linux.md). \ No newline at end of file diff --git a/scenarios/azure-stack-docs/azure-stack/user/azure-stack-quick-create-vm-linux-cli.md b/scenarios/azure-stack-docs/azure-stack/user/azure-stack-quick-create-vm-linux-cli.md new file mode 100644 index 000000000..e60b44bd3 --- /dev/null +++ b/scenarios/azure-stack-docs/azure-stack/user/azure-stack-quick-create-vm-linux-cli.md @@ -0,0 +1,188 @@ +--- +title: Create Linux VM with Azure CLI in Azure Stack Hub +description: Create a Linux virtual machine by using the Azure CLI in Azure Stack Hub. +author: sethmanheim +ms.topic: quickstart +ms.date: 03/06/2025 +ms.author: sethm +ms.custom: mode-api, devx-track-azurecli, linux-related-content +--- + +# Quickstart: Create a Linux server VM by using the Azure CLI in Azure Stack Hub + +You can create an Ubuntu Server 20.04 LTS virtual machine (VM) by using the Azure CLI. In this article, you create and use a virtual machine. This article also shows you how to: + +* Connect to the virtual machine with a remote client. +* Install an NGINX web server and view the default home page. +* Clean up unused resources. + +## Prerequisites + +Before you begin, make sure you have the following prerequisites: + +* A Linux image in the Azure Stack Hub Marketplace + + The Azure Stack Hub Marketplace doesn't contain a Linux image by default. Have the Azure Stack Hub operator provide the Ubuntu Server 20.04 LTS image you need. The operator can use the instructions in [Download Marketplace items from Azure to Azure Stack Hub](../operator/azure-stack-download-azure-marketplace-item.md). + +* Azure Stack Hub requires a specific version of the Azure CLI to create and manage its resources. If you don't have the Azure CLI configured for Azure Stack Hub, sign in to a Windows-based external client if you're connected through VPN, and follow the instructions for [installing and configuring the Azure CLI](azure-stack-version-profiles-azurecli2.md). + +* A public Secure Shell (SSH) key with the name id_rsa.pub saved in the **.ssh** directory of your Windows user profile. For more information about creating SSH keys, see [Use an SSH key pair with Azure Stack Hub](azure-stack-dev-start-howto-ssh-public-key.md). + +## Create a resource group + +A resource group is a logical container where you can deploy and manage Azure Stack Hub resources. From your Azure Stack Hub integrated system, run the [az group create](/cli/azure/group#az-group-create) command to create a resource group. + +> [!NOTE] +> We assigned values for all variables in the following code examples. However, you can assign your own values. + +The following example creates a resource group named myResourceGroup with a random suffix in the local location: + +```azurecli +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP="myResourceGroup$RANDOM_SUFFIX" +export LOCATION="eastus2" +az group create --name $RESOURCE_GROUP --location $LOCATION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx", + "location": "local", + "managedBy": null, + "name": "myResourceGroupxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create a virtual machine + +Create a virtual machine by using the [az vm create](/cli/azure/vm#az-vm-create) command. The following example creates a VM named myVM. The example uses Demouser as the admin username. Change these values to something that's appropriate for your environment. + +```azurecli +export VM_NAME="myVM$RANDOM_SUFFIX" +az vm create \ + --resource-group $RESOURCE_GROUP \ + --name $VM_NAME \ + --image "Ubuntu2204" \ + --admin-username "azureuser" \ + --assign-identity \ + --generate-ssh-keys \ + --public-ip-sku Standard \ + --location $LOCATION +``` + +Results: + + +```JSON +{ + "fqdns": "", + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxx/providers/Microsoft.Compute/virtualMachines/myVMxxx", + "location": "local", + "name": "myVMxxx", + "osProfile": { + "computerName": "myVMxxx", + "adminUsername": "Demouser" + }, + "publicIpAddress": "x.x.x.x", + "powerState": "VM running", + "provisioningState": "Succeeded" +} +``` + +The public IP address is returned in the PublicIpAddress parameter. Note the address for later use with the virtual machine. + +## Open port 80 for web traffic + +Because this virtual machine runs the IIS web server, you must open port 80 to internet traffic. To open the port, use the [az vm open-port](/cli/azure/vm) command: + +```azurecli +az vm open-port --port 80 --resource-group $RESOURCE_GROUP --name $VM_NAME +``` + +Results: + + +```JSON +{ + "endPort": 80, + "name": "openPort80", + "port": 80, + "protocol": "Tcp", + "provisioningState": "Succeeded", + "resourceGroup": "myResourceGroupxxx", + "startPort": 80 +} +``` + +## Use SSH to connect to the virtual machine + +From a client computer with SSH installed, connect to the virtual machine. If you work on a Windows client, use [PuTTY](https://www.putty.org/) to create the connection. To connect to the virtual machine, you can use the `ssh` command. + +## Install the NGINX web server + +To update package resources and install the latest NGINX package, run the following script: + +```bash +output=$(az vm run-command invoke --resource-group $RESOURCE_GROUP --name $VM_NAME --command-id RunShellScript --scripts 'apt-get -y install nginx') +value=$(echo "$output" | jq -r '.value[0].message') +extracted=$(echo "$value" | awk '/\[stdout\]/,/\[stderr\]/' | sed '/\[stdout\]/d' | sed '/\[stderr\]/d') +echo "$extracted" +``` + +## View the NGINX welcome page + +With the NGINX web server installed, and port 80 open on your virtual machine, you can access the web server by using the virtual machine's public IP address. To do so, open a browser, and go to http://. Alternatively, you can use the curl command to view the NGINX welcome page: + +```bash +export PUBLIC_IP=$(az vm show -d -g $RESOURCE_GROUP -n $VM_NAME --query publicIps -o tsv) + +output=$(az vm run-command invoke --resource-group $RESOURCE_GROUP --name $VM_NAME --command-id RunShellScript --scripts 'curl -v http://localhost') +value=$(echo "$output" | jq -r '.value[0].message') +extracted=$(echo "$value" | awk '/\[stdout\]/,/\[stderr\]/' | sed '/\[stdout\]/d' | sed '/\[stderr\]/d') +echo "$extracted" +``` + +Results: + + +```HTML + + + +Welcome to nginx! + + + +

Welcome to nginx!

+

If you see this page, the nginx web server is successfully installed and +working. Further configuration is required.

+ +

For online documentation and support please refer to +nginx.org.
+Commercial support is available at +nginx.com.

+ +

Thank you for using nginx.

+ + +``` + +![The NGINX web server Welcome page](./media/azure-stack-quick-create-vm-linux-cli/nginx.png) + +## Next steps + +In this quickstart, you deployed a basic Linux server virtual machine with a web server. To learn more about Azure Stack Hub virtual machines, see [Considerations for virtual machines in Azure Stack Hub](azure-stack-vm-considerations.md). \ No newline at end of file diff --git a/scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md b/scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md new file mode 100644 index 000000000..9e2b53e3b --- /dev/null +++ b/scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md @@ -0,0 +1,1245 @@ +--- +title: "Docker: Install Containers for SQL Server on Linux" +description: This quickstart shows how to use Docker to run the SQL Server Linux container images. You connect to a database and run a query. +author: amitkh-msft +ms.author: amitkh +ms.reviewer: vanto, randolphwest +ms.date: 11/18/2024 +ms.service: sql +ms.subservice: linux +ms.topic: quickstart +ms.custom: + - intro-quickstart + - kr2b-contr-experiment + - linux-related-content +zone_pivot_groups: cs1-command-shell +monikerRange: ">=sql-server-linux-2017 || >=sql-server-2017" +--- +# Quickstart: Run SQL Server Linux container images with Docker + +[!INCLUDE [SQL Server - Linux](../includes/applies-to-version/sql-linux.md)] + + +::: moniker range="=sql-server-linux-2017 || =sql-server-2017" + +In this quickstart, you use Docker to pull and run the [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] Linux container image, [mssql-server-linux](https://mcr.microsoft.com/product/mssql/server/about). Then you can connect with **sqlcmd** to create your first database and run queries. + +For more information on supported platforms, see [Release notes for SQL Server 2017 on Linux](sql-server-linux-release-notes-2017.md). + +> [!WARNING] +> When you stop and remove a container, your [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] data in the container is permanently deleted. For more information on preserving your data, [create and copy a backup file out of the container](tutorial-restore-backup-in-sql-server-container.md) or use a [container data persistence technique](sql-server-linux-docker-container-configure.md#persist). + +This quickstart creates [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the versions of this article for [[!INCLUDE [sssql19-md](../includes/sssql19-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver15&preserve-view=true#pullandrun2019) or [[!INCLUDE [sssql22-md](../includes/sssql22-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver16&preserve-view=true#pullandrun2022) versions of this article. + +::: moniker-end + + +::: moniker range="=sql-server-linux-ver15 || =sql-server-ver15" + +In this quickstart, you use Docker to pull and run the [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] Linux container image, [mssql-server-linux](https://mcr.microsoft.com/product/mssql/server/about). Then you can connect with **sqlcmd** to create your first database and run queries. + +For more information on supported platforms, see [Release notes for SQL Server 2019 on Linux](sql-server-linux-release-notes-2019.md). + +> [!WARNING] +> When you stop and remove a container, your [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] data in the container is permanently deleted. For more information on preserving your data, [create and copy a backup file out of the container](tutorial-restore-backup-in-sql-server-container.md) or use a [container data persistence technique](sql-server-linux-docker-container-configure.md#persist). + +This quickstart creates [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql17-md](../includes/sssql17-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-2017&preserve-view=true#pullandrun2017) or [[!INCLUDE [sssql22-md](../includes/sssql22-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver16&preserve-view=true#pullandrun2022) versions of this article. + +::: moniker-end + + +::: moniker range=">= sql-server-linux-ver16 || >= sql-server-ver16" + +In this quickstart, you use Docker to pull and run the [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] Linux container image, [mssql-server-linux](https://mcr.microsoft.com/product/mssql/server/about). Then you can connect with **sqlcmd** to create your first database and run queries. + +For more information on supported platforms, see [Release notes for SQL Server 2022 on Linux](sql-server-linux-release-notes-2022.md). + +> [!WARNING] +> When you stop and remove a container, your [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] data in the container is permanently deleted. For more information on preserving your data, [create and copy a backup file out of the container](tutorial-restore-backup-in-sql-server-container.md) or use a [container data persistence technique](sql-server-linux-docker-container-configure.md#persist). + +This quickstart creates [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql17-md](../includes/sssql17-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-2017&preserve-view=true#pullandrun2017) or [[!INCLUDE [sssql19-md](../includes/sssql19-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver15&preserve-view=true#pullandrun2019) versions of this article. + +::: moniker-end + +This image consists of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] running on Linux based on Ubuntu. It can be used with the Docker Engine 1.8+ on Linux. + +Starting with [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] CU 14 and [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] CU 28, the container images include the [new mssql-tools18](sql-server-linux-setup-tools.md#install-tools-on-linux) package. The previous directory `/opt/mssql-tools/bin` is being phased out. The new directory for Microsoft ODBC 18 tools is `/opt/mssql-tools18/bin`, aligning with the latest tools offering. For more information about changes and security enhancements, see [ODBC Driver 18.0 for SQL Server Released](https://techcommunity.microsoft.com/blog/sqlserver/odbc-driver-18-0-for-sql-server-released/3169228). + +The examples in this article use the `docker` command. However, most of these commands also work with Podman. Podman provides a command-line interface similar to the Docker Engine. You can [find out more about Podman](https://docs.podman.io/en/latest). + +> [!IMPORTANT] +> **sqlcmd** doesn't currently support the `MSSQL_PID` parameter when creating containers. If you use the **sqlcmd** instructions in this quickstart, you create a container with the Developer edition of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)]. Use the command line interface (CLI) instructions to create a container using the license of your choice. For more information, see [Deploy and connect to SQL Server Linux containers](sql-server-linux-docker-container-deployment.md). + + + +## Prerequisites + +- Docker Engine 1.8+ on any supported Linux distribution. For more information, see [Install Docker](https://docs.docker.com/engine/installation/). + + +::: moniker range="=sql-server-linux-2017 || =sql-server-2017" + +- For more information on hardware requirements and processor support, see [SQL Server 2016 and 2017: Hardware and software requirements](../sql-server/install/hardware-and-software-requirements-for-installing-sql-server.md) + +::: moniker-end + + +::: moniker range="=sql-server-linux-ver15 || =sql-server-ver15" + +- For more information on hardware requirements and processor support, see [SQL Server 2019: Hardware and software requirements](../sql-server/install/hardware-and-software-requirements-for-installing-sql-server-2019.md) + +::: moniker-end + + +::: moniker range=">= sql-server-linux-ver16 || >= sql-server-ver16" + +- For more information on hardware requirements and processor support, see [SQL Server 2022: Hardware and software requirements](../sql-server/install/hardware-and-software-requirements-for-installing-sql-server-2022.md) + +::: moniker-end + +- Docker `overlay2` storage driver. This driver is the default for most users. If you aren't using this storage provider and need to change, see the instructions and warnings in the [Docker documentation for configuring overlay2](https://docs.docker.com/engine/storage/drivers/overlayfs-driver/#configure-docker-with-the-overlay-or-overlay2-storage-driver). + +- Install the latest **[sqlcmd](../tools/sqlcmd/sqlcmd-utility.md?&tabs=go)** on your Docker host. + +- At least 2 GB of disk space. + +- At least 2 GB of RAM. + +- [System requirements for SQL Server on Linux](sql-server-linux-setup.md#system). + + +::: moniker range="=sql-server-linux-2017 || =sql-server-2017" + + + +## Pull and run the SQL Server Linux container image + +Before starting the following steps, make sure that you select your preferred shell (**bash**, **PowerShell**, or **cmd**) at the top of this article. + +::: zone pivot="cs1-bash" +For the bash commands in this article, `sudo` is used. If you don't want to use `sudo` to run Docker, you can configure a `docker` group and add users to that group. For more information, see [Post-installation steps for Linux](https://docs.docker.com/engine/install/linux-postinstall). +::: zone-end + +## [CLI](#tab/cli) + +### Pull the container image from the registry + +Pull the [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] Linux container image from the Microsoft Container Registry. + +::: zone pivot="cs1-bash" + +```bash +sudo docker pull mcr.microsoft.com/mssql/server:2017-latest +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +docker pull mcr.microsoft.com/mssql/server:2017-latest +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +docker pull mcr.microsoft.com/mssql/server:2017-latest +``` + +::: zone-end + +This quickstart creates [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql19-md](../includes/sssql19-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver15&preserve-view=true#pullandrun2019) or [[!INCLUDE [sssql22-md](../includes/sssql22-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver16&preserve-view=true#pullandrun2022) versions of this article. + +The previous command pulls the latest [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] Linux container image. If you want to pull a specific image, you add a colon and the tag name, such as `mcr.microsoft.com/mssql/server:2017-GA-ubuntu`. To see all available images, see the [Microsoft Artifact Registry](https://mcr.microsoft.com/product/mssql/server/about). + +### Run the container + +To run the Linux container image with Docker, you can use the following command from a bash shell or elevated PowerShell command prompt. + +> [!IMPORTANT] +> The `SA_PASSWORD` environment variable is deprecated. Use `MSSQL_SA_PASSWORD` instead. + +::: zone pivot="cs1-bash" + +```bash +sudo docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" \ + -p 1433:1433 --name sql1 --hostname sql1 \ + -d \ + mcr.microsoft.com/mssql/server:2017-latest +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +If you're using PowerShell Core, replace the double quotes with single quotes. + +```powershell +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" ` + -p 1433:1433 --name sql1 --hostname sql1 ` + -d ` + mcr.microsoft.com/mssql/server:2017-latest +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" ` + -p 1433:1433 --name sql1 --hostname sql1 ` + -d ` + mcr.microsoft.com/mssql/server:2017-latest +``` + +::: zone-end + +> [!CAUTION] +> [!INCLUDE [password-complexity](includes/password-complexity.md)] If you don't follow these password requirements, the container can't set up [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], and stops working. You can examine the error log by using the [`docker logs`](https://docs.docker.com/reference/cli/docker/container/logs) command. + +By default, this quickstart creates a container with the Developer edition of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)]. The process for running production editions in containers is slightly different. For more information, see [Run production container images](./sql-server-linux-docker-container-deployment.md#production). + +The following table provides a description of the parameters in the previous `docker run` example: + +| Parameter | Description | +| --- | --- | +| `-e "ACCEPT_EULA=Y"` | Set the `ACCEPT_EULA` variable to any value to confirm your acceptance of the End-User Licensing Agreement. Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `-e "MSSQL_SA_PASSWORD="` | Specify your own strong password that is at least eight characters and meets the [Password Policy](../relational-databases/security/password-policy.md). Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `-e "MSSQL_COLLATION="` | Specify a custom [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] collation, instead of the default `SQL_Latin1_General_CP1_CI_AS`. | +| `-p 1433:1433` | Map a TCP port on the host environment (first value) with a TCP port in the container (second value). In this example, [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is listening on TCP 1433 in the container and this container port is then exposed to TCP port 1433 on the host. | +| `--name sql1` | Specify a custom name for the container rather than a randomly generated one. If you run more than one container, you can't reuse this same name. | +| `--hostname sql1` | Used to explicitly set the container hostname. If you don't specify the hostname, it defaults to the container ID, which is a randomly generated system GUID. | +| `-d` | Run the container in the background (daemon). | +| `mcr.microsoft.com/mssql/server:2017-latest` | The [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] Linux container image. | + +## [sqlcmd](#tab/sqlcmd) + +### Pull and run the container + +Pull and run the [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] Linux container image from the Microsoft Container Registry. + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd create mssql --tag 2017-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd create mssql --tag 2017-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd create mssql --tag 2017-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +This quickstart creates [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql19-md](../includes/sssql19-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver15&preserve-view=true#pullandrun2019) or [[!INCLUDE [sssql22-md](../includes/sssql22-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver16&preserve-view=true#pullandrun2022) versions of this article. + +The previous command uses the latest [!INCLUDE [sssql17-md](../includes/sssql17-md.md)] Linux container image. If you want to pull a specific image, change the tag name, such as `2017-GA-ubuntu`. To see all available images, run the following command: + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd create mssql get-tags +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd create mssql get-tags +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd create mssql get-tags +``` + +::: zone-end + +The following table provides a description of the parameters in the previous `sqlcmd create mssql` example: + +| Parameter | Description | +| --- | --- | +| `--ACCEPT-EULA` | Include the `ACCEPT-EULA` flag to confirm your acceptance of the End-User Licensing Agreement. Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `--port 1433` | Map a TCP port on the host environment and a TCP port in the container. In this example, [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is listening on TCP 1433 in the container and this container port is then exposed to TCP port 1433 on the host. | +| `--name sql1` | Specify a custom name for the container rather than a randomly generated one. If you run more than one container, you can't reuse this same name. | +| `--hostname sql1` | Used to explicitly set the container hostname. If you don't specify the hostname, it defaults to the container ID, which is a randomly generated system GUID. | +| `--tag 2017-latest` | The [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] Linux container image. | + +--- + +### View list of containers + +1. To view your Docker containers, use the `docker ps` command. + + ::: zone pivot="cs1-bash" + + ```bash + sudo docker ps -a + ``` + + ::: zone-end + + ::: zone pivot="cs1-powershell" + + ```powershell + docker ps -a + ``` + + ::: zone-end + + ::: zone pivot="cs1-cmd" + + ```cmd + docker ps -a + ``` + + ::: zone-end + + You should see output similar to the following example: + + ```output + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + d4a1999ef83e mcr.microsoft.com/mssql/server:2017-latest "/opt/mssql/bin/perm..." 2 minutes ago Up 2 minutes 0.0.0.0:1433->1433/tcp, :::1433->1433/tcp sql1 + ``` + +1. If the `STATUS` column shows a status of `Up`, then [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is running in the container and listening on the port specified in the `PORTS` column. If the `STATUS` column for your [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] container shows `Exited`, see [Troubleshoot SQL Server Docker containers](sql-server-linux-docker-container-troubleshooting.md). The server is ready for connections once the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] error logs display the message: `SQL Server is now ready for client connections. This is an informational message; no user action is required`. You can review the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] error log inside the container using the command: + + ```bash + sudo docker exec -t sql1 cat /var/opt/mssql/log/errorlog | grep connection + ``` + + The `--hostname` parameter, as discussed previously, changes the internal name of the container to a custom value. This value is the name you see returned in the following Transact-SQL query: + + ```sql + SELECT @@SERVERNAME, + SERVERPROPERTY('ComputerNamePhysicalNetBIOS'), + SERVERPROPERTY('MachineName'), + SERVERPROPERTY('ServerName'); + ``` + + Setting `--hostname` and `--name` to the same value is a good way to easily identify the target container. + +1. As a final step, [change your SA password](#sapassword) in a production environment, because the `MSSQL_SA_PASSWORD` is visible in `ps -eax` output and stored in the environment variable of the same name. + +::: moniker-end + + +::: moniker range="=sql-server-linux-ver15 || =sql-server-ver15" + + + +## Pull and run the SQL Server Linux container image + +Before starting the following steps, make sure that you select your preferred shell (**bash**, **PowerShell**, or **cmd**) at the top of this article. + +::: zone pivot="cs1-bash" +For the bash commands in this article, `sudo` is used. If you don't want to use `sudo` to run Docker, you can configure a `docker` group and add users to that group. For more information, see [Post-installation steps for Linux](https://docs.docker.com/engine/install/linux-postinstall). +::: zone-end + +## [CLI](#tab/cli) + +### Pull the container from the registry + +Pull the [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] Linux container image from the Microsoft Container Registry. + +::: zone pivot="cs1-bash" + +```bash +docker pull mcr.microsoft.com/mssql/server:2019-latest +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +docker pull mcr.microsoft.com/mssql/server:2019-latest +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +docker pull mcr.microsoft.com/mssql/server:2019-latest +``` + +::: zone-end + +This quickstart creates [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql17-md](../includes/sssql17-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-2017&preserve-view=true#pullandrun2017) or [[!INCLUDE [sssql22-md](../includes/sssql22-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver16&preserve-view=true#pullandrun2022) versions of this article. + +The previous command pulls the latest [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] Linux container image. If you want to pull a specific image, you add a colon and the tag name, such as `mcr.microsoft.com/mssql/server:2019-GA-ubuntu`. To see all available images, see the [Microsoft Artifact Registry](https://mcr.microsoft.com/product/mssql/server/about). + +### Run the container + +To run the Linux container image with Docker, you can use the following command from a bash shell or elevated PowerShell command prompt. + +> [!IMPORTANT] +> The `SA_PASSWORD` environment variable is deprecated. Use `MSSQL_SA_PASSWORD` instead. + +::: zone pivot="cs1-bash" + +```bash +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" \ + -p 1433:1433 --name sql1 --hostname sql1 \ + -d \ + mcr.microsoft.com/mssql/server:2019-latest +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +If you're using PowerShell Core, replace the double quotes with single quotes. + +```powershell +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" ` + -p 1433:1433 --name sql1 --hostname sql1 ` + -d ` + mcr.microsoft.com/mssql/server:2019-latest +``` + +> [!CAUTION] +> [!INCLUDE [password-complexity](includes/password-complexity.md)] + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" ` + -p 1433:1433 --name sql1 --hostname sql1 ` + -d ` + mcr.microsoft.com/mssql/server:2019-latest +``` + +::: zone-end + +> [!CAUTION] +> [!INCLUDE [password-complexity](includes/password-complexity.md)] If you don't follow these password requirements, the container can't set up [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], and stops working. You can examine the error log by using the [`docker logs`](https://docs.docker.com/reference/cli/docker/container/logs) command. + +By default, this quickstart creates a container with the Developer edition of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)]. The process for running production editions in containers is slightly different. For more information, see [Run production container images](./sql-server-linux-docker-container-deployment.md#production). + +The following table provides a description of the parameters in the previous `docker run` example: + +| Parameter | Description | +| --- | --- | +| `-e "ACCEPT_EULA=Y"` | Set the `ACCEPT_EULA` variable to any value to confirm your acceptance of the End-User Licensing Agreement. Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `-e "MSSQL_SA_PASSWORD="` | Specify your own strong password that is at least eight characters and meets the [Password Policy](../relational-databases/security/password-policy.md). Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `-e "MSSQL_COLLATION="` | Specify a custom [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] collation, instead of the default `SQL_Latin1_General_CP1_CI_AS`. | +| `-p 1433:1433` | Map a TCP port on the host environment (first value) with a TCP port in the container (second value). In this example, [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is listening on TCP 1433 in the container and this container port is then exposed to TCP port 1433 on the host. | +| `--name sql1` | Specify a custom name for the container rather than a randomly generated one. If you run more than one container, you can't reuse this same name. | +| `--hostname sql1` | Used to explicitly set the container hostname. If you don't specify the hostname, it defaults to the container ID, which is a randomly generated system GUID. | +| `-d` | Run the container in the background (daemon). | +| `mcr.microsoft.com/mssql/server:2019-latest` | The [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] Linux container image. | + +## [sqlcmd](#tab/sqlcmd) + +### Pull and run the container + +Pull and run the [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] Linux container image from the Microsoft Container Registry. + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd create mssql --tag 2019-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd create mssql --tag 2019-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd create mssql --tag 2019-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +This quickstart creates [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql17-md](../includes/sssql17-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-2017&preserve-view=true#pullandrun2017) or [[!INCLUDE [sssql22-md](../includes/sssql22-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver16&preserve-view=true#pullandrun2022) versions of this article. + +The previous command pulls the latest [!INCLUDE [sssql19-md](../includes/sssql19-md.md)] Linux container image. If you want to pull a specific image, change the tag name, such as `2019-GA-ubuntu-16.04`. To see all available images, run the following command: + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd create mssql get-tags +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd create mssql get-tags +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd create mssql get-tags +``` + +::: zone-end + +By default, this quickstart creates a container with the Developer edition of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)]. The process for running production editions in containers is slightly different. For more information, see [Run production container images](./sql-server-linux-docker-container-deployment.md#production). + +The following table provides a description of the parameters in the previous `docker run` example: + +| Parameter | Description | +| --- | --- | +| `--ACCEPT_EULA` | Include the `ACCEPT_EULA` flag to confirm your acceptance of the End-User Licensing Agreement. Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `--port 1433` | Map a TCP port on the host environment and a TCP port in the container. In this example, [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is listening on TCP 1433 in the container and this container port is then exposed to TCP port 1433 on the host. | +| `--name sql1` | Specify a custom name for the container rather than a randomly generated one. If you run more than one container, you can't reuse this same name. | +| `--hostname sql1` | Used to explicitly set the container hostname. If you don't specify the hostname, it defaults to the container ID, which is a randomly generated system GUID. | +| `--tag 2019-latest` | The [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] Linux container image. | + +--- + +### View list of containers + +1. To view your Docker containers, use the `docker ps` command. + + ::: zone pivot="cs1-bash" + + ```bash + docker ps -a + ``` + + ::: zone-end + + ::: zone pivot="cs1-powershell" + + ```powershell + docker ps -a + ``` + + ::: zone-end + + ::: zone pivot="cs1-cmd" + + ```cmd + docker ps -a + ``` + + ::: zone-end + + You should see output similar to the following example: + + ```output + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + d4a1999ef83e mcr.microsoft.com/mssql/server:2019-latest "/opt/mssql/bin/perm..." 2 minutes ago Up 2 minutes 0.0.0.0:1433->1433/tcp, :::1433->1433/tcp sql1 + ``` + +1. If the `STATUS` column shows a status of `Up`, then [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is running in the container and listening on the port specified in the `PORTS` column. If the `STATUS` column for your [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] container shows `Exited`, see [Troubleshoot SQL Server Docker containers](sql-server-linux-docker-container-troubleshooting.md). The server is ready for connections once the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] error logs display the message: `SQL Server is now ready for client connections. This is an informational message; no user action is required`. You can review the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] error log inside the container using the command: + + ```bash + docker exec -t sql1 cat /var/opt/mssql/log/errorlog | grep connection + ``` + + The `--hostname` parameter, as discussed previously, changes the internal name of the container to a custom value. This value is the name you see returned in the following Transact-SQL query: + + ```sql + SELECT @@SERVERNAME, + SERVERPROPERTY('ComputerNamePhysicalNetBIOS'), + SERVERPROPERTY('MachineName'), + SERVERPROPERTY('ServerName'); + ``` + + Setting `--hostname` and `--name` to the same value is a good way to easily identify the target container. + +1. As a final step, [change your SA password](#sapassword) in a production environment, because the `MSSQL_SA_PASSWORD` is visible in `ps -eax` output and stored in the environment variable of the same name. + +::: moniker-end + + +::: moniker range=">= sql-server-linux-ver16 || >= sql-server-ver16" + + + +## Pull and run the SQL Server Linux container image + +Before starting the following steps, make sure that you select your preferred shell (**bash**, **PowerShell**, or **cmd**) at the top of this article. + +::: zone pivot="cs1-bash" +For the bash commands in this article, `sudo` is used. If you don't want to use `sudo` to run Docker, you can configure a `docker` group and add users to that group. For more information, see [Post-installation steps for Linux](https://docs.docker.com/engine/install/linux-postinstall). +::: zone-end + +## [CLI](#tab/cli) + +### Pull the container image from the registry + +Pull the [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] Linux container image from the Microsoft Container Registry. + +::: zone pivot="cs1-bash" + +```bash +docker pull mcr.microsoft.com/mssql/server:2022-latest +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +docker pull mcr.microsoft.com/mssql/server:2022-latest +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +docker pull mcr.microsoft.com/mssql/server:2022-latest +``` + +::: zone-end + +This quickstart creates [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql17-md](../includes/sssql17-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-2017&preserve-view=true#pullandrun2017) or [[!INCLUDE [sssql19-md](../includes/sssql19-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver15&preserve-view=true#pullandrun2019) versions of this article. + +The previous command pulls the latest [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] Linux container image. If you want to pull a specific image, you add a colon and the tag name, such as `mcr.microsoft.com/mssql/server:2022-GA-ubuntu`. To see all available images, see the [Microsoft Artifact Registry](https://mcr.microsoft.com/product/mssql/server/about). + +### Run the container + +To run the Linux container image with Docker, you can use the following command from a bash shell or elevated PowerShell command prompt. + +> [!IMPORTANT] +> The `SA_PASSWORD` environment variable is deprecated. Use `MSSQL_SA_PASSWORD` instead. + +::: zone pivot="cs1-bash" + +```bash +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" \ + -p 1433:1433 --name sql1 --hostname sql1 \ + -d \ + mcr.microsoft.com/mssql/server:2022-latest +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +If you're using PowerShell Core, replace the double quotes with single quotes. + +```powershell +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" ` + -p 1433:1433 --name sql1 --hostname sql1 ` + -d ` + mcr.microsoft.com/mssql/server:2022-latest +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=" ` + -p 1433:1433 --name sql1 --hostname sql1 ` + -d ` + mcr.microsoft.com/mssql/server:2022-latest +``` + +::: zone-end + +> [!CAUTION] +> [!INCLUDE [password-complexity](includes/password-complexity.md)] If you don't follow these password requirements, the container can't set up [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], and stops working. You can examine the error log by using the [`docker logs`](https://docs.docker.com/reference/cli/docker/container/logs) command. + +By default, this quickstart creates a container with the Developer edition of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)]. The process for running production editions in containers is slightly different. For more information, see [Run production container images](./sql-server-linux-docker-container-deployment.md#production). + +The following table provides a description of the parameters in the previous `docker run` example: + +| Parameter | Description | +| --- | --- | +| `-e "ACCEPT_EULA=Y"` | Set the `ACCEPT_EULA` variable to any value to confirm your acceptance of the End-User Licensing Agreement. Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `-e "MSSQL_SA_PASSWORD="` | Specify your own strong password that is at least eight characters and meets the [Password Policy](../relational-databases/security/password-policy.md). Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `-e "MSSQL_COLLATION="` | Specify a custom [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] collation, instead of the default `SQL_Latin1_General_CP1_CI_AS`. | +| `-p 1433:1433` | Map a TCP port on the host environment (first value) with a TCP port in the container (second value). In this example, [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is listening on TCP 1433 in the container and this container port is then exposed to TCP port 1433 on the host. | +| `--name sql1` | Specify a custom name for the container rather than a randomly generated one. If you run more than one container, you can't reuse this same name. | +| `--hostname sql1` | Used to explicitly set the container hostname. If you don't specify the hostname, it defaults to the container ID, which is a randomly generated system GUID. | +| `-d` | Run the container in the background (daemon). | +| `mcr.microsoft.com/mssql/server:2022-latest` | The [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] Linux container image. | + + + +## Change the system administrator password + +The system administrator (`sa`) account is a system administrator on the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] instance that gets created during setup. After you create your [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] container, the `MSSQL_SA_PASSWORD` environment variable you specified is discoverable by running `echo $MSSQL_SA_PASSWORD` in the container. For security purposes, you should change your `sa` password in a production environment. + +1. Choose a strong password to use for the `sa` account. [!INCLUDE [password-complexity](includes/password-complexity.md)] + +1. Use `docker exec` to run **sqlcmd** to change the password using Transact-SQL. In the following example, the old and new passwords are read from user input. + + ::: zone pivot="cs1-bash" + + ```bash + docker exec -it sql1 /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U sa \ + -P "$(read -sp "Enter current SA password: "; echo "${REPLY}")" \ + -Q "ALTER LOGIN sa WITH PASSWORD=\"$(read -sp "Enter new SA password: "; echo "${REPLY}")\"" + ``` + + ::: zone-end + + ::: zone pivot="cs1-powershell" + + ```powershell + docker exec -it sql1 /opt/mssql-tools18/bin/sqlcmd ` + -S localhost -U sa -P "" ` + -Q "ALTER LOGIN sa WITH PASSWORD=''" + ``` + + ::: zone-end + + ::: zone pivot="cs1-cmd" + + ```cmd + docker exec -it sql1 /opt/mssql-tools18/bin/sqlcmd ` + -S localhost -U sa -P "" ` + -Q "ALTER LOGIN sa WITH PASSWORD=''" + ``` + + ::: zone-end + + > [!CAUTION] + > [!INCLUDE [password-complexity](includes/password-complexity.md)] + + Recent versions of **sqlcmd** are secure by default. For more information about connection encryption, see [sqlcmd utility](../tools/sqlcmd/sqlcmd-utility.md) for Windows, and [Connecting with sqlcmd](../connect/odbc/linux-mac/connecting-with-sqlcmd.md) for Linux and macOS. If the connection doesn't succeed, you can add the `-No` option to **sqlcmd** to specify that encryption is optional, not mandatory. + +## Disable the SA account as a best practice + +> [!IMPORTANT] +> You'll need these credentials for later steps. Be sure to write down the user ID and password that you enter here. + +[!INCLUDE [connect-with-sa](includes/connect-with-sa.md)] + +## [sqlcmd](#tab/sqlcmd) + +### Pull and run the container + +Pull and run the [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] Linux container image from the Microsoft Container Registry. + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd create mssql --tag 2022-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd create mssql --tag 2022-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd create mssql --tag 2022-latest --hostname sql1 --name sql1 --port 1433 --accept-eula +``` + +::: zone-end + +This quickstart creates [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] containers. If you prefer to create Linux containers for different versions of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)], see the [[!INCLUDE [sssql17-md](../includes/sssql17-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-2017&preserve-view=true#pullandrun2017) or [[!INCLUDE [sssql19-md](../includes/sssql19-md.md)]](quickstart-install-connect-docker.md?view=sql-server-linux-ver15&preserve-view=true#pullandrun2019) versions of this article. + +The previous command pulls the latest [!INCLUDE [sssql22-md](../includes/sssql22-md.md)] Linux container image. If you want to pull a specific image, change the tag name, such as `2022-CU11-ubuntu-22.04`. To see all available images, run the following command: + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd create mssql get-tags +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd create mssql get-tags +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd create mssql get-tags +``` + +::: zone-end + +By default, this quickstart creates a container with the Developer edition of [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)]. The process for running production editions in containers is slightly different. For more information, see [Run production container images](./sql-server-linux-docker-container-deployment.md#production). + +The following table provides a description of the parameters in the previous `docker run` example: + +| Parameter | Description | +| --- | --- | +| `--ACCEPT-EULA` | Include the `--ACCEPT-EULA` flag to confirm your acceptance of the End-User Licensing Agreement. Required setting for the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] image. | +| `--port 1433` | Map a TCP port on the host environment and a TCP port in the container. In this example, [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is listening on TCP 1433 in the container and this container port is then exposed to TCP port 1433 on the host. | +| `--name sql1` | Specify a custom name for the container rather than a randomly generated one. If you run more than one container, you can't reuse this same name. | +| `--hostname sql1` | Used to explicitly set the container hostname. If you don't specify the hostname, it defaults to the container ID, which is a randomly generated system GUID. | +| `--tag 2022-latest` | The [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] Linux container image. | + +**sqlcmd** disables the `sa` password and creates a new login based on the current user when it creates a container. Use the following command to view your login information. You need it in later steps. + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd config view --raw +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd config view --raw +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd config view --raw +``` + +::: zone-end + +--- + +### View list of containers + +1. To view your Docker containers, use the `docker ps` command. + + ::: zone pivot="cs1-bash" + + ```bash + docker ps -a + ``` + + ::: zone-end + + ::: zone pivot="cs1-powershell" + + ```powershell + docker ps -a + ``` + + ::: zone-end + + ::: zone pivot="cs1-cmd" + + ```cmd + docker ps -a + ``` + + ::: zone-end + + You should see output similar to the following example: + + ```output + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + d4a1999ef83e mcr.microsoft.com/mssql/server:2022-latest "/opt/mssql/bin/perm..." 2 minutes ago Up 2 minutes 0.0.0.0:1433->1433/tcp, :::1433->1433/tcp sql1 + ``` + +1. If the `STATUS` column shows a status of `Up`, then [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] is running in the container and listening on the port specified in the `PORTS` column. If the `STATUS` column for your [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] container shows `Exited`, see [Troubleshoot SQL Server Docker containers](sql-server-linux-docker-container-troubleshooting.md). The server is ready for connections once the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] error logs display the message: `SQL Server is now ready for client connections. This is an informational message; no user action is required`. You can review the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] error log inside the container using the command: + + ```bash + docker exec -t sql1 cat /var/opt/mssql/log/errorlog | grep connection + ``` + + The `--hostname` parameter, as discussed previously, changes the internal name of the container to a custom value. This value is the name you see returned in the following Transact-SQL query: + + ```sql + SELECT @@SERVERNAME, + SERVERPROPERTY('ComputerNamePhysicalNetBIOS'), + SERVERPROPERTY('MachineName'), + SERVERPROPERTY('ServerName'); + ``` + + Setting `--hostname` and `--name` to the same value is a good way to easily identify the target container. + +::: moniker-end + +## Connect to SQL Server + +The following steps use the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] command-line tool, [sqlcmd utility](../tools/sqlcmd/sqlcmd-utility.md), inside the container to connect to [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)]. + +1. Use the `docker exec -it` command to start an interactive bash shell inside your running container. In the following example, `sql1` is name specified by the `--name` parameter when you created the container. + + ::: zone pivot="cs1-bash" + + ```bash + docker exec -it sql1 "bash" + ``` + + ::: zone-end + + ::: zone pivot="cs1-powershell" + + ```powershell + docker exec -it sql1 "bash" + ``` + + ::: zone-end + + ::: zone pivot="cs1-cmd" + + ```cmd + docker exec -it sql1 "bash" + ``` + + ::: zone-end + + +::: moniker range="=sql-server-linux-2017 || =sql-server-2017" + +1. Once inside the container, connect locally with **sqlcmd**, using its full path. + + ```bash + /opt/mssql-tools/bin/sqlcmd -S localhost -U -P "" + ``` + + Recent versions of **sqlcmd** are secure by default. For more information about connection encryption, see [sqlcmd utility](../tools/sqlcmd/sqlcmd-utility.md) for Windows, and [Connecting with sqlcmd](../connect/odbc/linux-mac/connecting-with-sqlcmd.md) for Linux and macOS. If the connection doesn't succeed, you can add the `-No` option to **sqlcmd** to specify that encryption is optional, not mandatory. + + You can omit the password on the command-line to be prompted to enter it. For example: + + ```bash + /opt/mssql-tools/bin/sqlcmd -S localhost -U + ``` + +::: moniker-end + + +::: moniker range="=sql-server-linux-ver15 || =sql-server-ver15" + +1. Once inside the container, connect locally with **sqlcmd**, using its full path. + + ```bash + /opt/mssql-tools18/bin/sqlcmd -S localhost -U -P "" + ``` + + Recent versions of **sqlcmd** are secure by default. For more information about connection encryption, see [sqlcmd utility](../tools/sqlcmd/sqlcmd-utility.md) for Windows, and [Connecting with sqlcmd](../connect/odbc/linux-mac/connecting-with-sqlcmd.md) for Linux and macOS. If the connection doesn't succeed, you can add the `-No` option to **sqlcmd** to specify that encryption is optional, not mandatory. + + You can omit the password on the command-line to be prompted to enter it. For example: + + ```bash + /opt/mssql-tools18/bin/sqlcmd -S localhost -U + ``` + +::: moniker-end + + +::: moniker range="= sql-server-linux-ver16 || = sql-server-ver16" + +1. Once inside the container, connect locally with **sqlcmd**, using its full path. + + ```bash + /opt/mssql-tools18/bin/sqlcmd -S localhost -U -P "" + ``` + + Recent versions of **sqlcmd** are secure by default. For more information about connection encryption, see [sqlcmd utility](../tools/sqlcmd/sqlcmd-utility.md) for Windows, and [Connecting with sqlcmd](../connect/odbc/linux-mac/connecting-with-sqlcmd.md) for Linux and macOS. If the connection doesn't succeed, you can add the `-No` option to **sqlcmd** to specify that encryption is optional, not mandatory. + + You can omit the password on the command-line to be prompted to enter it. For example: + + ```bash + /opt/mssql-tools18/bin/sqlcmd -S localhost -U + ``` + +::: moniker-end + +1. If successful, you should get to a **sqlcmd** command prompt: `1>`. + +## Create and query data + +The following sections walk you through using **sqlcmd** and Transact-SQL to create a new database, add data, and run a query. + +### Create a new database + +The following steps create a new database named `TestDB`. + +1. From the **sqlcmd** command prompt, paste the following Transact-SQL command to create a test database: + + ```sql + CREATE DATABASE TestDB; + ``` + +1. On the next line, write a query to return the name of all of the databases on your server: + + ```sql + SELECT name + FROM sys.databases; + ``` + +1. The previous two commands weren't run immediately. Type `GO` on a new line to run the previous commands: + + ```sql + GO + ``` + +### Insert data + +Next create a new table, `Inventory`, and insert two new rows. + +1. From the *sqlcmd* command prompt, switch context to the new `TestDB` database: + + ```sql + USE TestDB; + ``` + +1. Create new table named `Inventory`: + + ```sql + CREATE TABLE Inventory + ( + id INT, + name NVARCHAR (50), + quantity INT + ); + ``` + +1. Insert data into the new table: + + ```sql + INSERT INTO Inventory + VALUES (1, 'banana', 150); + + INSERT INTO Inventory + VALUES (2, 'orange', 154); + ``` + +1. Type `GO` to run the previous commands: + + ```sql + GO + ``` + +### Select data + +Now, run a query to return data from the `Inventory` table. + +1. From the **sqlcmd** command prompt, enter a query that returns rows from the `Inventory` table where the quantity is greater than 152: + + ```sql + SELECT * + FROM Inventory + WHERE quantity > 152; + ``` + +1. Run the command: + + ```sql + GO + ``` + +### Exit the sqlcmd command prompt + +1. To end your **sqlcmd** session, type `QUIT`: + + ```sql + QUIT + ``` + +1. To exit the interactive command-prompt in your container, type `exit`. Your container continues to run after you exit the interactive bash shell. + + + +## Connect from outside the container + +## [CLI](#tab/cli) + +You can also connect to the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] instance on your Docker machine from any external Linux, Windows, or macOS tool that supports SQL connections. The external tool uses the IP address for the host machine. + +The following steps use **sqlcmd** outside of your container to connect to [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] running in the container. These steps assume that you already have the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] command-line tools installed outside of your container. The same principles apply when using other tools, but the process of connecting is unique to each tool. + +1. Find the IP address for your container's host machine, using `ifconfig` or `ip addr`. + +1. For this example, install the **sqlcmd** tool on your client machine. For more information, see [sqlcmd utility](../tools/sqlcmd/sqlcmd-utility.md) or [Install the SQL Server command-line tools sqlcmd and bcp on Linux](sql-server-linux-setup-tools.md). + +1. Run **sqlcmd** specifying the IP address and the port mapped to port 1433 in your container. In this example, the port is the same as port 1433 on the host machine. If you specified a different mapped port on the host machine, you would use it here. You also need to open the appropriate inbound port on your firewall to allow the connection. + + Recent versions of **sqlcmd** are secure by default. If the connection doesn't succeed, and you're using version 18 or higher, you can add the `-No` option to **sqlcmd** to specify that encryption is optional, not mandatory. + + ::: zone pivot="cs1-bash" + + ```bash + sudo sqlcmd -S ,1433 -U -P "" + ``` + + ::: zone-end + + ::: zone pivot="cs1-powershell" + + ```powershell + sqlcmd -S ,1433 -U -P "" + ``` + + ::: zone-end + + ::: zone pivot="cs1-cmd" + + ```cmd + sqlcmd -S ,1433 -U -P "" + ``` + + ::: zone-end + + > [!CAUTION] + > [!INCLUDE [password-complexity](includes/password-complexity.md)] + +1. Run Transact-SQL commands. When finished, type `QUIT`. + +## [sqlcmd](#tab/sqlcmd) + +You can also connect to the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] instance on your Docker machine from any external Linux, Windows, or macOS tool that supports SQL connections. The external tool uses the IP address for the host machine. + +The following steps use **sqlcmd** outside of your container to connect to [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] running in the container. The same principles apply when using other tools, but the process of connecting is unique to each tool. + +1. Run **sqlcmd** in the same session you used to create your container. It keeps track of the connection information via contexts so you can easily connect at any time. `sqlcmd config view` can be used to view your available contexts. + + ::: zone pivot="cs1-bash" + + ```bash + sudo sqlcmd + ``` + + ::: zone-end + + ::: zone pivot="cs1-powershell" + + ```powershell + sqlcmd query + ``` + + ::: zone-end + + ::: zone pivot="cs1-cmd" + + ```cmd + sqlcmd query + ``` + + ::: zone-end + +1. Run Transact-SQL commands. When finished, type `QUIT`. + +--- + +Other common tools to connect to [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] include: + +- [SQL Server extension for Visual Studio Code](../tools/visual-studio-code/sql-server-develop-use-vscode.md) +- [Use SQL Server Management Studio on Windows to manage SQL Server on Linux](sql-server-linux-manage-ssms.md) +- [What is Azure Data Studio?](/azure-data-studio/what-is-azure-data-studio) +- [mssql-cli (Preview)](https://github.com/dbcli/mssql-cli/blob/master/doc/usage_guide.md) +- [Manage SQL Server on Linux with PowerShell Core](sql-server-linux-manage-powershell-core.md) + +## Remove your container + +## [CLI](#tab/cli) + +If you want to remove the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] container used in this tutorial, run the following commands: + +::: zone pivot="cs1-bash" + +```bash +docker stop sql1 +docker rm sql1 +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +docker stop sql1 +docker rm sql1 +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +docker stop sql1 +docker rm sql1 +``` + +::: zone-end + +## [sqlcmd](#tab/sqlcmd) + +If you want to remove the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] container used in this tutorial, run the following command: + +::: zone pivot="cs1-bash" + +```bash +sudo sqlcmd delete --force +``` + +::: zone-end + +::: zone pivot="cs1-powershell" + +```powershell +sqlcmd delete --force +``` + +::: zone-end + +::: zone pivot="cs1-cmd" + +```cmd +sqlcmd delete --force +``` + +::: zone-end + +--- + +## Docker demo + +After you finish using the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md.md)] Linux container image for Docker, you might want to know how Docker is used to improve development and testing. The following video shows how Docker can be used in a continuous integration and deployment scenario. + +> [!VIDEO https://channel9.msdn.com/Events/Connect/2017/T152/player] + +## Related tasks + +- [Run multiple SQL Server containers](sql-server-linux-docker-container-deployment.md#multiple) +- [Persist your data](sql-server-linux-docker-container-configure.md#persist) + +## Related content + +- [Restore a SQL Server database in a Linux container](tutorial-restore-backup-in-sql-server-container.md) +- [Troubleshoot SQL Server Docker containers](sql-server-linux-docker-container-troubleshooting.md) +- [mssql-docker GitHub repository](https://github.com/microsoft/mssql-docker) + +[!INCLUDE [contribute-to-content](../includes/paragraph-content/contribute-to-content.md)] \ No newline at end of file diff --git a/tools/abc.md b/tools/abc.md new file mode 100644 index 000000000..b66506601 --- /dev/null +++ b/tools/abc.md @@ -0,0 +1,253 @@ +--- +title: 'Quickstart: Create an Azure Container Instance with a public IP address using Terraform' +description: 'In this article, you create an Azure Container Instance with a public IP address using Terraform' +ms.topic: quickstart +ms.service: azure-container-instances +ms.date: 08/29/2024 +ms.custom: devx-track-terraform, linux-related-content, innovation-engine +author: TomArcherMsft +ms.author: tarcher +content_well_notification: + - AI-contribution +ai-usage: ai-assisted +--- + +# Quickstart: Create an Azure Container Instance with a public IP address using Terraform + +Use Azure Container Instances to run serverless Docker containers in Azure with simplicity and speed. Deploy an application to a container instance on-demand when you don't need a full container orchestration platform like Azure Kubernetes Service. In this article, you use [Terraform](/azure/terraform) to deploy an isolated Docker container and make its web application available with a public IP address. + +[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] + +In this article, you learn how to: + +> [!div class="checklist"] +> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/resource_group/pet) +> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) +> * Create a random value for the container name using [random_string](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) +> * Create an Azure container group using [azurerm_container_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_group) + +## Prerequisites + +- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) + +## Implement the Terraform code + +> [!NOTE] +> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip/TestRecord.md). +> +> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) + +1. Create a directory in which to test and run the sample Terraform code and make it the current directory. + +1. Create a file named main.tf and insert the following code: + +```text +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +resource "azurerm_resource_group" "rg" { + name = random_pet.rg_name.id + location = var.resource_group_location +} + +resource "random_string" "container_name" { + length = 25 + lower = true + upper = false + special = false +} + +resource "azurerm_container_group" "container" { + name = "${var.container_group_name_prefix}-${random_string.container_name.result}" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + ip_address_type = "Public" + os_type = "Linux" + restart_policy = var.restart_policy + + container { + name = "${var.container_name_prefix}-${random_string.container_name.result}" + image = var.image + cpu = var.cpu_cores + memory = var.memory_in_gb + + ports { + port = var.port + protocol = "TCP" + } + } +} +``` + +1. Create a file named outputs.tf and insert the following code: + +```text +output "container_ipv4_address" { + value = azurerm_container_group.container.ip_address +} +``` + +1. Create a file named providers.tf and insert the following code: + +```text +terraform { + required_version = ">=1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} +provider "azurerm" { + features {} +} +``` + +1. Create a file named variables.tf and insert the following code: + +```text +variable "resource_group_location" { + type = string + default = "eastus" + description = "Location for all resources." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random value so name is unique in your Azure subscription." +} + +variable "container_group_name_prefix" { + type = string + description = "Prefix of the container group name that's combined with a random value so name is unique in your Azure subscription." + default = "acigroup" +} + +variable "container_name_prefix" { + type = string + description = "Prefix of the container name that's combined with a random value so name is unique in your Azure subscription." + default = "aci" +} + +variable "image" { + type = string + description = "Container image to deploy. Should be of the form repoName/imagename:tag for images stored in public Docker Hub, or a fully qualified URI for other registries. Images from private registries require additional registry credentials." + default = "mcr.microsoft.com/azuredocs/aci-helloworld" +} + +variable "port" { + type = number + description = "Port to open on the container and the public IP address." + default = 80 +} + +variable "cpu_cores" { + type = number + description = "The number of CPU cores to allocate to the container." + default = 1 +} + +variable "memory_in_gb" { + type = number + description = "The amount of memory to allocate to the container in gigabytes." + default = 2 +} + +variable "restart_policy" { + type = string + description = "The behavior of Azure runtime if container has stopped." + default = "Always" + validation { + condition = contains(["Always", "Never", "OnFailure"], var.restart_policy) + error_message = "The restart_policy must be one of the following: Always, Never, OnFailure." + } +} +``` + +## Initialize Terraform + +Before initializing Terraform, set the necessary environment variables. These variables are used by Terraform to provide default values for variables defined in the configuration files. + +```bash +export TF_VAR_resource_group_location="eastus" +export TF_VAR_resource_group_name_prefix="rg" +export TF_VAR_container_group_name_prefix="acigroup" +export TF_VAR_container_name_prefix="aci" +export TF_VAR_image="mcr.microsoft.com/azuredocs/aci-helloworld" +export TF_VAR_port=80 +export TF_VAR_cpu_cores=1 +export TF_VAR_memory_in_gb=2 +export TF_VAR_restart_policy="Always" +``` + +In this section, Terraform is initialized; this command downloads the Azure provider required to manage your Azure resources. Before running the command, ensure you are in the directory where you created the Terraform files. + +```bash +terraform init -upgrade +``` + +Key points: + +- The -upgrade parameter upgrades the necessary provider plugins to the newest version that complies with the configuration's version constraints. + +## Create a Terraform execution plan + +Run terraform plan to create an execution plan. + +```bash +terraform plan -out main.tfplan +``` + +Key points: + +- The terraform plan command creates an execution plan, but doesn't execute it. Instead, it determines what actions are necessary to create the configuration specified in your configuration files. This pattern allows you to verify whether the execution plan matches your expectations before making any changes to actual resources. +- The optional -out parameter allows you to specify an output file for the plan. Using the -out parameter ensures that the plan you reviewed is exactly what is applied. + +## Apply a Terraform execution plan + +Run terraform apply to execute the execution plan. + +```bash +terraform apply main.tfplan +``` + +Key points: + +- The example terraform apply command assumes you previously ran terraform plan -out main.tfplan. +- If you specified a different filename for the -out parameter, use that same filename in the call to terraform apply. +- If you didn't use the -out parameter, call terraform apply without any parameters. + +## Verify the results + +1. When you apply the execution plan, Terraform outputs the public IP address. To display the IP address again, run [terraform output](https://developer.hashicorp.com/terraform/cli/commands/output). + + ```bash + terraform output -raw container_ipv4_address + ``` + + +```text +"xxx.xxx.xxx.xxx" +``` + +2. Enter the sample's public IP address in your browser's address bar. + + :::image type="content" source="./media/container-instances-quickstart-terraform/azure-container-instances-demo.png" alt-text="Screenshot of the Azure Container Instances sample page" ::: + + + +## Troubleshoot Terraform on Azure + +[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) + +## Next steps + +> [!div class="nextstepaction"] +> [Tutorial: Create a container image for deployment to Azure Container Instances](./container-instances-tutorial-prepare-app.md) \ No newline at end of file diff --git a/tools/ansible.md b/tools/ansible.md new file mode 100644 index 000000000..8c6eca78a --- /dev/null +++ b/tools/ansible.md @@ -0,0 +1,157 @@ +--- +title: Create a Linux virtual machines in Azure using Ansible +description: Learn how to create a Linux virtual machine in Azure using Ansible +keywords: ansible, azure, devops, virtual machine +ms.topic: tutorial +ms.date: 08/14/2024 +ms.custom: devx-track-ansible, linux-related-content, innovation-engine +author: +ms.author: +--- + +# Create a Linux virtual machines in Azure using Ansible + +This article presents a sample Ansible playbook for configuring a Linux virtual machine. + +In this article, you learn how to: + +> [!div class="checklist"] +> * Create a resource group +> * Create a virtual network +> * Create a public IP address +> * Create a network security group +> * Create a virtual network interface card +> * Create a virtual machine + +## Configure your environment + +- **Azure subscription**: If you don't have an Azure subscription, create a [free account](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio) before you begin. +- **Install Ansible**: Do one of the following options: + + - [Install](/azure/ansible/ansible-install-configure#install-ansible-on-an-azure-linux-virtual-machine) and [configure](/azure/ansible/ansible-install-configure#create-azure-credentials) Ansible on a Linux virtual machine + - [Configure Azure Cloud Shell](/azure/cloud-shell/quickstart) + +## Implement the Ansible playbook + +1. Create a directory in which to test and run the sample Ansible code and make it the current directory. + +2. Create a file named main.yml and insert the following code. In the playbook below the resource group name and other relevant properties use environment variables so that they are unique for each run. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="eastus2" +export MY_RESOURCE_GROUP="myResourceGroup$RANDOM_SUFFIX" +export MY_VM_NAME="myVM$RANDOM_SUFFIX" +export MY_VNET_NAME="myVnet$RANDOM_SUFFIX" +export MY_SUBNET_NAME="mySubnet$RANDOM_SUFFIX" +export MY_NIC_NAME="myNIC$RANDOM_SUFFIX" +export MY_PUBLIC_IP_NAME="myPublicIP$RANDOM_SUFFIX" +export MY_NSG_NAME="myNetworkSecurityGroup$RANDOM_SUFFIX" + +cat > main.yml <<'EOF' +- name: Create Azure VM + hosts: localhost + connection: local + tasks: + - name: Create resource group + azure_rm_resourcegroup: + name: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + location: "{{ lookup('env', 'REGION') }}" + - name: Create virtual network + azure_rm_virtualnetwork: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_VNET_NAME') }}" + address_prefixes: "10.0.0.0/16" + - name: Add subnet + azure_rm_subnet: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_SUBNET_NAME') }}" + address_prefix: "10.0.1.0/24" + virtual_network: "{{ lookup('env', 'MY_VNET_NAME') }}" + - name: Create public IP address + azure_rm_publicipaddress: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + allocation_method: Static + name: "{{ lookup('env', 'MY_PUBLIC_IP_NAME') }}" + register: output_ip_address + - name: Public IP of VM + debug: + msg: "The public IP is {{ output_ip_address.state.ip_address }}." + - name: Create Network Security Group that allows SSH + azure_rm_securitygroup: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_NSG_NAME') }}" + rules: + - name: SSH + protocol: Tcp + destination_port_range: 22 + access: Allow + priority: 1001 + direction: Inbound + - name: Create virtual network interface card + azure_rm_networkinterface: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_NIC_NAME') }}" + virtual_network: "{{ lookup('env', 'MY_VNET_NAME') }}" + subnet_name: "{{ lookup('env', 'MY_SUBNET_NAME') }}" + security_group: "{{ lookup('env', 'MY_NSG_NAME') }}" + ip_configurations: + - name: ipconfig1 + public_ip_address_name: "{{ lookup('env', 'MY_PUBLIC_IP_NAME') }}" + primary: yes + - name: Create VM + azure_rm_virtualmachine: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_VM_NAME') }}" + vm_size: Standard_DS1_v2 + admin_username: azureuser + ssh_password_enabled: false + generate_ssh_keys: yes # This will automatically generate keys if they don't exist + network_interfaces: "{{ lookup('env', 'MY_NIC_NAME') }}" + image: + offer: 0001-com-ubuntu-server-jammy + publisher: Canonical + sku: 22_04-lts + version: latest +EOF +``` + +## Run the playbook + +Run the Ansible playbook using the ansible-playbook command. + +```bash +ansible-playbook main.yml +``` + +## Verify the results + +Run the following command to verify the VM was created. This command filters the VMs by name. + +```azurecli +az vm list -d -o table --query "[?name=='${MY_VM_NAME}']" +``` + + +```JSON +[ + { + "name": "myVM", + "powerState": "running", + "publicIps": "xxx.xxx.xxx.xxx" + } +] +``` + +## Connect to the VM + +Run the SSH command to connect to your new Linux VM. Replace the placeholder with the IP address obtained from the previous step. + +```bash +ssh -o StrictHostKeyChecking=no azureuser@$MY_PUBLIC_IP_NAME +``` + +## Next steps + +> [!div class="nextstepaction"] +> [Manage a Linux virtual machine in Azure using Ansible](./vm-manage.md) \ No newline at end of file diff --git a/tools/demo_notes.txt b/tools/demo_notes.txt deleted file mode 100644 index 2201427b5..000000000 --- a/tools/demo_notes.txt +++ /dev/null @@ -1,71 +0,0 @@ -# Show AKS cluster: - -kubectl get nodes - -# Install kubectl plugin: - -kubectl krew install gadget - -# Verify version and server status: - -kubectl gadget version -# Expected output: -# Client version: vX.Y.Z -# Server version: not available - -# Deploy Inspektor Gadget: - -kubectl gadget deploy --otel-metrics-listen --otel-metrics-listen-address 0.0.0.0:2223 - -# Verify version and server status: - -kubectl gadget version -# Expected output: -# Client version: vX.Y.Z -# Server version: not available - -# Run simple example with trace_exec: - -# Run gadget -kubectl gadget run trace_exec:v0.38.0 - -# Run test pod -kubectl run -ti 1p-demo-pod --rm --image=ubuntu -# /bin/bash - -# Run gadget with JSON -kubectl gadget run trace_exec:v0.38.0 --output jsonpretty - -# Run gadget with filtering - -kubectl gadget run trace_exec:v0.38.0 --all-namespaces --filter proc.comm=bash - -# Generate a metric based on these events: - -vi alert-bad-process.yaml - -# Run gadget manifest to export metrics: - -kubectl gadget run -f alert-bad-process.yaml --annotate exec:metrics.collect=true,exec:metrics.implicit-counter.name=shell_executions,exec.k8s.namespace:metrics.type=key,exec.k8s.podname:metrics.type=key,exec.k8s.containername:metrics.type=key --detach - -# Verify gadget is running in headless mode: - -kubectl gadget list - -kubectl gadget attach alert-bad-process - -# Configure managed Prometheus to collect data from the OTEL listener endpoint we expose on each IG pod? -# Documentation: https://learn.microsoft.com/en-us/azure/azure-monitor/containers/prometheus-metrics-scrape-configuration?tabs=CRDConfig%2CCRDScrapeConfig%2CConfigFileScrapeConfigBasicAuth%2CConfigFileScrapeConfigTLSAuth#configmaps - -kubectl get configmaps -n kube-system ama-metrics-settings-configmap - -# It should contain: pod-annotation-based-scraping: podannotationnamespaceregex = "gadget" -kubectl get configmaps -n kube-system ama-metrics-settings-configmap -o yaml | less - -# Show shell_executions_total metric in Grafana dashboard: shell_executions_total -# Documentation: https://learn.microsoft.com/en-us/azure/managed-grafana/overview - -# Create a prometheus group alert with the rule "shell_executions_total > 0" -# Documentation: https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-rule-groups - -# Undeploy IG -kubectl gadget undeploy diff --git a/tools/main.tf b/tools/main.tf new file mode 100644 index 000000000..907434694 --- /dev/null +++ b/tools/main.tf @@ -0,0 +1,36 @@ +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +resource "azurerm_resource_group" "rg" { + name = random_pet.rg_name.id + location = var.resource_group_location +} + +resource "random_string" "container_name" { + length = 25 + lower = true + upper = false + special = false +} + +resource "azurerm_container_group" "container" { + name = "${var.container_group_name_prefix}-${random_string.container_name.result}" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + ip_address_type = "Public" + os_type = "Linux" + restart_policy = var.restart_policy + + container { + name = "${var.container_name_prefix}-${random_string.container_name.result}" + image = var.image + cpu = var.cpu_cores + memory = var.memory_in_gb + + ports { + port = var.port + protocol = "TCP" + } + } +} \ No newline at end of file diff --git a/tools/main.yml b/tools/main.yml new file mode 100644 index 000000000..e931e54c4 --- /dev/null +++ b/tools/main.yml @@ -0,0 +1,64 @@ +- name: Create Azure VM + hosts: localhost + connection: local + tasks: + - name: Create resource group + azure_rm_resourcegroup: + name: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + location: "{{ lookup('env', 'REGION') }}" + - name: Create virtual network + azure_rm_virtualnetwork: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_VNET_NAME') }}" + address_prefixes: "10.0.0.0/16" + - name: Add subnet + azure_rm_subnet: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_SUBNET_NAME') }}" + address_prefix: "10.0.1.0/24" + virtual_network: "{{ lookup('env', 'MY_VNET_NAME') }}" + - name: Create public IP address + azure_rm_publicipaddress: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + allocation_method: Static + name: "{{ lookup('env', 'MY_PUBLIC_IP_NAME') }}" + register: output_ip_address + - name: Public IP of VM + debug: + msg: "The public IP is {{ output_ip_address.state.ip_address }}." + - name: Create Network Security Group that allows SSH + azure_rm_securitygroup: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_NSG_NAME') }}" + rules: + - name: SSH + protocol: Tcp + destination_port_range: 22 + access: Allow + priority: 1001 + direction: Inbound + - name: Create virtual network interface card + azure_rm_networkinterface: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_NIC_NAME') }}" + virtual_network: "{{ lookup('env', 'MY_VNET_NAME') }}" + subnet_name: "{{ lookup('env', 'MY_SUBNET_NAME') }}" + security_group: "{{ lookup('env', 'MY_NSG_NAME') }}" + ip_configurations: + - name: ipconfig1 + public_ip_address_name: "{{ lookup('env', 'MY_PUBLIC_IP_NAME') }}" + primary: yes + - name: Create VM + azure_rm_virtualmachine: + resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" + name: "{{ lookup('env', 'MY_VM_NAME') }}" + vm_size: Standard_DS1_v2 + admin_username: azureuser + ssh_password_enabled: false + generate_ssh_keys: yes # This will automatically generate keys if they don't exist + network_interfaces: "{{ lookup('env', 'MY_NIC_NAME') }}" + image: + offer: 0001-com-ubuntu-server-jammy + publisher: Canonical + sku: 22_04-lts + version: latest diff --git a/tools/outputs.tf b/tools/outputs.tf new file mode 100644 index 000000000..20f77aa8e --- /dev/null +++ b/tools/outputs.tf @@ -0,0 +1,3 @@ +output "container_ipv4_address" { + value = azurerm_container_group.container.ip_address +} \ No newline at end of file diff --git a/tools/providers.tf b/tools/providers.tf new file mode 100644 index 000000000..4fd5f6ba7 --- /dev/null +++ b/tools/providers.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">=1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} +provider "azurerm" { + features {} +} \ No newline at end of file diff --git a/tools/test.md b/tools/test.md new file mode 100644 index 000000000..480bd2bf1 --- /dev/null +++ b/tools/test.md @@ -0,0 +1,355 @@ +--- +title: Azure Linux Container Host for AKS tutorial - Migrating to Azure Linux +description: In this Azure Linux Container Host for AKS tutorial, you learn how to migrate your nodes to Azure Linux nodes. +author: suhuruli +ms.author: suhuruli +ms.reviewer: schaffererin +ms.service: microsoft-linux +ms.custom: devx-track-azurecli, linux-related-content +ms.topic: tutorial +ms.date: 01/19/2024 +--- + +# Tutorial: Migrate nodes to Azure Linux + +In this tutorial, part three of five, you migrate your existing nodes to Azure Linux. You can migrate your existing nodes to Azure Linux using one of the following methods: + +* Remove existing node pools and add new Azure Linux node pools. +* In-place OS SKU migration. + +If you don't have any existing nodes to migrate to Azure Linux, skip to the [next tutorial](./tutorial-azure-linux-telemetry-monitor.md). In later tutorials, you learn how to enable telemetry and monitoring in your clusters and upgrade Azure Linux nodes. + +## Prerequisites + +* In previous tutorials, you created and deployed an Azure Linux Container Host for AKS cluster. To complete this tutorial, you need to add an Azure Linux node pool to your existing cluster. If you haven't done this step and would like to follow along, start with [Tutorial 2: Add an Azure Linux node pool to your existing AKS cluster](./tutorial-azure-linux-add-nodepool.md). + + > [!NOTE] + > When adding a new Azure Linux node pool, you need to add at least one as `--mode System`. Otherwise, AKS won't allow you to delete your existing node pool. + +* You need the latest version of Azure CLI. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). + +## Add Azure Linux node pools and remove existing node pools + +1. Add a new Azure Linux node pool using the `az aks nodepool add` command. This command adds a new node pool to your cluster with the `--mode System` flag, which makes it a system node pool. System node pools are required for Azure Linux clusters. + + ```azurecli-interactive + az aks nodepool add --resource-group --cluster-name --name --mode System --os-sku AzureLinux + ``` + +2. Remove your existing nodes using the `az aks nodepool delete` command. + + ```azurecli-interactive + az aks nodepool delete --resource-group --cluster-name --name + ``` + +## In-place OS SKU migration + +You can now migrate your existing Ubuntu node pools to Azure Linux by changing the OS SKU of the node pool, which rolls the cluster through the standard node image upgrade process. This new feature doesn't require the creation of new node pools. + +### Limitations + +There are several settings that can block the OS SKU migration request. To ensure a successful migration, review the following guidelines and limitations: + +* The OS SKU migration feature isn't available through PowerShell or the Azure portal. +* The OS SKU migration feature isn't able to rename existing node pools. +* Ubuntu and Azure Linux are the only supported Linux OS SKU migration targets. +* An Ubuntu OS SKU with `UseGPUDedicatedVHD` enabled can't perform an OS SKU migration. +* An Ubuntu OS SKU with CVM 20.04 enabled can't perform an OS SKU migration. +* Node pools with Kata enabled can't perform an OS SKU migration. +* Windows OS SKU migration isn't supported. +* OS SKU migration from Mariner to Azure Linux is supported, but rolling back to Mariner is not supported. + +### Prerequisites + +* An existing AKS cluster with at least one Ubuntu node pool. +* We recommend that you ensure your workloads configure and run successfully on the Azure Linux container host before attempting to use the OS SKU migration feature by [deploying an Azure Linux cluster](./quickstart-azure-cli.md) in dev/prod and verifying your service remains healthy. +* Ensure the migration feature is working for you in test/dev before using the process on a production cluster. +* Ensure that your pods have enough [Pod Disruption Budget](/azure/aks/operator-best-practices-scheduler#plan-for-availability-using-pod-disruption-budgets) to allow AKS to move pods between VMs during the upgrade. +* You need Azure CLI version [2.61.0](/cli/azure/release-notes-azure-cli#may-21-2024) or higher. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). +* If you are using Terraform, you must have [v3.111.0](https://github.com/hashicorp/terraform-provider-azurerm/releases/tag/v3.111.0) or greater of the AzureRM Terraform module. + +### [Azure CLI](#tab/azure-cli) + +#### Migrate the OS SKU of your Ubuntu node pool + +* Migrate the OS SKU of your node pool to Azure Linux using the `az aks nodepool update` command. This command updates the OS SKU for your node pool from Ubuntu to Azure Linux. The OS SKU change triggers an immediate upgrade operation, which takes several minutes to complete. + + ```azurecli-interactive + az aks nodepool update --resource-group --cluster-name --name --os-sku AzureLinux + ``` + + > [!NOTE] + > If you experience issues during the OS SKU migration, you can [roll back to your previous OS SKU](#rollback). + +### [ARM template](#tab/arm-template) + +#### Example ARM templates + +##### 0base.json + +```json + { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2023-07-01", + "name": "akstestcluster", + "location": "[resourceGroup().location]", + "tags": { + "displayname": "Demo of AKS Nodepool Migration" + }, + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "enableRBAC": true, + "dnsPrefix": "testcluster", + "agentPoolProfiles": [ + { + "name": "testnp", + "count": 3, + "vmSize": "Standard_D4a_v4", + "osType": "Linux", + "osSku": "Ubuntu", + "mode": "System" + } + ] + } + } + ] +} +``` + +##### 1mcupdate.json + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.ContainerService/managedClusters", + "apiVersion": "2023-07-01", + "name": "akstestcluster", + "location": "[resourceGroup().location]", + "tags": { + "displayname": "Demo of AKS Nodepool Migration" + }, + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "enableRBAC": true, + "dnsPrefix": "testcluster", + "agentPoolProfiles": [ + { + "name": "testnp", + "osType": "Linux", + "osSku": "AzureLinux", + "mode": "System" + } + ] + } + } + ] +} +``` + +##### 2apupdate.json + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "apiVersion": "2023-07-01", + "type": "Microsoft.ContainerService/managedClusters/agentPools", + "name": "akstestcluster/testnp", + "location": "[resourceGroup().location]", + "properties": { + "osType": "Linux", + "osSku": "Ubuntu", + "mode": "System" + } + } + ] +} +``` + +#### Deploy a test cluster + +1. Create a resource group for the test cluster using the `az group create` command. + + ```azurecli-interactive + az group create --name testRG --location eastus + ``` + +2. Deploy a baseline Ubuntu OS SKU cluster with three nodes using the `az deployment group create` command and the [0base.json example ARM template](#0basejson). + + ```azurecli-interactive + az deployment group create --resource-group testRG --template-file 0base.json + ``` + +3. Migrate the OS SKU of your system node pool to Azure Linux using the `az deployment group create` command. + + ```azurecli-interactive + az deployment group create --resource-group testRG --template-file 1mcupdate.json + ``` + +4. Migrate the OS SKU of your system node pool back to Ubuntu using the `az deployment group create` command. + + ```azurecli-interactive + az deployment group create --resource-group testRG --template-file 2apupdate.json + ``` + +### [Terraform](#tab/terraform) + +#### Example Terraform template + +1. Confirm that your `providers.tf` file is updated to pick up the required version of the Azure provider. + +##### providers.tf + +```terraform +terraform { + required_version = ">=1.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.111.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } + } + + provider "azurerm" { + features {} + } +``` + +2. For brevity, only the snippet of the Terraform template that is of interest is displayed below. In this initial configuration, an AKS cluster with a nodepool of **os_sku** with **Ubuntu** is deployed. + +##### base.tf + +```terraform +resource "azurerm_kubernetes_cluster" "k8s" { + location = azurerm_resource_group.rg.location + name = var.cluster_name + resource_group_name = azurerm_resource_group.rg.name + dns_prefix = var.dns_prefix + tags = { + Environment = "Development" + } + + default_node_pool { + name = "azurelinuxpool" + vm_size = "Standard_D2_v2" + node_count = var.agent_count + os_sku = "Ubuntu" + } + linux_profile { + admin_username = "azurelinux" + + ssh_key { + key_data = file(var.ssh_public_key) + } + } + network_profile { + network_plugin = "kubenet" + load_balancer_sku = "standard" + } + service_principal { + client_id = var.aks_service_principal_app_id + client_secret = var.aks_service_principal_client_secret + } +} +``` + +3. To run an in-place OS SKU migration, just replace the **os_sku** to **AzureLinux** and re-apply the Terraform plan. + +##### update.tf + +```terraform +resource "azurerm_kubernetes_cluster" "k8s" { + location = azurerm_resource_group.rg.location + name = var.cluster_name + resource_group_name = azurerm_resource_group.rg.name + dns_prefix = var.dns_prefix + tags = { + Environment = "Development" + } + + default_node_pool { + name = "azurelinuxpool" + vm_size = "Standard_D2_v2" + node_count = var.agent_count + os_sku = "AzureLinux" + } + linux_profile { + admin_username = "azurelinux" + + ssh_key { + key_data = file(var.ssh_public_key) + } + } + network_profile { + network_plugin = "kubenet" + load_balancer_sku = "standard" + } + service_principal { + client_id = var.aks_service_principal_app_id + client_secret = var.aks_service_principal_client_secret + } +} +``` + +--- + +### Verify the OS SKU migration + +Once the migration is complete on your test clusters, you should verify the following to ensure a successful migration: + +* If your migration target is Azure Linux, run the `kubectl get nodes -o wide` command. The output should show `CBL-Mariner/Linux` as your OS image and `.cm2` at the end of your kernel version. +* Run the `kubectl get pods -o wide -A` command to verify that all of your pods and daemonsets are running on the new node pool. +* Run the `kubectl get nodes --show-labels` command to verify that all of the node labels in your upgraded node pool are what you expect. + +> [!TIP] +> We recommend monitoring the health of your service for a couple weeks before migrating your production clusters. + +### Run the OS SKU migration on your production clusters + +1. Update your existing templates to set `OSSKU=AzureLinux`. In ARM templates, you use `"OSSKU: "AzureLinux"` in the `agentPoolProfile` section. In Bicep, you use `osSku: "AzureLinux"` in the `agentPoolProfile` section. Lastly, for Terraform, you use `"os_sku = "AzureLinux"` in the `default_node_pool` section. Make sure that your `apiVersion` is set to `2023-07-01` or later. +2. Redeploy your ARM, Bicep, or Terraform template for the cluster to apply the new `OSSKU` setting. During this deploy, your cluster behaves as if it's taking a node image upgrade. Your cluster surges capacity, and then reboots your existing nodes one by one into the latest AKS image from your new OS SKU. + +### Rollback + +If you experience issues during the OS SKU migration, you can roll back to your previous OS SKU. To do this, you need to change the OS SKU field in your template and resubmit the deployment, which triggers another upgrade operation and restores the node pool to its previous OS SKU. + + > [!NOTE] + > + > OS SKU migration does not support rolling back to OS SKU Mariner. + +* Roll back to your previous OS SKU using the `az aks nodepool update` command. This command updates the OS SKU for your node pool from Azure Linux back to Ubuntu. + + ```azurecli-interactive + az aks nodepool update --resource-group myResourceGroup --cluster-name myAKSCluster --name mynodepool --os-sku Ubuntu + ``` + +## Next steps + +In this tutorial, you migrated existing nodes to Azure Linux using one of the following methods: + +* Remove existing node pools and add new Azure Linux node pools. +* In-place OS SKU migration. + +In the next tutorial, you learn how to enable telemetry to monitor your clusters. + +> [!div class="nextstepaction"] +> [Enable telemetry and monitoring](./tutorial-azure-linux-telemetry-monitor.md) \ No newline at end of file diff --git a/tools/variables.tf b/tools/variables.tf new file mode 100644 index 000000000..cfadf0f4a --- /dev/null +++ b/tools/variables.tf @@ -0,0 +1,57 @@ +variable "resource_group_location" { + type = string + default = "eastus" + description = "Location for all resources." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random value so name is unique in your Azure subscription." +} + +variable "container_group_name_prefix" { + type = string + description = "Prefix of the container group name that's combined with a random value so name is unique in your Azure subscription." + default = "acigroup" +} + +variable "container_name_prefix" { + type = string + description = "Prefix of the container name that's combined with a random value so name is unique in your Azure subscription." + default = "aci" +} + +variable "image" { + type = string + description = "Container image to deploy. Should be of the form repoName/imagename:tag for images stored in public Docker Hub, or a fully qualified URI for other registries. Images from private registries require additional registry credentials." + default = "mcr.microsoft.com/azuredocs/aci-helloworld" +} + +variable "port" { + type = number + description = "Port to open on the container and the public IP address." + default = 80 +} + +variable "cpu_cores" { + type = number + description = "The number of CPU cores to allocate to the container." + default = 1 +} + +variable "memory_in_gb" { + type = number + description = "The amount of memory to allocate to the container in gigabytes." + default = 2 +} + +variable "restart_policy" { + type = string + description = "The behavior of Azure runtime if container has stopped." + default = "Always" + validation { + condition = contains(["Always", "Never", "OnFailure"], var.restart_policy) + error_message = "The restart_policy must be one of the following: Always, Never, OnFailure." + } +} \ No newline at end of file From 42bf42deb52ae468d797fea5d6ddc3e62b49da00 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Thu, 27 Mar 2025 19:30:36 -0700 Subject: [PATCH 49/87] updated latest metadata pending new converted docs --- .../articles/aks/node-image-upgrade.md | 173 +++++++++ ...ontainer-instances-quickstart-terraform.md | 93 ----- ...ontainer-instances-quickstart-terraform.md | 253 +++++++++++++ .../main.tf | 0 .../outputs.tf | 0 .../providers.tf | 0 .../variables.tf | 0 .../{ => quick-create-terraform}/main.tf | 0 .../{ => quick-create-terraform}/outputs.tf | 0 .../{ => quick-create-terraform}/providers.tf | 0 .../quick-create-terraform.md | 0 .../linux/{ => quick-create-terraform}/ssh.tf | 0 .../{ => quick-create-terraform}/variables.tf | 0 .../aks/learn/aks-store-quickstart.yaml | 0 .../aks/learn/quick-kubernetes-deploy-cli.md | 0 ...flexible-virtual-machine-scale-sets-cli.md | 0 .../tutorial-use-custom-image-cli.md | 0 .../linux/quick-create-cli.md | 0 .../linux/tutorial-lemp-stack.md | 0 scenarios/metadata.json | 30 +- tools/abc.md | 338 +++++++----------- tools/main.yml | 64 ---- 22 files changed, 568 insertions(+), 383 deletions(-) create mode 100644 scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md delete mode 100644 scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md create mode 100644 scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/container-instances-quickstart-terraform.md rename {tools => scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform}/main.tf (100%) rename {tools => scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform}/outputs.tf (100%) rename {tools => scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform}/providers.tf (100%) rename {tools => scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform}/variables.tf (100%) rename scenarios/azure-compute-docs/articles/virtual-machines/linux/{ => quick-create-terraform}/main.tf (100%) rename scenarios/azure-compute-docs/articles/virtual-machines/linux/{ => quick-create-terraform}/outputs.tf (100%) rename scenarios/azure-compute-docs/articles/virtual-machines/linux/{ => quick-create-terraform}/providers.tf (100%) rename scenarios/azure-compute-docs/articles/virtual-machines/linux/{ => quick-create-terraform}/quick-create-terraform.md (100%) rename scenarios/azure-compute-docs/articles/virtual-machines/linux/{ => quick-create-terraform}/ssh.tf (100%) rename scenarios/azure-compute-docs/articles/virtual-machines/linux/{ => quick-create-terraform}/variables.tf (100%) rename scenarios/{azure-aks-docs => azure-docs}/articles/aks/learn/aks-store-quickstart.yaml (100%) rename scenarios/{azure-aks-docs => azure-docs}/articles/aks/learn/quick-kubernetes-deploy-cli.md (100%) rename scenarios/{azure-compute-docs => azure-docs}/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md (100%) rename scenarios/{azure-compute-docs => azure-docs}/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md (100%) rename scenarios/{azure-compute-docs => azure-docs}/articles/virtual-machines/linux/quick-create-cli.md (100%) rename scenarios/{azure-compute-docs => azure-docs}/articles/virtual-machines/linux/tutorial-lemp-stack.md (100%) delete mode 100644 tools/main.yml diff --git a/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md b/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md new file mode 100644 index 000000000..bb08cd960 --- /dev/null +++ b/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md @@ -0,0 +1,173 @@ +--- +title: Upgrade Azure Kubernetes Service (AKS) node images +description: Learn how to upgrade the images on AKS cluster nodes and node pools. +ms.topic: how-to +ms.custom: devx-track-azurecli, innovation-engine +ms.subservice: aks-upgrade +ms.service: azure-kubernetes-service +ms.date: 09/20/2024 +author: schaffererin +ms.author: schaffererin +--- + +## Environment Variables + +The following environment variables are declared and will be used in subsequent code blocks. They replace the placeholder parameters in the original document with standardized variable names. + +```bash +export AKS_NODEPOOL="nodepool1" +export AKS_CLUSTER="apache-airflow-aks" +export AKS_RESOURCE_GROUP="apache-airflow-rg" +``` + +# Upgrade Azure Kubernetes Service (AKS) node images + +Azure Kubernetes Service (AKS) regularly provides new node images, so it's beneficial to upgrade your node images frequently to use the latest AKS features. Linux node images are updated weekly, and Windows node images are updated monthly. Image upgrade announcements are included in the [AKS release notes](https://github.com/Azure/AKS/releases), and it can take up to a week for these updates to be rolled out across all regions. You can also perform node image upgrades automatically and schedule them using planned maintenance. For more information, see [Automatically upgrade node images][auto-upgrade-node-image]. + +This article shows you how to upgrade AKS cluster node images and how to update node pool images without upgrading the Kubernetes version. For information on upgrading the Kubernetes version for your cluster, see [Upgrade an AKS cluster][upgrade-cluster]. + +> [!NOTE] +> The AKS cluster must use virtual machine scale sets for the nodes. +> +> It's not possible to downgrade a node image version (for example *AKSUbuntu-2204 to AKSUbuntu-1804*, or *AKSUbuntu-2204-202308.01.0 to AKSUbuntu-2204-202307.27.0*). + +## Check for available node image upgrades + +1. Check for available node image upgrades using the [`az aks nodepool get-upgrades`][az-aks-nodepool-get-upgrades] command. + + ```azurecli-interactive + az aks nodepool get-upgrades \ + --nodepool-name $AKS_NODEPOOL \ + --cluster-name $AKS_CLUSTER \ + --resource-group $AKS_RESOURCE_GROUP + ``` + +1. In the output, find and make note of the `latestNodeImageVersion` value. This value is the latest node image version available for your node pool. +1. Check your current node image version to compare with the latest version using the [`az aks nodepool show`][az-aks-nodepool-show] command. + + ```azurecli-interactive + az aks nodepool show \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL \ + --query nodeImageVersion + ``` + +1. If the `nodeImageVersion` value is different from the `latestNodeImageVersion`, you can upgrade your node image. + +## Upgrade all node images in all node pools + +1. Upgrade all node images in all node pools in your cluster using the [`az aks upgrade`][az-aks-upgrade] command with the `--node-image-only` flag. + + ```text + az aks upgrade \ + --resource-group $AKS_RESOURCE_GROUP \ + --name $AKS_CLUSTER \ + --node-image-only \ + --yes + ``` + +1. You can check the status of the node images using the `kubectl get nodes` command. + + > [!NOTE] + > This command might differ slightly depending on the shell you use. For more information on Windows and PowerShell environments, see the [Kubernetes JSONPath documentation][kubernetes-json-path]. + + ```bash + kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' + ``` + +1. When the upgrade completes, use the [`az aks show`][az-aks-show] command to get the updated node pool details. The current node image is shown in the `nodeImageVersion` property. + + ```azurecli-interactive + az aks show \ + --resource-group $AKS_RESOURCE_GROUP \ + --name $AKS_CLUSTER + ``` + +## Upgrade a specific node pool + +1. Update the OS image of a node pool without doing a Kubernetes cluster upgrade using the [`az aks nodepool upgrade`][az-aks-nodepool-upgrade] command with the `--node-image-only` flag. + + ```azurecli-interactive + az aks nodepool upgrade \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL \ + --node-image-only + ``` + +1. You can check the status of the node images with the `kubectl get nodes` command. + + > [!NOTE] + > This command may differ slightly depending on the shell you use. For more information on Windows and PowerShell environments, see the [Kubernetes JSONPath documentation][kubernetes-json-path]. + + ```bash + kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' + ``` + +1. When the upgrade completes, use the [`az aks nodepool show`][az-aks-nodepool-show] command to get the updated node pool details. The current node image is shown in the `nodeImageVersion` property. + + ```azurecli-interactive + az aks nodepool show \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL + ``` + +## Upgrade node images with node surge + +To speed up the node image upgrade process, you can upgrade your node images using a customizable node surge value. By default, AKS uses one extra node to configure upgrades. + +1. Upgrade node images with node surge using the [`az aks nodepool update`][az-aks-nodepool-update] command with the `--max-surge` flag to configure the number of nodes used for upgrades. + + > [!NOTE] + > To learn more about the trade-offs of various `--max-surge` settings, see [Customize node surge upgrade][max-surge]. + + ```azurecli-interactive + az aks nodepool update \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL \ + --max-surge 33% \ + --no-wait + ``` + +1. You can check the status of the node images with the `kubectl get nodes` command. + + ```bash + kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' + ``` + +1. Get the updated node pool details using the [`az aks nodepool show`][az-aks-nodepool-show] command. The current node image is shown in the `nodeImageVersion` property. + + ```azurecli-interactive + az aks nodepool show \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL + ``` + +## Next steps + +- For information about the latest node images, see the [AKS release notes](https://github.com/Azure/AKS/releases). +- Learn how to upgrade the Kubernetes version with [Upgrade an AKS cluster][upgrade-cluster]. +- [Automatically apply cluster and node pool upgrades with GitHub Actions][github-schedule]. +- Learn more about multiple node pools with [Create multiple node pools][use-multiple-node-pools]. +- Learn about upgrading best practices with [AKS patch and upgrade guidance][upgrade-operators-guide]. + + +[kubernetes-json-path]: https://kubernetes.io/docs/reference/kubectl/jsonpath/ + + +[upgrade-cluster]: upgrade-aks-cluster.md +[github-schedule]: node-upgrade-github-actions.md +[use-multiple-node-pools]: create-node-pools.md +[max-surge]: upgrade-aks-cluster.md#customize-node-surge-upgrade +[auto-upgrade-node-image]: auto-upgrade-node-image.md +[az-aks-nodepool-get-upgrades]: /cli/azure/aks/nodepool#az_aks_nodepool_get_upgrades +[az-aks-nodepool-show]: /cli/azure/aks/nodepool#az_aks_nodepool_show +[az-aks-nodepool-upgrade]: /cli/azure/aks/nodepool#az_aks_nodepool_upgrade +[az-aks-nodepool-update]: /cli/azure/aks/nodepool#az_aks_nodepool_update +[az-aks-upgrade]: /cli/azure/aks#az_aks_upgrade +[az-aks-show]: /cli/azure/aks#az_aks_show +[upgrade-operators-guide]: /azure/architecture/operator-guides/aks/aks-upgrade-practices \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md deleted file mode 100644 index 3c7b39cab..000000000 --- a/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: 'Quickstart: Create an Azure Container Instance with a public IP address using Terraform' -description: 'In this article, you create an Azure Container Instance with a public IP address using Terraform' -ms.topic: quickstart -ms.service: azure-container-instances -ms.date: 08/29/2024 -ms.custom: devx-track-terraform, linux-related-content -author: TomArcherMsft -ms.author: tarcher -content_well_notification: - - AI-contribution -ai-usage: ai-assisted ---- - -# Quickstart: Create an Azure Container Instance with a public IP address using Terraform - -Use Azure Container Instances to run serverless Docker containers in Azure with simplicity and speed. Deploy an application to a container instance on-demand when you don't need a full container orchestration platform like Azure Kubernetes Service. In this article, you use [Terraform](/azure/terraform) to deploy an isolated Docker container and make its web application available with a public IP address. - -[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] - -In this article, you learn how to: - -> [!div class="checklist"] -> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/resource_group/pet) -> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) -> * Create a random value for the container name using [random_string](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) -> * Create an Azure container group using [azurerm_container_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_group) - -## Prerequisites - -- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) - -## Implement the Terraform code - -> [!NOTE] -> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip/TestRecord.md). -> -> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) - -1. Create a directory in which to test and run the sample Terraform code and make it the current directory. - -1. Create a file named `main.tf` and insert the following code: - - [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/main.tf)] - -1. Create a file named `outputs.tf` and insert the following code: - - [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/outputs.tf)] - -1. Create a file named `providers.tf` and insert the following code: - - [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/providers.tf)] - -1. Create a file named `variables.tf` and insert the following code: - - [!code-terraform[master](~/terraform_samples/quickstart/101-aci-linuxcontainer-public-ip/variables.tf)] - -## Initialize Terraform - -[!INCLUDE [terraform-init.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-init.md)] - -## Create a Terraform execution plan - -[!INCLUDE [terraform-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan.md)] - -## Apply a Terraform execution plan - -[!INCLUDE [terraform-apply-plan.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-apply-plan.md)] - -## Verify the results - -1. When you apply the execution plan, Terraform outputs the public IP address. To display the IP address again, run [terraform output](https://developer.hashicorp.com/terraform/cli/commands/output). - - ```console - terraform output -raw container_ipv4_address - ``` - -1. Enter the sample's public IP address in your browser's address bar. - - :::image type="content" source="./media/container-instances-quickstart-terraform/azure-container-instances-demo.png" alt-text="Screenshot of the Azure Container Instances sample page"::: - -## Clean up resources - -[!INCLUDE [terraform-plan-destroy.md](~/azure-dev-docs-pr/articles/terraform/includes/terraform-plan-destroy.md)] - -## Troubleshoot Terraform on Azure - -[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) - -## Next steps - -> [!div class="nextstepaction"] -> [Tutorial: Create a container image for deployment to Azure Container Instances](./container-instances-tutorial-prepare-app.md) \ No newline at end of file diff --git a/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/container-instances-quickstart-terraform.md b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/container-instances-quickstart-terraform.md new file mode 100644 index 000000000..b66506601 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/container-instances-quickstart-terraform.md @@ -0,0 +1,253 @@ +--- +title: 'Quickstart: Create an Azure Container Instance with a public IP address using Terraform' +description: 'In this article, you create an Azure Container Instance with a public IP address using Terraform' +ms.topic: quickstart +ms.service: azure-container-instances +ms.date: 08/29/2024 +ms.custom: devx-track-terraform, linux-related-content, innovation-engine +author: TomArcherMsft +ms.author: tarcher +content_well_notification: + - AI-contribution +ai-usage: ai-assisted +--- + +# Quickstart: Create an Azure Container Instance with a public IP address using Terraform + +Use Azure Container Instances to run serverless Docker containers in Azure with simplicity and speed. Deploy an application to a container instance on-demand when you don't need a full container orchestration platform like Azure Kubernetes Service. In this article, you use [Terraform](/azure/terraform) to deploy an isolated Docker container and make its web application available with a public IP address. + +[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] + +In this article, you learn how to: + +> [!div class="checklist"] +> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/resource_group/pet) +> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) +> * Create a random value for the container name using [random_string](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) +> * Create an Azure container group using [azurerm_container_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_group) + +## Prerequisites + +- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) + +## Implement the Terraform code + +> [!NOTE] +> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip/TestRecord.md). +> +> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) + +1. Create a directory in which to test and run the sample Terraform code and make it the current directory. + +1. Create a file named main.tf and insert the following code: + +```text +resource "random_pet" "rg_name" { + prefix = var.resource_group_name_prefix +} + +resource "azurerm_resource_group" "rg" { + name = random_pet.rg_name.id + location = var.resource_group_location +} + +resource "random_string" "container_name" { + length = 25 + lower = true + upper = false + special = false +} + +resource "azurerm_container_group" "container" { + name = "${var.container_group_name_prefix}-${random_string.container_name.result}" + location = azurerm_resource_group.rg.location + resource_group_name = azurerm_resource_group.rg.name + ip_address_type = "Public" + os_type = "Linux" + restart_policy = var.restart_policy + + container { + name = "${var.container_name_prefix}-${random_string.container_name.result}" + image = var.image + cpu = var.cpu_cores + memory = var.memory_in_gb + + ports { + port = var.port + protocol = "TCP" + } + } +} +``` + +1. Create a file named outputs.tf and insert the following code: + +```text +output "container_ipv4_address" { + value = azurerm_container_group.container.ip_address +} +``` + +1. Create a file named providers.tf and insert the following code: + +```text +terraform { + required_version = ">=1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } + } +} +provider "azurerm" { + features {} +} +``` + +1. Create a file named variables.tf and insert the following code: + +```text +variable "resource_group_location" { + type = string + default = "eastus" + description = "Location for all resources." +} + +variable "resource_group_name_prefix" { + type = string + default = "rg" + description = "Prefix of the resource group name that's combined with a random value so name is unique in your Azure subscription." +} + +variable "container_group_name_prefix" { + type = string + description = "Prefix of the container group name that's combined with a random value so name is unique in your Azure subscription." + default = "acigroup" +} + +variable "container_name_prefix" { + type = string + description = "Prefix of the container name that's combined with a random value so name is unique in your Azure subscription." + default = "aci" +} + +variable "image" { + type = string + description = "Container image to deploy. Should be of the form repoName/imagename:tag for images stored in public Docker Hub, or a fully qualified URI for other registries. Images from private registries require additional registry credentials." + default = "mcr.microsoft.com/azuredocs/aci-helloworld" +} + +variable "port" { + type = number + description = "Port to open on the container and the public IP address." + default = 80 +} + +variable "cpu_cores" { + type = number + description = "The number of CPU cores to allocate to the container." + default = 1 +} + +variable "memory_in_gb" { + type = number + description = "The amount of memory to allocate to the container in gigabytes." + default = 2 +} + +variable "restart_policy" { + type = string + description = "The behavior of Azure runtime if container has stopped." + default = "Always" + validation { + condition = contains(["Always", "Never", "OnFailure"], var.restart_policy) + error_message = "The restart_policy must be one of the following: Always, Never, OnFailure." + } +} +``` + +## Initialize Terraform + +Before initializing Terraform, set the necessary environment variables. These variables are used by Terraform to provide default values for variables defined in the configuration files. + +```bash +export TF_VAR_resource_group_location="eastus" +export TF_VAR_resource_group_name_prefix="rg" +export TF_VAR_container_group_name_prefix="acigroup" +export TF_VAR_container_name_prefix="aci" +export TF_VAR_image="mcr.microsoft.com/azuredocs/aci-helloworld" +export TF_VAR_port=80 +export TF_VAR_cpu_cores=1 +export TF_VAR_memory_in_gb=2 +export TF_VAR_restart_policy="Always" +``` + +In this section, Terraform is initialized; this command downloads the Azure provider required to manage your Azure resources. Before running the command, ensure you are in the directory where you created the Terraform files. + +```bash +terraform init -upgrade +``` + +Key points: + +- The -upgrade parameter upgrades the necessary provider plugins to the newest version that complies with the configuration's version constraints. + +## Create a Terraform execution plan + +Run terraform plan to create an execution plan. + +```bash +terraform plan -out main.tfplan +``` + +Key points: + +- The terraform plan command creates an execution plan, but doesn't execute it. Instead, it determines what actions are necessary to create the configuration specified in your configuration files. This pattern allows you to verify whether the execution plan matches your expectations before making any changes to actual resources. +- The optional -out parameter allows you to specify an output file for the plan. Using the -out parameter ensures that the plan you reviewed is exactly what is applied. + +## Apply a Terraform execution plan + +Run terraform apply to execute the execution plan. + +```bash +terraform apply main.tfplan +``` + +Key points: + +- The example terraform apply command assumes you previously ran terraform plan -out main.tfplan. +- If you specified a different filename for the -out parameter, use that same filename in the call to terraform apply. +- If you didn't use the -out parameter, call terraform apply without any parameters. + +## Verify the results + +1. When you apply the execution plan, Terraform outputs the public IP address. To display the IP address again, run [terraform output](https://developer.hashicorp.com/terraform/cli/commands/output). + + ```bash + terraform output -raw container_ipv4_address + ``` + + +```text +"xxx.xxx.xxx.xxx" +``` + +2. Enter the sample's public IP address in your browser's address bar. + + :::image type="content" source="./media/container-instances-quickstart-terraform/azure-container-instances-demo.png" alt-text="Screenshot of the Azure Container Instances sample page" ::: + + + +## Troubleshoot Terraform on Azure + +[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) + +## Next steps + +> [!div class="nextstepaction"] +> [Tutorial: Create a container image for deployment to Azure Container Instances](./container-instances-tutorial-prepare-app.md) \ No newline at end of file diff --git a/tools/main.tf b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/main.tf similarity index 100% rename from tools/main.tf rename to scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/main.tf diff --git a/tools/outputs.tf b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/outputs.tf similarity index 100% rename from tools/outputs.tf rename to scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/outputs.tf diff --git a/tools/providers.tf b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/providers.tf similarity index 100% rename from tools/providers.tf rename to scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/providers.tf diff --git a/tools/variables.tf b/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/variables.tf similarity index 100% rename from tools/variables.tf rename to scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/variables.tf diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/main.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/main.tf similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/main.tf rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/main.tf diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/outputs.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/outputs.tf similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/outputs.tf rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/outputs.tf diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/providers.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/providers.tf similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/providers.tf rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/providers.tf diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/quick-create-terraform.md similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/quick-create-terraform.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/ssh.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/ssh.tf similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/ssh.tf rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/ssh.tf diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/variables.tf b/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/variables.tf similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/variables.tf rename to scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/variables.tf diff --git a/scenarios/azure-aks-docs/articles/aks/learn/aks-store-quickstart.yaml b/scenarios/azure-docs/articles/aks/learn/aks-store-quickstart.yaml similarity index 100% rename from scenarios/azure-aks-docs/articles/aks/learn/aks-store-quickstart.yaml rename to scenarios/azure-docs/articles/aks/learn/aks-store-quickstart.yaml diff --git a/scenarios/azure-aks-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md b/scenarios/azure-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md similarity index 100% rename from scenarios/azure-aks-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md rename to scenarios/azure-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md b/scenarios/azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md rename to scenarios/azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md b/scenarios/azure-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md rename to scenarios/azure-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md b/scenarios/azure-docs/articles/virtual-machines/linux/quick-create-cli.md similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md rename to scenarios/azure-docs/articles/virtual-machines/linux/quick-create-cli.md diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md b/scenarios/azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md similarity index 100% rename from scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md rename to scenarios/azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 3c75d45ed..05075a28d 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1,11 +1,11 @@ [ { "status": "active", - "key": "azure-aks-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md", + "key": "azure-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md", "title": "Deploy an Azure Kubernetes Service (AKS) cluster", "description": "Learn how to quickly deploy a Kubernetes cluster and deploy an application in Azure Kubernetes Service (AKS) using Azure CLI", "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/aks/learn/quick-kubernetes-deploy-cli.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/learn/quick-kubernetes-deploy-cli", "nextSteps": [ { @@ -159,11 +159,11 @@ }, { "status": "active", - "key": "azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", + "key": "azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", "title": "Create virtual machines in a Flexible scale set using Azure CLI", "description": "Learn how to create a Virtual Machine Scale Set in Flexible orchestration mode using Azure CLI.", "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/flexible-virtual-machine-scale-sets-cli", "nextSteps": [ { @@ -184,7 +184,7 @@ }, { "status": "active", - "key": "azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md", + "key": "azure-docs/articles/virtual-machines/linux/quick-create-cli.md", "title": "Deploy a Linux virtual machine", "description": "In this quickstart, you learn how to use the Azure CLI to create a Linux virtual machine", "stackDetails": [ @@ -193,7 +193,7 @@ "Network interface with public IP and network security group", "Port 22 will be opened" ], - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-cli.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/virtual-machines/linux/quick-create-cli.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/quick-create-cli", "nextSteps": [ { @@ -230,11 +230,11 @@ }, { "status": "active", - "key": "azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", + "key": "azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", "title": "Tutorial - Deploy a LEMP stack using WordPress on a VM", "description": "In this tutorial, you learn how to install the LEMP stack, and WordPress, on a Linux virtual machine in Azure.", "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/virtual-machines/linux/tutorial-lemp-stack.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-lemp-stack", "nextSteps": [ { @@ -482,10 +482,6 @@ "configurations": { } }, - { - "status": "inactive", - "key": "BlobVisionOnAKS/blob-vision-aks.md" - }, { "status": "inactive", "key": "DeployHAPGonARO/deploy-ha-pg-on-aro.md", @@ -660,7 +656,7 @@ "title": "Deploy an AI model on AKS with the AI toolchain operator", "description": "Learn how to enable the AI toolchain operator add-on on Azure Kubernetes Service (AKS) to simplify OSS AI model management and deployment", "stackDetails": "", - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/AKSKaito/aks-kaito.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/AksKaito/README.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/ai-toolchain-operator", "nextSteps": [ { @@ -783,11 +779,11 @@ }, { "status": "active", - "key": "azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md", + "key": "azure-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md", "title": "Tutorial - Use a custom VM image in a scale set with Azure CLI", "description": "Learn how to use the Azure CLI to create a custom VM image that you can use to deploy a Virtual Machine Scale Set", "stackDetails": [], - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/virtual-machine-scale-sets/tutorial-use-custom-image-cli.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-use-custom-image-cli", "nextSteps": [ { @@ -1172,12 +1168,12 @@ }, { "status": "active", - "key": "azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md", + "key": "azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/quick-create-terraform.md", "title": "Quickstart: Use Terraform to create a Linux VM", "description": "In this quickstart, you learn how to use Terraform to create a Linux virtual machine.", "stackDetails": [ ], - "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform.md", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/quick-create-terraform/quick-create-terraform.md", "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/quick-create-terraform?tabs=azure-cli", "nextSteps": [ { diff --git a/tools/abc.md b/tools/abc.md index b66506601..bb08cd960 100644 --- a/tools/abc.md +++ b/tools/abc.md @@ -1,253 +1,173 @@ --- -title: 'Quickstart: Create an Azure Container Instance with a public IP address using Terraform' -description: 'In this article, you create an Azure Container Instance with a public IP address using Terraform' -ms.topic: quickstart -ms.service: azure-container-instances -ms.date: 08/29/2024 -ms.custom: devx-track-terraform, linux-related-content, innovation-engine -author: TomArcherMsft -ms.author: tarcher -content_well_notification: - - AI-contribution -ai-usage: ai-assisted +title: Upgrade Azure Kubernetes Service (AKS) node images +description: Learn how to upgrade the images on AKS cluster nodes and node pools. +ms.topic: how-to +ms.custom: devx-track-azurecli, innovation-engine +ms.subservice: aks-upgrade +ms.service: azure-kubernetes-service +ms.date: 09/20/2024 +author: schaffererin +ms.author: schaffererin --- -# Quickstart: Create an Azure Container Instance with a public IP address using Terraform +## Environment Variables -Use Azure Container Instances to run serverless Docker containers in Azure with simplicity and speed. Deploy an application to a container instance on-demand when you don't need a full container orchestration platform like Azure Kubernetes Service. In this article, you use [Terraform](/azure/terraform) to deploy an isolated Docker container and make its web application available with a public IP address. +The following environment variables are declared and will be used in subsequent code blocks. They replace the placeholder parameters in the original document with standardized variable names. -[!INCLUDE [Terraform abstract](~/azure-dev-docs-pr/articles/terraform/includes/abstract.md)] - -In this article, you learn how to: - -> [!div class="checklist"] -> * Create a random value for the Azure resource group name using [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/resource_group/pet) -> * Create an Azure resource group using [azurerm_resource_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) -> * Create a random value for the container name using [random_string](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) -> * Create an Azure container group using [azurerm_container_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_group) +```bash +export AKS_NODEPOOL="nodepool1" +export AKS_CLUSTER="apache-airflow-aks" +export AKS_RESOURCE_GROUP="apache-airflow-rg" +``` -## Prerequisites +# Upgrade Azure Kubernetes Service (AKS) node images -- [Install and configure Terraform](/azure/developer/terraform/quickstart-configure) +Azure Kubernetes Service (AKS) regularly provides new node images, so it's beneficial to upgrade your node images frequently to use the latest AKS features. Linux node images are updated weekly, and Windows node images are updated monthly. Image upgrade announcements are included in the [AKS release notes](https://github.com/Azure/AKS/releases), and it can take up to a week for these updates to be rolled out across all regions. You can also perform node image upgrades automatically and schedule them using planned maintenance. For more information, see [Automatically upgrade node images][auto-upgrade-node-image]. -## Implement the Terraform code +This article shows you how to upgrade AKS cluster node images and how to update node pool images without upgrading the Kubernetes version. For information on upgrading the Kubernetes version for your cluster, see [Upgrade an AKS cluster][upgrade-cluster]. > [!NOTE] -> The sample code for this article is located in the [Azure Terraform GitHub repo](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip). You can view the log file containing the [test results from current and previous versions of Terraform](https://github.com/Azure/terraform/tree/master/quickstart/101-aci-linuxcontainer-public-ip/TestRecord.md). -> -> See more [articles and sample code showing how to use Terraform to manage Azure resources](/azure/terraform) - -1. Create a directory in which to test and run the sample Terraform code and make it the current directory. - -1. Create a file named main.tf and insert the following code: - -```text -resource "random_pet" "rg_name" { - prefix = var.resource_group_name_prefix -} - -resource "azurerm_resource_group" "rg" { - name = random_pet.rg_name.id - location = var.resource_group_location -} - -resource "random_string" "container_name" { - length = 25 - lower = true - upper = false - special = false -} - -resource "azurerm_container_group" "container" { - name = "${var.container_group_name_prefix}-${random_string.container_name.result}" - location = azurerm_resource_group.rg.location - resource_group_name = azurerm_resource_group.rg.name - ip_address_type = "Public" - os_type = "Linux" - restart_policy = var.restart_policy - - container { - name = "${var.container_name_prefix}-${random_string.container_name.result}" - image = var.image - cpu = var.cpu_cores - memory = var.memory_in_gb - - ports { - port = var.port - protocol = "TCP" - } - } -} -``` +> The AKS cluster must use virtual machine scale sets for the nodes. +> +> It's not possible to downgrade a node image version (for example *AKSUbuntu-2204 to AKSUbuntu-1804*, or *AKSUbuntu-2204-202308.01.0 to AKSUbuntu-2204-202307.27.0*). -1. Create a file named outputs.tf and insert the following code: +## Check for available node image upgrades -```text -output "container_ipv4_address" { - value = azurerm_container_group.container.ip_address -} -``` +1. Check for available node image upgrades using the [`az aks nodepool get-upgrades`][az-aks-nodepool-get-upgrades] command. -1. Create a file named providers.tf and insert the following code: - -```text -terraform { - required_version = ">=1.0" - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~>3.0" - } - random = { - source = "hashicorp/random" - version = "~>3.0" - } - } -} -provider "azurerm" { - features {} -} -``` + ```azurecli-interactive + az aks nodepool get-upgrades \ + --nodepool-name $AKS_NODEPOOL \ + --cluster-name $AKS_CLUSTER \ + --resource-group $AKS_RESOURCE_GROUP + ``` -1. Create a file named variables.tf and insert the following code: - -```text -variable "resource_group_location" { - type = string - default = "eastus" - description = "Location for all resources." -} - -variable "resource_group_name_prefix" { - type = string - default = "rg" - description = "Prefix of the resource group name that's combined with a random value so name is unique in your Azure subscription." -} - -variable "container_group_name_prefix" { - type = string - description = "Prefix of the container group name that's combined with a random value so name is unique in your Azure subscription." - default = "acigroup" -} - -variable "container_name_prefix" { - type = string - description = "Prefix of the container name that's combined with a random value so name is unique in your Azure subscription." - default = "aci" -} - -variable "image" { - type = string - description = "Container image to deploy. Should be of the form repoName/imagename:tag for images stored in public Docker Hub, or a fully qualified URI for other registries. Images from private registries require additional registry credentials." - default = "mcr.microsoft.com/azuredocs/aci-helloworld" -} - -variable "port" { - type = number - description = "Port to open on the container and the public IP address." - default = 80 -} - -variable "cpu_cores" { - type = number - description = "The number of CPU cores to allocate to the container." - default = 1 -} - -variable "memory_in_gb" { - type = number - description = "The amount of memory to allocate to the container in gigabytes." - default = 2 -} - -variable "restart_policy" { - type = string - description = "The behavior of Azure runtime if container has stopped." - default = "Always" - validation { - condition = contains(["Always", "Never", "OnFailure"], var.restart_policy) - error_message = "The restart_policy must be one of the following: Always, Never, OnFailure." - } -} -``` +1. In the output, find and make note of the `latestNodeImageVersion` value. This value is the latest node image version available for your node pool. +1. Check your current node image version to compare with the latest version using the [`az aks nodepool show`][az-aks-nodepool-show] command. -## Initialize Terraform + ```azurecli-interactive + az aks nodepool show \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL \ + --query nodeImageVersion + ``` -Before initializing Terraform, set the necessary environment variables. These variables are used by Terraform to provide default values for variables defined in the configuration files. +1. If the `nodeImageVersion` value is different from the `latestNodeImageVersion`, you can upgrade your node image. -```bash -export TF_VAR_resource_group_location="eastus" -export TF_VAR_resource_group_name_prefix="rg" -export TF_VAR_container_group_name_prefix="acigroup" -export TF_VAR_container_name_prefix="aci" -export TF_VAR_image="mcr.microsoft.com/azuredocs/aci-helloworld" -export TF_VAR_port=80 -export TF_VAR_cpu_cores=1 -export TF_VAR_memory_in_gb=2 -export TF_VAR_restart_policy="Always" -``` +## Upgrade all node images in all node pools -In this section, Terraform is initialized; this command downloads the Azure provider required to manage your Azure resources. Before running the command, ensure you are in the directory where you created the Terraform files. +1. Upgrade all node images in all node pools in your cluster using the [`az aks upgrade`][az-aks-upgrade] command with the `--node-image-only` flag. -```bash -terraform init -upgrade -``` + ```text + az aks upgrade \ + --resource-group $AKS_RESOURCE_GROUP \ + --name $AKS_CLUSTER \ + --node-image-only \ + --yes + ``` -Key points: +1. You can check the status of the node images using the `kubectl get nodes` command. -- The -upgrade parameter upgrades the necessary provider plugins to the newest version that complies with the configuration's version constraints. + > [!NOTE] + > This command might differ slightly depending on the shell you use. For more information on Windows and PowerShell environments, see the [Kubernetes JSONPath documentation][kubernetes-json-path]. -## Create a Terraform execution plan + ```bash + kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' + ``` -Run terraform plan to create an execution plan. +1. When the upgrade completes, use the [`az aks show`][az-aks-show] command to get the updated node pool details. The current node image is shown in the `nodeImageVersion` property. -```bash -terraform plan -out main.tfplan -``` + ```azurecli-interactive + az aks show \ + --resource-group $AKS_RESOURCE_GROUP \ + --name $AKS_CLUSTER + ``` -Key points: +## Upgrade a specific node pool -- The terraform plan command creates an execution plan, but doesn't execute it. Instead, it determines what actions are necessary to create the configuration specified in your configuration files. This pattern allows you to verify whether the execution plan matches your expectations before making any changes to actual resources. -- The optional -out parameter allows you to specify an output file for the plan. Using the -out parameter ensures that the plan you reviewed is exactly what is applied. +1. Update the OS image of a node pool without doing a Kubernetes cluster upgrade using the [`az aks nodepool upgrade`][az-aks-nodepool-upgrade] command with the `--node-image-only` flag. -## Apply a Terraform execution plan + ```azurecli-interactive + az aks nodepool upgrade \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL \ + --node-image-only + ``` -Run terraform apply to execute the execution plan. +1. You can check the status of the node images with the `kubectl get nodes` command. -```bash -terraform apply main.tfplan -``` + > [!NOTE] + > This command may differ slightly depending on the shell you use. For more information on Windows and PowerShell environments, see the [Kubernetes JSONPath documentation][kubernetes-json-path]. -Key points: + ```bash + kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' + ``` -- The example terraform apply command assumes you previously ran terraform plan -out main.tfplan. -- If you specified a different filename for the -out parameter, use that same filename in the call to terraform apply. -- If you didn't use the -out parameter, call terraform apply without any parameters. +1. When the upgrade completes, use the [`az aks nodepool show`][az-aks-nodepool-show] command to get the updated node pool details. The current node image is shown in the `nodeImageVersion` property. -## Verify the results + ```azurecli-interactive + az aks nodepool show \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL + ``` -1. When you apply the execution plan, Terraform outputs the public IP address. To display the IP address again, run [terraform output](https://developer.hashicorp.com/terraform/cli/commands/output). +## Upgrade node images with node surge - ```bash - terraform output -raw container_ipv4_address - ``` +To speed up the node image upgrade process, you can upgrade your node images using a customizable node surge value. By default, AKS uses one extra node to configure upgrades. - -```text -"xxx.xxx.xxx.xxx" -``` +1. Upgrade node images with node surge using the [`az aks nodepool update`][az-aks-nodepool-update] command with the `--max-surge` flag to configure the number of nodes used for upgrades. -2. Enter the sample's public IP address in your browser's address bar. + > [!NOTE] + > To learn more about the trade-offs of various `--max-surge` settings, see [Customize node surge upgrade][max-surge]. - :::image type="content" source="./media/container-instances-quickstart-terraform/azure-container-instances-demo.png" alt-text="Screenshot of the Azure Container Instances sample page" ::: + ```azurecli-interactive + az aks nodepool update \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL \ + --max-surge 33% \ + --no-wait + ``` +1. You can check the status of the node images with the `kubectl get nodes` command. + ```bash + kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' + ``` -## Troubleshoot Terraform on Azure +1. Get the updated node pool details using the [`az aks nodepool show`][az-aks-nodepool-show] command. The current node image is shown in the `nodeImageVersion` property. -[Troubleshoot common problems when using Terraform on Azure](/azure/developer/terraform/troubleshoot) + ```azurecli-interactive + az aks nodepool show \ + --resource-group $AKS_RESOURCE_GROUP \ + --cluster-name $AKS_CLUSTER \ + --name $AKS_NODEPOOL + ``` ## Next steps -> [!div class="nextstepaction"] -> [Tutorial: Create a container image for deployment to Azure Container Instances](./container-instances-tutorial-prepare-app.md) \ No newline at end of file +- For information about the latest node images, see the [AKS release notes](https://github.com/Azure/AKS/releases). +- Learn how to upgrade the Kubernetes version with [Upgrade an AKS cluster][upgrade-cluster]. +- [Automatically apply cluster and node pool upgrades with GitHub Actions][github-schedule]. +- Learn more about multiple node pools with [Create multiple node pools][use-multiple-node-pools]. +- Learn about upgrading best practices with [AKS patch and upgrade guidance][upgrade-operators-guide]. + + +[kubernetes-json-path]: https://kubernetes.io/docs/reference/kubectl/jsonpath/ + + +[upgrade-cluster]: upgrade-aks-cluster.md +[github-schedule]: node-upgrade-github-actions.md +[use-multiple-node-pools]: create-node-pools.md +[max-surge]: upgrade-aks-cluster.md#customize-node-surge-upgrade +[auto-upgrade-node-image]: auto-upgrade-node-image.md +[az-aks-nodepool-get-upgrades]: /cli/azure/aks/nodepool#az_aks_nodepool_get_upgrades +[az-aks-nodepool-show]: /cli/azure/aks/nodepool#az_aks_nodepool_show +[az-aks-nodepool-upgrade]: /cli/azure/aks/nodepool#az_aks_nodepool_upgrade +[az-aks-nodepool-update]: /cli/azure/aks/nodepool#az_aks_nodepool_update +[az-aks-upgrade]: /cli/azure/aks#az_aks_upgrade +[az-aks-show]: /cli/azure/aks#az_aks_show +[upgrade-operators-guide]: /azure/architecture/operator-guides/aks/aks-upgrade-practices \ No newline at end of file diff --git a/tools/main.yml b/tools/main.yml deleted file mode 100644 index e931e54c4..000000000 --- a/tools/main.yml +++ /dev/null @@ -1,64 +0,0 @@ -- name: Create Azure VM - hosts: localhost - connection: local - tasks: - - name: Create resource group - azure_rm_resourcegroup: - name: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" - location: "{{ lookup('env', 'REGION') }}" - - name: Create virtual network - azure_rm_virtualnetwork: - resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" - name: "{{ lookup('env', 'MY_VNET_NAME') }}" - address_prefixes: "10.0.0.0/16" - - name: Add subnet - azure_rm_subnet: - resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" - name: "{{ lookup('env', 'MY_SUBNET_NAME') }}" - address_prefix: "10.0.1.0/24" - virtual_network: "{{ lookup('env', 'MY_VNET_NAME') }}" - - name: Create public IP address - azure_rm_publicipaddress: - resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" - allocation_method: Static - name: "{{ lookup('env', 'MY_PUBLIC_IP_NAME') }}" - register: output_ip_address - - name: Public IP of VM - debug: - msg: "The public IP is {{ output_ip_address.state.ip_address }}." - - name: Create Network Security Group that allows SSH - azure_rm_securitygroup: - resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" - name: "{{ lookup('env', 'MY_NSG_NAME') }}" - rules: - - name: SSH - protocol: Tcp - destination_port_range: 22 - access: Allow - priority: 1001 - direction: Inbound - - name: Create virtual network interface card - azure_rm_networkinterface: - resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" - name: "{{ lookup('env', 'MY_NIC_NAME') }}" - virtual_network: "{{ lookup('env', 'MY_VNET_NAME') }}" - subnet_name: "{{ lookup('env', 'MY_SUBNET_NAME') }}" - security_group: "{{ lookup('env', 'MY_NSG_NAME') }}" - ip_configurations: - - name: ipconfig1 - public_ip_address_name: "{{ lookup('env', 'MY_PUBLIC_IP_NAME') }}" - primary: yes - - name: Create VM - azure_rm_virtualmachine: - resource_group: "{{ lookup('env', 'MY_RESOURCE_GROUP') }}" - name: "{{ lookup('env', 'MY_VM_NAME') }}" - vm_size: Standard_DS1_v2 - admin_username: azureuser - ssh_password_enabled: false - generate_ssh_keys: yes # This will automatically generate keys if they don't exist - network_interfaces: "{{ lookup('env', 'MY_NIC_NAME') }}" - image: - offer: 0001-com-ubuntu-server-jammy - publisher: Canonical - sku: 22_04-lts - version: latest From 30d720535c2cdaad780e59d23fedcd364f122e10 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Fri, 28 Mar 2025 13:09:31 -0700 Subject: [PATCH 50/87] added a bunch of new converted docs and updated their metadata to be still compliant with prod metadata --- .../tutorial-modify-scale-sets-cli.md | 13 +- .../tutorial-azure-linux-add-nodepool.md | 2 - .../tutorial-azure-linux-create-cluster.md | 5 +- .../tutorial-azure-linux-telemetry-monitor.md | 9 - scenarios/metadata.json | 441 ++++++++++++++++++ 5 files changed, 447 insertions(+), 23 deletions(-) diff --git a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md index 6f0bd2656..94c5a5c89 100644 --- a/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md +++ b/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md @@ -15,18 +15,13 @@ Throughout the lifecycle of your applications, you may need to modify or update Below, we declare environment variables that will be used throughout this document. A random suffix is appended to resource names that need to be unique for each deployment. The `REGION` is set to *WestUS2*. -```bash -export RANDOM_SUFFIX=adcc95 -export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" -export SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" -export NEW_INSTANCE_NAME="myNewInstance$RANDOM_SUFFIX" -export REGION="WestUS2" -``` - ## Setup Resource Group Before proceeding, ensure the resource group exists. This step creates the resource group if it does not already exist. ```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export MY_RESOURCE_GROUP_NAME="myResourceGroup$RANDOM_SUFFIX" +export REGION="WestUS2" az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION ``` @@ -49,6 +44,7 @@ az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION To ensure that subsequent update and query commands have a valid resource to work on, create a Virtual Machine Scale Set. In this step, we deploy a basic scale set using a valid image (*Ubuntu2204*) and set the instance count to 5 so that instance-specific updates can target an existing instance ID. ```azurecli-interactive +export SCALE_SET_NAME="myScaleSet$RANDOM_SUFFIX" az vmss create \ --resource-group $MY_RESOURCE_GROUP_NAME \ --name $SCALE_SET_NAME \ @@ -319,6 +315,7 @@ Running [az vm show](/cli/azure/vm#az-vm-show) again, we now will see that the V There are times where you might want to add a new VM to your scale set but want different configuration options than those listed in the scale set model. VMs can be added to a scale set during creation by using the [az vm create](/cli/azure/vmss#az-vmss-create) command and specifying the scale set name you want the instance added to. ```azurecli-interactive +export NEW_INSTANCE_NAME="myNewInstance$RANDOM_SUFFIX" az vm create --name $NEW_INSTANCE_NAME --resource-group $MY_RESOURCE_GROUP_NAME --vmss $SCALE_SET_NAME --image RHELRaw8LVMGen2 ``` diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md index 91ed7b58e..f88c2f19b 100644 --- a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md @@ -64,8 +64,6 @@ az aks nodepool add \ To see the status of your node pools, use the `az aks nodepool list` command and specify your resource group and cluster name. The same environment variable values declared earlier are used here. ```azurecli-interactive -export CLUSTER_NAME="myAKSClusterabcf37" -export RESOURCE_GROUP="myAKSResourceGroupabcf37" az aks nodepool list --resource-group $RESOURCE_GROUP --cluster-name $CLUSTER_NAME ``` diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md index e4bb92c5c..c9254eacf 100644 --- a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md @@ -38,8 +38,6 @@ When creating a resource group, it is required to specify a location. This locat Before running the command, environment variables are declared to ensure unique resource names for each deployment. ```bash -export RANDOM_SUFFIX=c97736 -export RESOURCE_GROUP_NAME="testAzureLinuxResourceGroup${RANDOM_SUFFIX}" export REGION="EastUS2" az group create --name $RESOURCE_GROUP_NAME --location $REGION ``` @@ -64,10 +62,9 @@ az group create --name $RESOURCE_GROUP_NAME --location $REGION ## Create an Azure Linux Container Host cluster -Create an AKS cluster using the `az aks create` command with the `--os-sku` parameter to provision the Azure Linux Container Host with an Azure Linux image. The following example creates an Azure Linux Container Host cluster. A unique cluster name is generated using the same RANDOM_SUFFIX used when creating the resource group. +Create an AKS cluster using the `az aks create` command with the `--os-sku` parameter to provision the Azure Linux Container Host with an Azure Linux image. The following example creates an Azure Linux Container Host cluster. ```bash -export CLUSTER_NAME="testAzureLinuxCluster${RANDOM_SUFFIX}" az aks create --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP_NAME --os-sku AzureLinux ``` diff --git a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md index 272c48050..926da4616 100644 --- a/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md +++ b/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md @@ -26,15 +26,6 @@ In the next and last tutorial, you'll learn how to upgrade your Azure Linux node - If you're connecting an existing AKS cluster to a Log Analytics workspace in another subscription, the Microsoft.ContainerService resource provider must be registered in the subscription with the Log Analytics workspace. For more information, see [Register resource provider](/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider). - You need the latest version of Azure CLI. Run `az --version` to find the version. If you need to install or upgrade, see [Install Azure CLI](/cli/azure/install-azure-cli). -## Environment Variables - -To ensure unique resource names for each deployment, we declare the following environment variables. These variables will be used throughout the tutorial. - -```bash -export RESOURCE_GROUP="myAKSResourceGroupabcf37" -export CLUSTER_NAME="myAKSClusterabcf37" -``` - ## Enable monitoring ## Connect to your cluster diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 05075a28d..f66e7bc40 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1212,5 +1212,446 @@ "documentationUrl": "https://www.flatcar.org/docs/latest/installing/cloud/azure/", "configurations": { } + }, + { + "status": "active", + "key": "azure-management-docs/articles/azure-linux/tutorial-azure-linux-migration.md", + "title": "Azure Linux Container Host for AKS tutorial - Migrating to Azure Linux", + "description": "In this Azure Linux Container Host for AKS tutorial, you learn how to migrate your nodes to Azure Linux nodes.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-migration.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/azure-linux/tutorial-azure-linux-migration?tabs=azure-cli", + "nextSteps": [ + { + "title": "Enable telemetry and monitoring", + "url": "https://github.com/MicrosoftDocs/azure-management-docs/blob/main/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "RESOURCE_GROUP", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "CLUSTER_NAME", + "title": "AKS Cluster Name", + "defaultValue": "" + } + ] + } + }, + { + "status": "active", + "key": "azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md", + "title": "Azure Linux Container Host for AKS tutorial - Create a cluster", + "description": "In this Azure Linux Container Host for AKS tutorial, you will learn how to create an AKS cluster with Azure Linux.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-create-cluster.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/azure-linux/tutorial-azure-linux-create-cluster", + "nextSteps": [ + { + "title": "Add an Azure Linux node pool", + "url": "https://learn.microsoft.com/en-us/azure/azure-linux/tutorial-azure-linux-add-nodepool" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "RESOURCE_GROUP_NAME", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "CLUSTER_NAME", + "title": "AKS Cluster Name", + "defaultValue": "" + } + ] + } + }, + { + "status": "active", + "key": "azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md", + "title": "Azure Linux Container Host for AKS tutorial - Add an Azure Linux node pool to your existing AKS cluster", + "description": "In this Azure Linux Container Host for AKS tutorial, you learn how to add an Azure Linux node pool to your existing cluster.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-add-nodepool.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/azure-linux/tutorial-azure-linux-add-nodepool", + "nextSteps": [ + { + "title": "Migrating to Azure Linux", + "url": "https://learn.microsoft.com/en-us/azure/azure-linux/tutorial-azure-linux-migration" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "RESOURCE_GROUP", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "CLUSTER_NAME", + "title": "AKS Cluster Name", + "defaultValue": "" + } + ] + } + }, + { + "status": "active", + "key": "azure-management-docs/articles/azure-linux/tutorial-azure-linux-upgrade.md", + "title": "Azure Linux Container Host for AKS tutorial - Upgrade Azure Linux Container Host nodes", + "description": "In this Azure Linux Container Host for AKS tutorial, you learn how to upgrade Azure Linux Container Host nodes.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-upgrade.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/azure-linux/tutorial-azure-linux-upgrade", + "nextSteps": [ + { + "title": "Azure Linux Container Host Overview", + "url": "https://learn.microsoft.com/en-us/azure/azure-linux/intro-azure-linux" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "AZ_LINUX_RG", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "AZ_LINUX_CLUSTER", + "title": "AKS Cluster Name", + "defaultValue": "" + } + ] + } + }, + { + "status": "active", + "key": "azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md", + "title": "Azure Linux Container Host for AKS tutorial - Enable telemetry and monitoring for the Azure Linux Container Host", + "description": "In this Azure Linux Container Host for AKS tutorial, you'll learn how to enable telemetry and monitoring for the Azure Linux Container Host.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-management-docs/articles/azure-linux/tutorial-azure-linux-telemetry-monitor.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/azure-linux/tutorial-azure-linux-telemetry-monitor", + "nextSteps": [ + { + "title": "Upgrade Azure Linux Nodes", + "url": "https://github.com/MicrosoftDocs/azure-management-docs/blob/main/articles/azure-linux/tutorial-azure-linux-upgrade.md" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "RESOURCE_GROUP", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "CLUSTER_NAME", + "title": "AKS Cluster Name", + "defaultValue": "" + } + ] + } + }, + { + "status": "active", + "key": "azure-stack-docs/azure-stack/user/azure-stack-quick-create-vm-linux-cli.md", + "title": "Create Linux VM with Azure CLI in Azure Stack Hub", + "description": "Create a Linux virtual machine by using the Azure CLI in Azure Stack Hub.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-stack-docs/azure-stack/user/azure-stack-quick-create-vm-linux-cli.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure-stack/user/azure-stack-quick-create-vm-linux-cli?view=azs-2501", + "nextSteps": [ + { + "title": "Considerations for virtual machines in Azure Stack Hub", + "url": "https://github.com/MicrosoftDocs/azure-stack-docs/blob/main/azure-stack/user/azure-stack-vm-considerations.md" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-aks-docs/articles/aks/azure-cni-powered-by-cilium.md", + "title": "Configure Azure CNI Powered by Cilium in Azure Kubernetes Service (AKS)", + "description": "Learn how to create an Azure Kubernetes Service (AKS) cluster with Azure CNI Powered by Cilium.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/azure-cni-powered-by-cilium.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/azure-cni-powered-by-cilium", + "nextSteps": [ + { + "title": "Upgrade Azure CNI IPAM modes and Dataplane Technology.", + "url": "https://learn.microsoft.com/en-us/azure/aks/upgrade-azure-cni" + }, + { + "title": "Use a static IP address with the Azure Kubernetes Service (AKS) load balancer", + "url": "https://learn.microsoft.com/en-us/azure/aks/static-ip" + }, + { + "title": "Use an internal load balancer with Azure Container Service (AKS)", + "url": "https://learn.microsoft.com/en-us/azure/aks/internal-lb" + }, + { + "title": "Create a basic ingress controller with external network connectivity", + "url": "https://learn.microsoft.com/en-us/azure/aks/ingress-basic" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/tutorial-automate-vm-deployment.md", + "title": "Tutorial - Customize a Linux VM with cloud-init in Azure", + "description": "In this tutorial, you learn how to use cloud-init and Key Vault to customize Linux VMs the first time they boot in Azure", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-automate-vm-deployment.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-automate-vm-deployment", + "nextSteps": [ + { + "title": "Create custom VM images", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-custom-images" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md", + "title": "Create a Linux VM in Azure with multiple NICs", + "description": "Learn how to create a Linux VM with multiple NICs attached to it using the Azure CLI or Resource Manager templates.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/multiple-nics.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/multiple-nics", + "nextSteps": [ + { + "title": "Review Linux VM Sizes", + "url": "https://github.com/MicrosoftDocs/azure-compute-docs/blob/main/articles/virtual-machines/sizes.md" + }, + { + "title": " Manage virtual machine access using just in time", + "url": "https://github.com/MicrosoftDocs/azure-compute-docs/blob/main/azure/security-center/security-center-just-in-time" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/disks-enable-performance.md", + "title": "Preview - Increase performance of Premium SSDs and Standard SSD/HDDs", + "description": "Increase the performance of Azure Premium SSDs and Standard SSD/HDDs using performance plus.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/disks-enable-performance.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/disks-enable-performance?tabs=azure-cli", + "nextSteps": [ + { + "title": "Create an incremental snapshot for managed disks", + "url": "https://github.com/MicrosoftDocs/azure-compute-docs/blob/main/articles/virtual-machines/disks-incremental-snapshots.md" + }, + { + "title": "Expand virtual hard disks on a Linux VM", + "url": "https://github.com/MicrosoftDocs/azure-compute-docs/blob/main/articles/virtual-machines/linux/expand-disks.md" + }, + { + "title": "How to expand virtual hard disks attached to a Windows virtual machine", + "url": "https://github.com/MicrosoftDocs/azure-compute-docs/blob/main/articles/virtual-machines/windows/expand-os-disk.md" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md", + "title": "Modify an Azure Virtual Machine Scale Set using Azure CLI", + "description": "Learn how to modify and update an Azure Virtual Machine Scale Set using Azure CLI.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-modify-scale-sets-cli", + "nextSteps": [ + { + "title": "Use data disks with scale sets", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-use-disks-powershell" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md", + "title": "Tutorial - Autoscale a scale set with the Azure CLI", + "description": "Learn how to use the Azure CLI to automatically scale a Virtual Machine Scale Set as CPU demands increases and decreases", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machine-scale-sets/tutorial-autoscale-cli.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/tutorial-autoscale-cli", + "nextSteps": [ + { + "title": "Learn about scale set instance protection", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-instance-protection" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md", + "title": "Tutorial - Create and manage Linux VMs with the Azure CLI", + "description": "In this tutorial, you learn how to use the Azure CLI to create and manage Linux VMs in Azure", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-manage-vm.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-manage-vm", + "nextSteps": [ + { + "title": "Create and Manage VM Disks", + "url": "https://github.com/MicrosoftDocs/azure-compute-docs/blob/main/articles/virtual-machines/linux/tutorial-manage-disks.md" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/tutorial-lamp-stack.md", + "title": "Tutorial - Deploy LAMP and WordPress on a VM", + "description": "In this tutorial, you learn how to install the LAMP stack, and WordPress, on a Linux virtual machine in Azure.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-lamp-stack.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-lamp-stack", + "nextSteps": [ + { + "title": "Secure web server with TLS", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-secure-web-server" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-docs/articles/batch/quick-create-cli.md", + "title": "Quickstart: Use the Azure CLI to create a Batch account and run a job", + "description": "Follow this quickstart to use the Azure CLI to create a Batch account, a pool of compute nodes, and a job that runs basic tasks on the pool.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/batch/quick-create-cli.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/batch/quick-create-cli", + "nextSteps": [ + { + "title": "Tutorial: Run a parallel workload with Azure Batch", + "url": "https://learn.microsoft.com/en-us/azure/batch/tutorial-parallel-python" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/container-instances-quickstart-terraform.md", + "title": "Quickstart: Create an Azure Container Instance with a public IP address using Terraform", + "description": "In this article, you create an Azure Container Instance with a public IP address using Terraform", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/container-instances/container-instances-quickstart-terraform/container-instances-quickstart-terraform.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/container-instances/container-instances-quickstart-terraform", + "nextSteps": [ + { + "title": "Tutorial: Create a container image for deployment to Azure Container Instances", + "url": "https://learn.microsoft.com/en-us/azure/container-instances/container-instances-tutorial-prepare-app" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-aks-docs/articles/aks/node-image-upgrade.md", + "title": "Upgrade Azure Kubernetes Service (AKS) node images", + "description": "Learn how to upgrade the images on AKS cluster nodes and node pools.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/aks/node-image-upgrade", + "nextSteps": [ + { + "title": "For information about the latest node images, see the AKS release notes", + "url": "https://github.com/Azure/AKS/releases" + }, + { + "title": "Learn how to upgrade the Kubernetes version with Upgrade an AKS cluster", + "url": "https://learn.microsoft.com/en-us/azure/aks/upgrade-aks-cluster" + }, + { + "title": "Automatically apply cluster and node pool upgrades with GitHub Actions", + "url": "https://learn.microsoft.com/en-us/azure/aks/node-upgrade-github-actions" + }, + { + "title": "Learn more about multiple node pools with Create multiple node pools", + "url": "https://learn.microsoft.com/en-us/azure/aks/create-node-pools" + }, + { + "title": "Learn about upgrading best practices with AKS patch and upgrade guidance", + "url": "https://learn.microsoft.com/en-us/azure/architecture/operator-guides/aks/aks-upgrade-practices" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [ + { + "inputType": "textInput", + "commandKey": "AKS_RESOURCE_GROUP", + "title": "Resource Group Name", + "defaultValue": "" + }, + { + "inputType": "textInput", + "commandKey": "AKS_CLUSTER", + "title": "AKS Cluster Name", + "defaultValue": "" + } + ] + } } ] \ No newline at end of file From 66135026c54057f8ceb13591e5085bcd4bfc33d1 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Sat, 29 Mar 2025 01:13:28 -0700 Subject: [PATCH 51/87] updated docs --- .../articles/aks/node-image-upgrade.md | 10 - .../linux/tutorial-elasticsearch.md | 304 +++++++++++++ scenarios/metadata.json | 19 + .../quickstart-install-connect-docker.md | 8 +- tools/abc.md | 378 +++++++++------ tools/abc_converted.md | 429 ++++++++++++++++++ tools/app.py | 3 + tools/cloud-init.txt | 15 + tools/def.md | 222 +++++++++ tools/ghi.md | 264 +++++++++++ tools/python-docs-hello-django | 1 + tools/python-docs-hello-world | 1 + 12 files changed, 1510 insertions(+), 144 deletions(-) create mode 100644 scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-elasticsearch.md create mode 100644 tools/abc_converted.md create mode 100644 tools/app.py create mode 100644 tools/cloud-init.txt create mode 100644 tools/def.md create mode 100644 tools/ghi.md create mode 160000 tools/python-docs-hello-django create mode 160000 tools/python-docs-hello-world diff --git a/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md b/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md index bb08cd960..5fc7dd41d 100644 --- a/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md +++ b/scenarios/azure-aks-docs/articles/aks/node-image-upgrade.md @@ -10,16 +10,6 @@ author: schaffererin ms.author: schaffererin --- -## Environment Variables - -The following environment variables are declared and will be used in subsequent code blocks. They replace the placeholder parameters in the original document with standardized variable names. - -```bash -export AKS_NODEPOOL="nodepool1" -export AKS_CLUSTER="apache-airflow-aks" -export AKS_RESOURCE_GROUP="apache-airflow-rg" -``` - # Upgrade Azure Kubernetes Service (AKS) node images Azure Kubernetes Service (AKS) regularly provides new node images, so it's beneficial to upgrade your node images frequently to use the latest AKS features. Linux node images are updated weekly, and Windows node images are updated monthly. Image upgrade announcements are included in the [AKS release notes](https://github.com/Azure/AKS/releases), and it can take up to a week for these updates to be rolled out across all regions. You can also perform node image upgrades automatically and schedule them using planned maintenance. For more information, see [Automatically upgrade node images][auto-upgrade-node-image]. diff --git a/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-elasticsearch.md b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-elasticsearch.md new file mode 100644 index 000000000..1bcd70639 --- /dev/null +++ b/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-elasticsearch.md @@ -0,0 +1,304 @@ +--- +title: Deploy ElasticSearch on a development virtual machine in Azure +description: Install the Elastic Stack (ELK) onto a development Linux VM in Azure +services: virtual-machines +author: rloutlaw +manager: justhe +ms.service: azure-virtual-machines +ms.collection: linux +ms.devlang: azurecli +ms.custom: devx-track-azurecli, linux-related-content, innovation-engine +ms.topic: how-to +ms.date: 10/11/2017 +ms.author: routlaw +--- + +# Install the Elastic Stack (ELK) on an Azure VM + +**Applies to:** :heavy_check_mark: Linux VMs :heavy_check_mark: Flexible scale sets + +This article walks you through how to deploy [Elasticsearch](https://www.elastic.co/products/elasticsearch), [Logstash](https://www.elastic.co/products/logstash), and [Kibana](https://www.elastic.co/products/kibana), on an Ubuntu VM in Azure. To see the Elastic Stack in action, you can optionally connect to Kibana and work with some sample logging data. + +Additionally, you can follow the [Deploy Elastic on Azure Virtual Machines](/training/modules/deploy-elastic-azure-virtual-machines/) module for a more guided tutorial on deploying Elastic on Azure Virtual Machines. + +In this tutorial you learn how to: + +> [!div class="checklist"] +> * Create an Ubuntu VM in an Azure resource group +> * Install Elasticsearch, Logstash, and Kibana on the VM +> * Send sample data to Elasticsearch with Logstash +> * Open ports and work with data in the Kibana console + +This deployment is suitable for basic development with the Elastic Stack. For more on the Elastic Stack, including recommendations for a production environment, see the [Elastic documentation](https://www.elastic.co/guide/index.html) and the [Azure Architecture Center](/azure/architecture/elasticsearch/). + +[!INCLUDE [azure-cli-prepare-your-environment.md](~/reusable-content/azure-cli/azure-cli-prepare-your-environment.md)] + +- This article requires version 2.0.4 or later of the Azure CLI. If using Azure Cloud Shell, the latest version is already installed. + +## Create a resource group + +In this section, environment variables are declared for use in subsequent commands. A random suffix is appended to resource names for uniqueness. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export RESOURCE_GROUP="myResourceGroup$RANDOM_SUFFIX" +export REGION="eastus2" +az group create --name $RESOURCE_GROUP --location $REGION +``` + +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxxxxx", + "location": "eastus", + "managedBy": null, + "name": "myResourceGroupxxxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create a virtual machine + +This section creates a VM with a unique name, while also generating SSH keys if they do not already exist. A random suffix is appended to ensure uniqueness. + +```bash +export VM_NAME="myVM$RANDOM_SUFFIX" +az vm create \ + --resource-group $RESOURCE_GROUP \ + --name $VM_NAME \ + --image Ubuntu2204 \ + --admin-username azureuser \ + --generate-ssh-keys +``` + +When the VM has been created, the Azure CLI shows information similar to the following example. Take note of the publicIpAddress. This address is used to access the VM. + +Results: + + +```JSON +{ + "fqdns": "", + "id": "/subscriptions/xxxxx/resourceGroups/myResourceGroupxxxxxx/providers/Microsoft.Compute/virtualMachines/myVMxxxxxx", + "location": "eastus", + "macAddress": "xx:xx:xx:xx:xx:xx", + "powerState": "VM running", + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "x.x.x.x", + "resourceGroup": "$RESOURCE_GROUP" +} +``` + +## SSH into your VM + +If you don't already know the public IP address of your VM, run the following command to list it: + +```azurecli-interactive +az network public-ip list --resource-group $RESOURCE_GROUP --query [].ipAddress +``` + +Use the following command to create an SSH session with the virtual machine. Substitute the correct public IP address of your virtual machine. In this example, the IP address is *40.68.254.142*. + +```bash +export PUBLIC_IP_ADDRESS=$(az network public-ip list --resource-group $RESOURCE_GROUP --query [].ipAddress -o tsv) +``` + +## Install the Elastic Stack + +In this section, you import the Elasticsearch signing key and update your APT sources list to include the Elastic package repository. This is followed by installing the Java runtime environment which is required for the Elastic Stack components. + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - +echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list +" +``` + +Install the Java Virtual Machine on the VM and configure the JAVA_HOME variable: + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +sudo apt install -y openjdk-8-jre-headless +export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +" +``` + +Run the following command to update Ubuntu package sources and install Elasticsearch, Kibana, and Logstash. + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " + wget -qO elasticsearch.gpg https://artifacts.elastic.co/GPG-KEY-elasticsearch + sudo mv elasticsearch.gpg /etc/apt/trusted.gpg.d/ + + echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-7.x.list + + sudo apt update + + # Now install the ELK stack + sudo apt install -y elasticsearch kibana logstash +" +``` + +> [!NOTE] +> Detailed installation instructions, including directory layouts and initial configuration, are maintained in [Elastic's documentation](https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html) + +## Start Elasticsearch + +Start Elasticsearch on your VM with the following command: + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +sudo systemctl start elasticsearch.service +" +``` + +This command produces no output, so verify that Elasticsearch is running on the VM with this curl command: + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +sleep 11 +sudo curl -XGET 'localhost:9200/' +" +``` + +If Elasticsearch is running, you see output like the following: + +Results: + + +```json +{ + "name" : "w6Z4NwR", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "SDzCajBoSK2EkXmHvJVaDQ", + "version" : { + "number" : "5.6.3", + "build_hash" : "1a2f265", + "build_date" : "2017-10-06T20:33:39.012Z", + "build_snapshot" : false, + "lucene_version" : "6.6.1" + }, + "tagline" : "You Know, for Search" +} +``` + +## Start Logstash and add data to Elasticsearch + +Start Logstash with the following command: + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +sudo systemctl start logstash.service +" +``` + +Test Logstash to make sure it's working correctly: + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +# Time-limited test with file input instead of stdin +sudo timeout 11s /usr/share/logstash/bin/logstash -e 'input { file { path => "/var/log/syslog" start_position => "end" sincedb_path => "/dev/null" stat_interval => "1 second" } } output { stdout { codec => json } }' || echo "Logstash test completed" +" +``` + +This is a basic Logstash [pipeline](https://www.elastic.co/guide/en/logstash/5.6/pipeline.html) that echoes standard input to standard output. + +Set up Logstash to forward the kernel messages from this VM to Elasticsearch. To create the Logstash configuration file, run the following command which writes the configuration to a new file called vm-syslog-logstash.conf: + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +cat << 'EOF' > vm-syslog-logstash.conf +input { + stdin { + type => "stdin-type" + } + + file { + type => "syslog" + path => [ "/var/log/*.log", "/var/log/*/*.log", "/var/log/messages", "/var/log/syslog" ] + start_position => "beginning" + } +} + +output { + + stdout { + codec => rubydebug + } + elasticsearch { + hosts => "localhost:9200" + } +} +EOF +" +``` + +Test this configuration and send the syslog data to Elasticsearch: + +```bash +# Run Logstash with the configuration for 60 seconds +sudo timeout 60s /usr/share/logstash/bin/logstash -f vm-syslog-logstash.conf & +LOGSTASH_PID=$! + +# Wait for data to be processed +echo "Processing logs for 60 seconds..." +sleep 65 + +# Verify data was sent to Elasticsearch with proper error handling +echo "Verifying data in Elasticsearch..." +ES_COUNT=$(sudo curl -s -XGET 'localhost:9200/_cat/count?v' | tail -n 1 | awk '{print $3}' 2>/dev/null || echo "0") + +# Make sure ES_COUNT is a number or default to 0 +if ! [[ "$ES_COUNT" =~ ^[0-9]+$ ]]; then + ES_COUNT=0 + echo "Warning: Could not get valid document count from Elasticsearch" +fi + +echo "Found $ES_COUNT documents in Elasticsearch" + +if [ "$ES_COUNT" -gt 0 ]; then + echo "✅ Logstash successfully sent data to Elasticsearch" +else + echo "❌ No data found in Elasticsearch, there might be an issue with Logstash configuration" +fi +``` + +You see the syslog entries in your terminal echoed as they are sent to Elasticsearch. Use CTRL+C to exit out of Logstash once you've sent some data. + +## Start Kibana and visualize the data in Elasticsearch + +Edit the Kibana configuration file (/etc/kibana/kibana.yml) and change the IP address Kibana listens on so you can access it from your web browser: + +```text +server.host: "0.0.0.0" +``` + +Start Kibana with the following command: + +```bash +ssh azureuser@$PUBLIC_IP_ADDRESS -o StrictHostKeyChecking=no " +sudo systemctl start kibana.service +" +``` + +Open port 5601 from the Azure CLI to allow remote access to the Kibana console: + +```azurecli-interactive +az vm open-port --port 5601 --resource-group $RESOURCE_GROUP --name $VM_NAME +``` + +## Next steps + +In this tutorial, you deployed the Elastic Stack into a development VM in Azure. You learned how to: + +> [!div class="checklist"] +> * Create an Ubuntu VM in an Azure resource group +> * Install Elasticsearch, Logstash, and Kibana on the VM +> * Send sample data to Elasticsearch from Logstash +> * Open ports and work with data in the Kibana console \ No newline at end of file diff --git a/scenarios/metadata.json b/scenarios/metadata.json index f66e7bc40..474fe2df6 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1653,5 +1653,24 @@ } ] } + }, + { + "status": "active", + "key": "azure-compute-docs/articles/virtual-machines/linux/tutorial-elasticsearch.md", + "title": "Deploy ElasticSearch on a development virtual machine in Azure", + "description": "Install the Elastic Stack (ELK) onto a development Linux VM in Azure", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-compute-docs/articles/virtual-machines/linux/tutorial-elasticsearch.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/tutorial-elasticsearch", + "nextSteps": [ + { + "title": "Create a Linux VM with the Azure CLI", + "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/linux/quick-create-cli" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } } ] \ No newline at end of file diff --git a/scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md b/scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md index 9e2b53e3b..66b00cb3d 100644 --- a/scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md +++ b/scenarios/sql-docs/docs/linux/quickstart-install-connect-docker.md @@ -1091,7 +1091,7 @@ The following steps use **sqlcmd** outside of your container to connect to [!INC ::: zone pivot="cs1-bash" - ```bash + ```text sudo sqlcmd -S ,1433 -U -P "" ``` @@ -1128,7 +1128,7 @@ The following steps use **sqlcmd** outside of your container to connect to [!INC ::: zone pivot="cs1-bash" - ```bash + ```text sudo sqlcmd ``` @@ -1170,7 +1170,7 @@ If you want to remove the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md. ::: zone pivot="cs1-bash" -```bash +```text docker stop sql1 docker rm sql1 ``` @@ -1201,7 +1201,7 @@ If you want to remove the [!INCLUDE [ssnoversion-md](../includes/ssnoversion-md. ::: zone pivot="cs1-bash" -```bash +```text sudo sqlcmd delete --force ``` diff --git a/tools/abc.md b/tools/abc.md index bb08cd960..ec5d50b9e 100644 --- a/tools/abc.md +++ b/tools/abc.md @@ -1,173 +1,291 @@ --- -title: Upgrade Azure Kubernetes Service (AKS) node images -description: Learn how to upgrade the images on AKS cluster nodes and node pools. -ms.topic: how-to -ms.custom: devx-track-azurecli, innovation-engine -ms.subservice: aks-upgrade -ms.service: azure-kubernetes-service -ms.date: 09/20/2024 -author: schaffererin -ms.author: schaffererin +title: "Quickstart: Create an Azure IoT Edge Device on Linux" +description: Learn to configure an Azure IoT Edge device on Linux. This guide walks you through creating an IoT Hub, registering a device, and deploying a simulated sensor module. +#customer intent: As a developer, I want to create an IoT Edge device on Linux so that I can deploy and test containerized modules. +author: PatAltimore +ms.author: patricka +ms.date: 03/27/2025 +ms.topic: quickstart +ms.service: azure-iot-edge +services: iot-edge +ms.custom: mvc, devx-track-azurecli, mode-other, linux-related-content --- -## Environment Variables +# Quickstart: Deploy your first IoT Edge module to a virtual Linux device -The following environment variables are declared and will be used in subsequent code blocks. They replace the placeholder parameters in the original document with standardized variable names. +[!INCLUDE [iot-edge-version-all-supported](includes/iot-edge-version-all-supported.md)] -```bash -export AKS_NODEPOOL="nodepool1" -export AKS_CLUSTER="apache-airflow-aks" -export AKS_RESOURCE_GROUP="apache-airflow-rg" -``` +Try Azure IoT Edge in this quickstart by deploying containerized code to a virtual Linux IoT Edge device. IoT Edge lets you remotely manage code on your devices so you can send more of your workloads to the edge. For this quickstart, use an Azure virtual machine for your IoT Edge device. It lets you quickly create a test machine and delete it when you're done. + +In this quickstart, you learn how to: + +* Create an IoT Hub. +* Register an IoT Edge device to your IoT hub. +* Install and start the IoT Edge runtime on a virtual device. +* Deploy a module remotely to an IoT Edge device. + +:::image type="content" source="./media/quickstart-linux/install-edge-full.png" alt-text="Diagram of Quickstart architecture for device and cloud."::: + +This quickstart walks you through creating a Linux virtual machine that's configured to be an IoT Edge device. Then, you deploy a module from the Azure portal to your device. This quickstart uses a simulated sensor module that generates temperature, humidity, and pressure data. The other Azure IoT Edge tutorials build upon the work you do here by deploying additional modules that analyze the simulated data for business insights. + +If you don't have an active Azure subscription, create a [free account](https://azure.microsoft.com/free) before you begin. + +## Prerequisites + +Set up your environment for the Azure CLI. + +[!INCLUDE [azure-cli-prepare-your-environment-no-header.md](~/reusable-content/azure-cli/azure-cli-prepare-your-environment-no-header.md)] + +A resource group to manage all the resources you use in this quickstart. This quickstart and the following tutorials use the example resource group name **IoTEdgeResources**. + + ```azurecli-interactive + `az group create --name IoTEdgeResources --location westus2` + ``` + +## Create an IoT Hub + +Start the quickstart by creating an IoT Hub with the Azure CLI. + +:::image type="content" source="./media/quickstart-linux/create-iot-hub.png" alt-text="Diagram that shows how to create an IoT Hub in the cloud."::: + +The free tier of IoT Hub works for this quickstart. If you've used IoT Hub in the past and already have a hub created, you can use that IoT hub. + +The following code creates a free **F1** hub in the resource group **IoTEdgeResources**. Replace `` with a unique name for your IoT Hub. Creating an IoT Hub might take a few minutes. + + ```azurecli-interactive + az iot hub create --resource-group IoTEdgeResources --name --sku F1 --partition-count 2 + ``` + + If you get an error because there's already one free IoT Hub in your subscription, change the SKU to **S1**. Each subscription can only have one free IoT hub. If you get an error that the IoT Hub name isn't available, it means that someone else already has a hub with that name. Try a new name. + +## Register an IoT Edge device + +Register an IoT Edge device with the IoT hub you just created. + +:::image type="content" source="./media/quickstart-linux/register-device.png" alt-text="Diagram of how to register a device with an IoT Hub identity."::: + +Create a device identity for your IoT Edge device so that it can communicate with your IoT hub. The device identity lives in the cloud, and you use a unique device connection string to associate a physical device to a device identity. + +Because IoT Edge devices behave and are managed differently from typical IoT devices, declare this identity as an IoT Edge device using the `--edge-enabled` flag. + +1. Enter the following command in Azure Cloud Shell to create a device named **myEdgeDevice** in your hub. + + ```azurecli-interactive + az iot hub device-identity create --device-id myEdgeDevice --edge-enabled --hub-name + ``` + + If you get an error about *iothubowner* policy keys, make sure that your Cloud Shell is running the latest version of the *azure-iot* extension. + +2. Check the connection string for your device, which links the physical device to its identity in IoT Hub. It includes the name of your IoT Hub, the name of your device, and a shared key that authenticates connections between them. You use this connection string again in the next section to set up your IoT Edge device. + + ```azurecli-interactive + az iot hub device-identity connection-string show --device-id myEdgeDevice --hub-name + ``` + + For example, the connection string should look similar to `HostName=contoso-hub.azure-devices.net;DeviceId=myEdgeDevice;SharedAccessKey=`. + +## Configure your IoT Edge device + +Create a virtual machine with the Azure IoT Edge runtime. + +:::image type="content" source="./media/quickstart-linux/start-runtime.png" alt-text="Diagram of how to start the runtime on a device."::: + +The IoT Edge runtime is deployed on all IoT Edge devices and has three components. The *IoT Edge security daemon* starts each time an IoT Edge device boots and bootstraps the device by starting the IoT Edge agent. The *IoT Edge agent* facilitates deployment and monitoring of modules on the IoT Edge device, including the IoT Edge hub. The *IoT Edge hub* manages communications between modules on the IoT Edge device, and between the device and IoT Hub. + +During runtime configuration, provide a device connection string. This string is retrieved from the Azure CLI. This string associates your physical device with the IoT Edge device identity in Azure. + +### Deploy the IoT Edge device + +This section uses an Azure Resource Manager template to create a new virtual machine and install the IoT Edge runtime on it. If you want to use your own Linux device instead, you can follow the installation steps in [Manually provision a single Linux IoT Edge device](how-to-provision-single-device-linux-symmetric.md), then return to this quickstart. -# Upgrade Azure Kubernetes Service (AKS) node images +Use the **Deploy to Azure** button or CLI commands to create an IoT Edge device based on the prebuilt [iotedge-vm-deploy](https://github.com/Azure/iotedge-vm-deploy) template. -Azure Kubernetes Service (AKS) regularly provides new node images, so it's beneficial to upgrade your node images frequently to use the latest AKS features. Linux node images are updated weekly, and Windows node images are updated monthly. Image upgrade announcements are included in the [AKS release notes](https://github.com/Azure/AKS/releases), and it can take up to a week for these updates to be rolled out across all regions. You can also perform node image upgrades automatically and schedule them using planned maintenance. For more information, see [Automatically upgrade node images][auto-upgrade-node-image]. +* Deploy using the IoT Edge Azure Resource Manager template. -This article shows you how to upgrade AKS cluster node images and how to update node pool images without upgrading the Kubernetes version. For information on upgrading the Kubernetes version for your cluster, see [Upgrade an AKS cluster][upgrade-cluster]. + [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fazure%2Fiotedge-vm-deploy%2Fmain%2FedgeDeploy.json) -> [!NOTE] -> The AKS cluster must use virtual machine scale sets for the nodes. -> -> It's not possible to downgrade a node image version (for example *AKSUbuntu-2204 to AKSUbuntu-1804*, or *AKSUbuntu-2204-202308.01.0 to AKSUbuntu-2204-202307.27.0*). +* For bash or Cloud Shell users, copy the following command into a text editor, replace the placeholder text with your information, then copy into your bash or Cloud Shell window: -## Check for available node image upgrades + ```azurecli-interactive + az deployment group create \ + --resource-group IoTEdgeResources \ + --template-uri "https://raw.githubusercontent.com/Azure/iotedge-vm-deploy/main/edgeDeploy.json" \ + --parameters dnsLabelPrefix='' \ + --parameters adminUsername='azureUser' \ + --parameters deviceConnectionString=$(az iot hub device-identity connection-string show --device-id myEdgeDevice --hub-name -o tsv) \ + --parameters authenticationType='password' \ + --parameters adminPasswordOrKey="" + ``` -1. Check for available node image upgrades using the [`az aks nodepool get-upgrades`][az-aks-nodepool-get-upgrades] command. +* For PowerShell users, copy the following command into your PowerShell window, then replace the placeholder text with your own information: - ```azurecli-interactive - az aks nodepool get-upgrades \ - --nodepool-name $AKS_NODEPOOL \ - --cluster-name $AKS_CLUSTER \ - --resource-group $AKS_RESOURCE_GROUP - ``` + ```azurecli + az deployment group create ` + --resource-group IoTEdgeResources ` + --template-uri "https://raw.githubusercontent.com/Azure/iotedge-vm-deploy/main/edgeDeploy.json" ` + --parameters dnsLabelPrefix='' ` + --parameters adminUsername='azureUser' ` + --parameters deviceConnectionString=$(az iot hub device-identity connection-string show --device-id myEdgeDevice --hub-name -o tsv) ` + --parameters authenticationType='password' ` + --parameters adminPasswordOrKey="" + ``` -1. In the output, find and make note of the `latestNodeImageVersion` value. This value is the latest node image version available for your node pool. -1. Check your current node image version to compare with the latest version using the [`az aks nodepool show`][az-aks-nodepool-show] command. +This template takes the following parameters: - ```azurecli-interactive - az aks nodepool show \ - --resource-group $AKS_RESOURCE_GROUP \ - --cluster-name $AKS_CLUSTER \ - --name $AKS_NODEPOOL \ - --query nodeImageVersion - ``` +| Parameter | Description | +| --------- | ----------- | +| **resource-group** | The resource group in which the resources are created. Use the default **IoTEdgeResources** that we've been using throughout this article or provide the name of an existing resource group in your subscription. | +| **template-uri** | A pointer to the Resource Manager template that we're using. | +| **dnsLabelPrefix** | A string that is used to create the virtual machine's hostname. Replace the placeholder text with a name for your virtual machine. | +| **adminUsername** | A username for the admin account of the virtual machine. Use the example **azureUser** or provide a new username. | +| **deviceConnectionString** | The connection string from the device identity in IoT Hub, which is used to configure the IoT Edge runtime on the virtual machine. The CLI command within this parameter grabs the connection string for you. Replace the placeholder text with your IoT hub name. | +| **authenticationType** | The authentication method for the admin account. This quickstart uses **password** authentication, but you can also set this parameter to **sshPublicKey**. | +| **adminPasswordOrKey** | The password or value of the SSH key for the admin account. Replace the placeholder text with a secure password. Your password must be at least 12 characters long and have three of four of the following: lowercase characters, uppercase characters, digits, and special characters. | -1. If the `nodeImageVersion` value is different from the `latestNodeImageVersion`, you can upgrade your node image. +After deployment completes, JSON-formatted output in the CLI contains the SSH information to connect to the virtual machine. Copy the value of the **public SSH** entry of the **outputs** section. For example, your SSH command should look similar to `ssh azureUser@edge-vm.westus2.cloudapp.azure.com`. -## Upgrade all node images in all node pools +### View the IoT Edge runtime status -1. Upgrade all node images in all node pools in your cluster using the [`az aks upgrade`][az-aks-upgrade] command with the `--node-image-only` flag. +The rest of the commands in this quickstart take place on your IoT Edge device itself, so that you can see what's happening on the device. If you're using a virtual machine, connect to that machine now using the admin username that you set up and the DNS name that was output by the deployment command. You can also find the DNS name on your virtual machine's overview page in the Azure portal. Use the following command to connect to your virtual machine. Replace `` and `` with your own values. - ```text - az aks upgrade \ - --resource-group $AKS_RESOURCE_GROUP \ - --name $AKS_CLUSTER \ - --node-image-only \ - --yes - ``` + ```console + ssh @ + ``` -1. You can check the status of the node images using the `kubectl get nodes` command. +Once connected to your virtual machine, verify that the runtime was successfully installed and configured on your IoT Edge device. - > [!NOTE] - > This command might differ slightly depending on the shell you use. For more information on Windows and PowerShell environments, see the [Kubernetes JSONPath documentation][kubernetes-json-path]. +1. Check if IoT Edge is running. The following command returns a status of **Ok** if IoT Edge is running or provides any service errors. - ```bash - kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' - ``` + ```bash + sudo iotedge system status + ``` -1. When the upgrade completes, use the [`az aks show`][az-aks-show] command to get the updated node pool details. The current node image is shown in the `nodeImageVersion` property. + >[!TIP] + >You need elevated privileges to run `iotedge` commands. Once you sign out of your machine and sign back in the first time after installing the IoT Edge runtime, your permissions are automatically updated. Until then, use `sudo` in front of the commands. - ```azurecli-interactive - az aks show \ - --resource-group $AKS_RESOURCE_GROUP \ - --name $AKS_CLUSTER - ``` +2. If you need to troubleshoot the service, retrieve the service logs. -## Upgrade a specific node pool + ```bash + sudo iotedge system logs + ``` -1. Update the OS image of a node pool without doing a Kubernetes cluster upgrade using the [`az aks nodepool upgrade`][az-aks-nodepool-upgrade] command with the `--node-image-only` flag. +3. View all the modules running on your IoT Edge device. Since the service just started for the first time, you should only see the **edgeAgent** module running. The edgeAgent module runs by default and helps to install and start any additional modules that you deploy to your device. - ```azurecli-interactive - az aks nodepool upgrade \ - --resource-group $AKS_RESOURCE_GROUP \ - --cluster-name $AKS_CLUSTER \ - --name $AKS_NODEPOOL \ - --node-image-only - ``` + ```bash + sudo iotedge list + ``` -1. You can check the status of the node images with the `kubectl get nodes` command. +Your IoT Edge device is now configured. It's ready to run cloud-deployed modules. - > [!NOTE] - > This command may differ slightly depending on the shell you use. For more information on Windows and PowerShell environments, see the [Kubernetes JSONPath documentation][kubernetes-json-path]. +## Deploy a module - ```bash - kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' - ``` +Manage your Azure IoT Edge device from the cloud to deploy a module that sends device telemetry data to IoT Hub. -1. When the upgrade completes, use the [`az aks nodepool show`][az-aks-nodepool-show] command to get the updated node pool details. The current node image is shown in the `nodeImageVersion` property. +:::image type="content" source="./media/quickstart-linux/deploy-module.png" alt-text="Diagram of how to deploy a module from cloud to device."::: - ```azurecli-interactive - az aks nodepool show \ - --resource-group $AKS_RESOURCE_GROUP \ - --cluster-name $AKS_CLUSTER \ - --name $AKS_NODEPOOL - ``` +A key capability of Azure IoT Edge is deploying code to your IoT Edge devices from the cloud. *IoT Edge modules* are executable packages implemented as containers. In this section, you deploy a pre-built module from the [IoT Edge Modules section of Microsoft Artifact Registry](https://mcr.microsoft.com/catalog?cat=IoT%20Edge%20Modules&alphaSort=asc&alphaSortKey=Name). -## Upgrade node images with node surge +The module that you deploy in this section simulates a sensor and sends generated data. This module is a useful piece of code when you're getting started with IoT Edge because you can use the simulated data for development and testing. If you want to see exactly what this module does, you can view the [simulated temperature sensor source code](https://github.com/Azure/iotedge/blob/main/edge-modules/SimulatedTemperatureSensor/src/Program.cs). -To speed up the node image upgrade process, you can upgrade your node images using a customizable node surge value. By default, AKS uses one extra node to configure upgrades. +Use these steps to deploy your first module. -1. Upgrade node images with node surge using the [`az aks nodepool update`][az-aks-nodepool-update] command with the `--max-surge` flag to configure the number of nodes used for upgrades. +1. Sign in to the [Azure portal](https://portal.azure.com) and go to your IoT Hub. - > [!NOTE] - > To learn more about the trade-offs of various `--max-surge` settings, see [Customize node surge upgrade][max-surge]. +1. From the menu on the left, under **Device Management**, select **Devices**. - ```azurecli-interactive - az aks nodepool update \ - --resource-group $AKS_RESOURCE_GROUP \ - --cluster-name $AKS_CLUSTER \ - --name $AKS_NODEPOOL \ - --max-surge 33% \ - --no-wait - ``` +1. Select the device ID of the target IoT Edge device from the list. -1. You can check the status of the node images with the `kubectl get nodes` command. + When you create a new IoT Edge device, it displays the status code `417 -- The device's deployment configuration is not set` in the Azure portal. This status is normal, and means that the device is ready to receive a module deployment. - ```bash - kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.kubernetes\.azure\.com\/node-image-version}{"\n"}{end}' - ``` +1. On the upper bar, select **Set Modules**. -1. Get the updated node pool details using the [`az aks nodepool show`][az-aks-nodepool-show] command. The current node image is shown in the `nodeImageVersion` property. + Select the modules you want to run on your device. You can choose from modules that you've built yourself or images in a container registry. In this quickstart, you deploy a module from the Microsoft container registry. - ```azurecli-interactive - az aks nodepool show \ - --resource-group $AKS_RESOURCE_GROUP \ - --cluster-name $AKS_CLUSTER \ - --name $AKS_NODEPOOL - ``` +1. In the **IoT Edge modules** section, select **Add** then choose **IoT Edge Module**. +1. Update the following module settings: + + | Setting | Value | + |--------------------|----------------------------------------------------------------------| + | IoT Module name | `SimulatedTemperatureSensor` | + | Image URI | `mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:latest` | + | Restart policy | always | + | Desired status | running | + +1. Select **Next: Routes** to continue to configure routes. + +1. Add a route that sends all messages from the simulated temperature module to IoT Hub. + + | Setting | Value | + |----------------------------------|--------------------------------------------| + | Name | `SimulatedTemperatureSensorToIoTHub` | + | Value | `FROM /messages/modules/SimulatedTemperatureSensor/* INTO $upstream` | + +1. Select **Next: Review + create**. + +1. Review the JSON file, and then select **Create**. The JSON file defines all the modules that you deploy to your IoT Edge device. + + > [!NOTE] + > When you submit a new deployment to an IoT Edge device, nothing is pushed to your device. Instead, the device queries IoT Hub regularly for any new instructions. If the device finds an updated deployment manifest, it uses the information about the new deployment to pull the module images from the cloud then starts running the modules locally. This process can take a few minutes. + +After you create the module deployment details, the wizard returns you to the device details page. View the deployment status on the **Modules** tab. + +You should see three modules: **$edgeAgent**, **$edgeHub**, and **SimulatedTemperatureSensor**. If one or more of the modules has **Yes** under **Specified in Deployment** but not under **Reported by Device**, your IoT Edge device is still starting them. Wait a few minutes and refresh the page. + +:::image type="content" source="./media/quickstart-linux/view-deployed-modules.png" alt-text="Screenshot that shows the SimulatedTemperatureSensor in the list of deployed modules." lightbox="./media/quickstart-linux/view-deployed-modules.png"::: + +If you have issues deploying modules, learn more in [Troubleshoot IoT Edge devices from the Azure portal](troubleshoot-in-portal.md). + +## View generated data + +In this quickstart, you create a new IoT Edge device and install the IoT Edge runtime on it. Then, you use the Azure portal to deploy an IoT Edge module to run on the device without making changes to the device itself. + +In this case, the module that you pushed generates sample environment data that you can use for testing later. The simulated sensor is monitoring both a machine and the environment around the machine. For example, this sensor can be in a server room, on a factory floor, or on a wind turbine. The message includes ambient temperature and humidity, machine temperature and pressure, and a timestamp. The IoT Edge tutorials use the data created by this module as test data for analytics. + +Open the command prompt on your IoT Edge device, or use the SSH connection from Azure CLI. Confirm that the module you deployed from the cloud is running on your IoT Edge device: + +```bash +sudo iotedge list +``` + +:::image type="content" source="./media/quickstart-linux/iot-edge-list.png" alt-text="Screenshot that shows three modules on your device." lightbox="./media/quickstart-linux/iot-edge-list.png"::: + +View the messages sent from the temperature sensor module: + +```bash +sudo iotedge logs SimulatedTemperatureSensor -f +``` + +:::image type="content" source="./media/quickstart-linux/iot-edge-logs.png" alt-text="Screenshot that shows data from your module in the output console." lightbox="./media/quickstart-linux/iot-edge-logs.png"::: + +>[!TIP] +>IoT Edge commands are case sensitive when referring to module names. + +## Clean up resources + +To continue with the IoT Edge tutorials, use the device you registered and set up in this quickstart. Otherwise, delete the Azure resources you created to avoid charges. + +If you created your virtual machine and IoT hub in a new resource group, you can delete that group and all the associated resources. Double-check the contents of the resource group to ensure there's nothing you want to keep. If you don't want to delete the whole group, you can delete individual resources instead. + +> [!IMPORTANT] +> Deleting a resource group is irreversible. + +Delete the **IoTEdgeResources** group. Deleting a resource group might take a few minutes. + +```azurecli-interactive +az group delete --name IoTEdgeResources --yes +``` + +Confirm the resource group is deleted by viewing the list of resource groups. + +```azurecli-interactive +az group list +``` ## Next steps -- For information about the latest node images, see the [AKS release notes](https://github.com/Azure/AKS/releases). -- Learn how to upgrade the Kubernetes version with [Upgrade an AKS cluster][upgrade-cluster]. -- [Automatically apply cluster and node pool upgrades with GitHub Actions][github-schedule]. -- Learn more about multiple node pools with [Create multiple node pools][use-multiple-node-pools]. -- Learn about upgrading best practices with [AKS patch and upgrade guidance][upgrade-operators-guide]. - - -[kubernetes-json-path]: https://kubernetes.io/docs/reference/kubectl/jsonpath/ - - -[upgrade-cluster]: upgrade-aks-cluster.md -[github-schedule]: node-upgrade-github-actions.md -[use-multiple-node-pools]: create-node-pools.md -[max-surge]: upgrade-aks-cluster.md#customize-node-surge-upgrade -[auto-upgrade-node-image]: auto-upgrade-node-image.md -[az-aks-nodepool-get-upgrades]: /cli/azure/aks/nodepool#az_aks_nodepool_get_upgrades -[az-aks-nodepool-show]: /cli/azure/aks/nodepool#az_aks_nodepool_show -[az-aks-nodepool-upgrade]: /cli/azure/aks/nodepool#az_aks_nodepool_upgrade -[az-aks-nodepool-update]: /cli/azure/aks/nodepool#az_aks_nodepool_update -[az-aks-upgrade]: /cli/azure/aks#az_aks_upgrade -[az-aks-show]: /cli/azure/aks#az_aks_show -[upgrade-operators-guide]: /azure/architecture/operator-guides/aks/aks-upgrade-practices \ No newline at end of file +In this quickstart, you created an IoT Edge device and used the Azure IoT Edge cloud interface to deploy code onto the device. Now, you use a test device that generates raw data about its environment. + +In the next tutorial, you'll learn how to monitor the activity and health of your device from the Azure portal. + +> [!div class="nextstepaction"] +> [Monitor IoT Edge devices](tutorial-monitor-with-workbooks.md) \ No newline at end of file diff --git a/tools/abc_converted.md b/tools/abc_converted.md new file mode 100644 index 000000000..4d1349885 --- /dev/null +++ b/tools/abc_converted.md @@ -0,0 +1,429 @@ +--- +title: "Quickstart: Create an Azure IoT Edge Device on Linux" +description: Learn to configure an Azure IoT Edge device on Linux. This guide walks you through creating an IoT Hub, registering a device, and deploying a simulated sensor module. +#customer intent: As a developer, I want to create an IoT Edge device on Linux so that I can deploy and test containerized modules. +author: PatAltimore +ms.author: patricka +ms.date: 03/27/2025 +ms.topic: quickstart +ms.service: azure-iot-edge +services: iot-edge +ms.custom: mvc, devx-track-azurecli, mode-other, linux-related-content +--- + +## Environment Variables + +In this section we declare environment variables that will be used throughout the Exec Doc. A random suffix is appended to resource names that must be unique for each deployment. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="eastus2" +export RESOURCE_GROUP="IoTEdgeResources$RANDOM_SUFFIX" +export IOTHUB_NAME="UniqueIoTHub$RANDOM_SUFFIX" +export VM_NAME="myvm$RANDOM_SUFFIX" +``` + +# Quickstart: Deploy your first IoT Edge module to a virtual Linux device + +[!INCLUDE [iot-edge-version-all-supported](includes/iot-edge-version-all-supported.md)] + +Try Azure IoT Edge in this quickstart by deploying containerized code to a virtual Linux IoT Edge device. IoT Edge lets you remotely manage code on your devices so you can send more of your workloads to the edge. For this quickstart, use an Azure virtual machine for your IoT Edge device. It lets you quickly create a test machine and delete it when you're done. + +In this quickstart, you learn how to: + +* Create an IoT Hub. +* Register an IoT Edge device to your IoT hub. +* Install and start the IoT Edge runtime on a virtual device. +* Deploy a module remotely to an IoT Edge device. + +:::image type="content" source="./media/quickstart-linux/install-edge-full.png" alt-text="Diagram of Quickstart architecture for device and cloud."::: + +This quickstart walks you through creating a Linux virtual machine that's configured to be an IoT Edge device. Then, you deploy a module from the Azure portal to your device. This quickstart uses a simulated sensor module that generates temperature, humidity, and pressure data. The other Azure IoT Edge tutorials build upon the work you do here by deploying additional modules that analyze the simulated data for business insights. + +If you don't have an active Azure subscription, create a [free account](https://azure.microsoft.com/free) before you begin. + +## Prerequisites + +Set up your environment for the Azure CLI. + +[!INCLUDE [azure-cli-prepare-your-environment-no-header.md](~/reusable-content/azure-cli/azure-cli-prepare-your-environment-no-header.md)] + +## Create a resource group + +A resource group to manage all the resources you use in this quickstart. This quickstart and the following tutorials use the example resource group name **IoTEdgeResources** with a randomized suffix. + + ```azurecli-interactive + az group create --name $RESOURCE_GROUP --location $REGION + ``` +Results: + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/IoTEdgeResourcesabcd12", + "location": "westus2", + "managedBy": null, + "name": "IoTEdgeResourcesabcd12", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create an IoT Hub + +Start the quickstart by creating an IoT Hub with the Azure CLI. + +:::image type="content" source="./media/quickstart-linux/create-iot-hub.png" alt-text="Diagram that shows how to create an IoT Hub in the cloud."::: + +The free tier of IoT Hub works for this quickstart. If you've used IoT Hub in the past and already have a hub created, you can use that IoT hub. + +The following code creates a **S1** hub in the resource group. Replace the placeholder with your preferred IoT Hub name if desired – here we use the environment variable $IOTHUB_NAME. Creating an IoT Hub might take a few minutes. + + ```azurecli-interactive + az iot hub create --resource-group $RESOURCE_GROUP --name $IOTHUB_NAME --sku S1 --partition-count 2 + ``` +Results: + + +```JSON +{ + "name": "UniqueIoTHubabcd12", + "sku": "S1", + "resourceGroup": "IoTEdgeResourcesabcd12", + "location": "westus2", + "state": "Active", + "skuCapacity": 1 +} +``` + +If you use F1 (the free tier), you can only create one IoT Hub per subscription. If you try to create a second hub, you'll receive an error message. In such a case, change the SKU to **S1**. Each subscription can only have one free IoT hub. If you get an error that the IoT Hub name isn't available, it means that someone else already has a hub with that name. Try a new name. + +## Register an IoT Edge device + +Register an IoT Edge device with the IoT hub you just created. + +:::image type="content" source="./media/quickstart-linux/register-device.png" alt-text="Diagram of how to register a device with an IoT Hub identity."::: + +Create a device identity for your IoT Edge device so that it can communicate with your IoT hub. The device identity lives in the cloud, and you use a unique device connection string to associate a physical device to a device identity. + +Because IoT Edge devices behave and are managed differently from typical IoT devices, declare this identity as an IoT Edge device using the --edge-enabled flag. + +1. Enter the following command in Azure Cloud Shell to create a device named **myEdgeDevice** in your hub. + + ```azurecli-interactive + az config set extension.use_dynamic_install=yes_without_prompt + az iot hub device-identity create --device-id myEdgeDevice --edge-enabled --hub-name $IOTHUB_NAME + ``` +Results: + + +```JSON +{ + "deviceId": "myEdgeDevice", + "generationId": "xxxxxxxx", + "status": "enabled", + "connectionState": "Disconnected", + "statusReason": null, + "connectionStateUpdatedTime": null, + "statusUpdatedTime": "2025-03-27T00:00:00.000Z", + "lastActivityTime": null, + "cloudToDeviceMessageCount": 0, + "authentication": { + "symmetricKey": { + "primaryKey": "xxxxxxxxxxxxxxxx==", + "secondaryKey": "xxxxxxxxxxxxxxxx==" + }, + "type": "sas" + }, + "capabilities": { + "iotEdge": true + }, + "etag": "xxxxxxxxxxxxxx" +} +``` + +2. Check the connection string for your device, which links the physical device to its identity in IoT Hub. It includes the name of your IoT Hub, the name of your device, and a shared key that authenticates connections between them. You use this connection string again in the next section to set up your IoT Edge device. + + ```azurecli-interactive + az iot hub device-identity connection-string show --device-id myEdgeDevice --hub-name $IOTHUB_NAME + ``` +Results: + + +```JSON +{ + "connectionString": "HostName=UniqueIoTHubabcd12.azure-devices.net;DeviceId=myEdgeDevice;SharedAccessKey=xxxxxxxxxxxxxxxxxxxxxxx" +} +``` + +For example, the connection string should look similar to +HostName=contoso-hub.azure-devices.net;DeviceId=myEdgeDevice;SharedAccessKey=. + +## Configure your IoT Edge device + +Create a virtual machine with the Azure IoT Edge runtime. + +:::image type="content" source="./media/quickstart-linux/start-runtime.png" alt-text="Diagram of how to start the runtime on a device."::: + +The IoT Edge runtime is deployed on all IoT Edge devices and has three components. The IoT Edge security daemon starts each time an IoT Edge device boots and bootstraps the device by starting the IoT Edge agent. The IoT Edge agent facilitates deployment and monitoring of modules on the IoT Edge device, including the IoT Edge hub. The IoT Edge hub manages communications between modules on the IoT Edge device, and between the device and IoT Hub. + +During runtime configuration, provide a device connection string. This string is retrieved from the Azure CLI. This string associates your physical device with the IoT Edge device identity in Azure. + +### Deploy the IoT Edge device + +This section uses an Azure Resource Manager template to create a new virtual machine and install the IoT Edge runtime on it. If you want to use your own Linux device instead, you can follow the installation steps in [Manually provision a single Linux IoT Edge device](how-to-provision-single-device-linux-symmetric.md), then return to this quickstart. + +Use the Deploy to Azure button or CLI commands to create an IoT Edge device based on the prebuilt [iotedge-vm-deploy](https://github.com/Azure/iotedge-vm-deploy) template. + +* Deploy using the IoT Edge Azure Resource Manager template. + + [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fazure%2Fiotedge-vm-deploy%2Fmain%2FedgeDeploy.json) + +* For bash or Cloud Shell users, first create a file named **cloud-init.txt** in your current working directory. This file contains the configuration for the IoT Edge runtime: + + ```text + #cloud-config + package_update: true + package_upgrade: true + runcmd: + - curl https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb > packages-microsoft-prod.deb + - dpkg -i packages-microsoft-prod.deb + - apt-get update + - apt-get install aziot-edge -y + - | + CONNECTION_STRING="$(az iot hub device-identity connection-string show \ + --device-id myEdgeDevice \ + --hub-name $IOTHUB_NAME \ + -o tsv)" + iotedge config mp --connection-string "$CONNECTION_STRING" + iotedge config apply + ``` + +* Then, copy the following command into a text editor, replace the placeholder text with your information, then copy into your bash or Cloud Shell window: + + ```azurecli-interactive + az vm create \ + --resource-group $RESOURCE_GROUP \ + --name $VM_NAME \ + --image Ubuntu2204 \ + --admin-username azureuser \ + --generate-ssh-keys \ + --custom-data cloud-init.txt + ``` + +* For PowerShell users, copy the following command into your PowerShell window, then replace the placeholder text with your own information: + + ```powershell + az deployment group create ` + --resource-group $RESOURCE_GROUP ` + --template-uri "https://raw.githubusercontent.com/Azure/iotedge-vm-deploy/main/edgeDeploy.json" ` + --parameters dnsLabelPrefix="$VM_NAME" ` + --parameters adminUsername='azureUser' ` + --parameters deviceConnectionString=$(az iot hub device-identity connection-string show --device-id myEdgeDevice --hub-name $IOTHUB_NAME -o tsv) ` + --parameters authenticationType='password' ` + --parameters adminPasswordOrKey="" + ``` + +This template takes the following parameters: + +| Parameter | Description | +| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **resource-group** | The resource group in which the resources are created. Use the default **IoTEdgeResources** that we've been using throughout this article or provide the name of an existing resource group in your subscription. | +| **template-uri** | A pointer to the Resource Manager template that we're using. | +| **dnsLabelPrefix** | A string that is used to create the virtual machine's hostname. Replace the placeholder text with a name for your virtual machine. | +| **adminUsername** | A username for the admin account of the virtual machine. Use the example **azureUser** or provide a new username. | +| **deviceConnectionString** | The connection string from the device identity in IoT Hub, which is used to configure the IoT Edge runtime on the virtual machine. The CLI command within this parameter grabs the connection string for you. Replace the placeholder text with your IoT hub name. | +| **authenticationType** | The authentication method for the admin account. This quickstart uses **password** authentication, but you can also set this parameter to **sshPublicKey**. | +| **adminPasswordOrKey** | The password or value of the SSH key for the admin account. Replace the placeholder text with a secure password. Your password must be at least 12 characters long and have three of four of the following: lowercase characters, uppercase characters, digits, and special characters. | + +After deployment completes, JSON-formatted output in the CLI contains the SSH information to connect to the virtual machine. The output includes the public IP address of the virtual machine, which you can use to connect to it. + + ```bash + export IP_ADDRESS=$(az vm show -d -g $RESOURCE_GROUP -n $VM_NAME --query publicIps -o tsv) + + ssh azureuser@$IP_ADDRESS -o StrictHostKeyChecking=no + ``` + +### View the IoT Edge runtime status + +The rest of the commands in this quickstart take place on your IoT Edge device itself, so that you can see what's happening on the device. If you're using a virtual machine, connect to that machine now using the admin username that you set up and the DNS name that was output by the deployment command. You can also find the DNS name on your virtual machine's overview page in the Azure portal. Use the following command to connect to your virtual machine. Replace and with your own values. + + ```text + ssh @ + ``` + +Once connected to your virtual machine, verify that the runtime was successfully installed and configured on your IoT Edge device. + +1. Check if IoT Edge is running. The following command returns a status of **Ok** if IoT Edge is running or provides any service errors. + + ```bash + sudo iotedge system status + ``` +Results: + + +```text +Status: Ok +``` + + >[!TIP] + >You need elevated privileges to run iotedge commands. Once you sign out of your machine and sign back in the first time after installing the IoT Edge runtime, your permissions are automatically updated. Until then, use sudo in front of the commands. + +2. If you need to troubleshoot the service, retrieve the service logs. + + ```bash + sudo iotedge system logs + ``` +Results: + + +```text +... (service log output redacted for brevity) ... +``` + +3. View all the modules running on your IoT Edge device. Since the service just started for the first time, you should only see the **edgeAgent** module running. The edgeAgent module runs by default and helps to install and start any additional modules that you deploy to your device. + + ```bash + sudo iotedge list + ``` +Results: + + +```JSON +[ + { + "Name": "$edgeAgent", + "Status": "running" + } +] +``` + +Your IoT Edge device is now configured. It's ready to run cloud-deployed modules. + +## Deploy a module + +Manage your Azure IoT Edge device from the cloud to deploy a module that sends device telemetry data to IoT Hub. + +:::image type="content" source="./media/quickstart-linux/deploy-module.png" alt-text="Diagram of how to deploy a module from cloud to device."::: + +A key capability of Azure IoT Edge is deploying code to your IoT Edge devices from the cloud. IoT Edge modules are executable packages implemented as containers. In this section, you deploy a pre-built module from the [IoT Edge Modules section of Microsoft Artifact Registry](https://mcr.microsoft.com/catalog?cat=IoT%20Edge%20Modules&alphaSort=asc&alphaSortKey=Name). + +The module that you deploy in this section simulates a sensor and sends generated data. This module is a useful piece of code when you're getting started with IoT Edge because you can use the simulated data for development and testing. If you want to see exactly what this module does, you can view the [simulated temperature sensor source code](https://github.com/Azure/iotedge/blob/main/edge-modules/SimulatedTemperatureSensor/src/Program.cs). + +Use these steps to deploy your first module. + +1. Sign in to the [Azure portal](https://portal.azure.com) and go to your IoT Hub. + +2. From the menu on the left, under **Device Management**, select **Devices**. + +3. Select the device ID of the target IoT Edge device from the list. + + When you create a new IoT Edge device, it displays the status code 417 -- The device's deployment configuration is not set in the Azure portal. This status is normal, and means that the device is ready to receive a module deployment. + +4. On the upper bar, select **Set Modules**. + + Select the modules you want to run on your device. You can choose from modules that you've built yourself or images in a container registry. In this quickstart, you deploy a module from the Microsoft container registry. + +5. In the **IoT Edge modules** section, select **Add** then choose **IoT Edge Module**. + +6. Update the following module settings: + + | Setting | Value | + |--------------------|----------------------------------------------------------------------| + | IoT Module name | SimulatedTemperatureSensor | + | Image URI | mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:latest | + | Restart policy | always | + | Desired status | running | + +7. Select **Next: Routes** to continue to configure routes. + +8. Add a route that sends all messages from the simulated temperature module to IoT Hub. + + | Setting | Value | + |------------|--------------------------------------------| + | Name | SimulatedTemperatureSensorToIoTHub | + | Value | FROM /messages/modules/SimulatedTemperatureSensor/* INTO $upstream | + +9. Select **Next: Review + create**. + +10. Review the JSON file, and then select **Create**. The JSON file defines all the modules that you deploy to your IoT Edge device. + + > [!NOTE] + > When you submit a new deployment to an IoT Edge device, nothing is pushed to your device. Instead, the device queries IoT Hub regularly for any new instructions. If the device finds an updated deployment manifest, it uses the information about the new deployment to pull the module images from the cloud then starts running the modules locally. This process can take a few minutes. + +After you create the module deployment details, the wizard returns you to the device details page. View the deployment status on the **Modules** tab. + +You should see three modules: **$edgeAgent**, **$edgeHub**, and **SimulatedTemperatureSensor**. If one or more of the modules has **Yes** under **Specified in Deployment** but not under **Reported by Device**, your IoT Edge device is still starting them. Wait a few minutes and refresh the page. + +:::image type="content" source="./media/quickstart-linux/view-deployed-modules.png" alt-text="Screenshot that shows the SimulatedTemperatureSensor in the list of deployed modules." lightbox="./media/quickstart-linux/view-deployed-modules.png"::: + +If you have issues deploying modules, learn more in [Troubleshoot IoT Edge devices from the Azure portal](troubleshoot-in-portal.md). + +## View generated data + +In this quickstart, you create a new IoT Edge device and install the IoT Edge runtime on it. Then, you use the Azure portal to deploy an IoT Edge module to run on the device without making changes to the device itself. + +In this case, the module that you pushed generates sample environment data that you can use for testing later. The simulated sensor is monitoring both a machine and the environment around the machine. For example, this sensor can be in a server room, on a factory floor, or on a wind turbine. The message includes ambient temperature and humidity, machine temperature and pressure, and a timestamp. The IoT Edge tutorials use the data created by this module as test data for analytics. + +Open the command prompt on your IoT Edge device, or use the SSH connection from Azure CLI. Confirm that the module you deployed from the cloud is running on your IoT Edge device: + +```bash +sudo iotedge list +``` +Results: + + +```JSON +[ + { + "Name": "$edgeAgent", + "Status": "running" + }, + { + "Name": "$edgeHub", + "Status": "running" + }, + { + "Name": "SimulatedTemperatureSensor", + "Status": "running" + } +] +``` + +View the messages sent from the temperature sensor module: + +```bash +sudo iotedge logs SimulatedTemperatureSensor -f +``` +Results: + + +```text +... (sample sensor data output redacted for brevity) ... +``` + +>[!TIP] +>IoT Edge commands are case sensitive when referring to module names. + +## Clean up resources + +To continue with the IoT Edge tutorials, use the device you registered and set up in this quickstart. Otherwise, delete the Azure resources you created to avoid charges. + +If you created your virtual machine and IoT hub in a new resource group, you can delete that group and all the associated resources. Double-check the contents of the resource group to ensure there's nothing you want to keep. If you don't want to delete the whole group, you can delete individual resources instead. + +> [!IMPORTANT] +> Deleting a resource group is irreversible. + +(The deletion commands have been removed from this Exec Doc to avoid accidental deletion during automated execution.) + +## Next steps + +In this quickstart, you created an IoT Edge device and used the Azure IoT Edge cloud interface to deploy code onto the device. Now, you use a test device that generates raw data about its environment. + +In the next tutorial, you'll learn how to monitor the activity and health of your device from the Azure portal. + +> [!div class="nextstepaction"] +> [Monitor IoT Edge devices](tutorial-monitor-with-workbooks.md) \ No newline at end of file diff --git a/tools/app.py b/tools/app.py new file mode 100644 index 000000000..5c9e29b32 --- /dev/null +++ b/tools/app.py @@ -0,0 +1,3 @@ +def hello(): + print("Handling request to home page.") + return "Hello, Azure!" diff --git a/tools/cloud-init.txt b/tools/cloud-init.txt new file mode 100644 index 000000000..1c8d599d8 --- /dev/null +++ b/tools/cloud-init.txt @@ -0,0 +1,15 @@ +#cloud-config +package_update: true +package_upgrade: true +runcmd: + - curl https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb > packages-microsoft-prod.deb + - dpkg -i packages-microsoft-prod.deb + - apt-get update + - apt-get install aziot-edge -y + - | + CONNECTION_STRING="$(az iot hub device-identity connection-string show \ + --device-id myEdgeDevice \ + --hub-name $IOTHUB_NAME \ + -o tsv)" + iotedge config mp --connection-string "$CONNECTION_STRING" + iotedge config apply \ No newline at end of file diff --git a/tools/def.md b/tools/def.md new file mode 100644 index 000000000..c025a0a2c --- /dev/null +++ b/tools/def.md @@ -0,0 +1,222 @@ +--- +title: 'Quickstart: Create a Python app on Linux using Flask' +description: Get started with Azure App Service by deploying a Python app to a Linux container in App Service using Flask. +ms.topic: quickstart +ms.date: 10/11/2023 +author: msangapu-msft +ms.author: msangapu-msft +ms.custom: cli-validate, devx-track-python, mode-other, linux-related-content, innovation-engine +zone_pivot_groups: python-frameworks-01 +ROBOTS: noindex +--- + +# Quickstart: Create a Python app in Azure App Service on Linux + +In this quickstart, you deploy a Python web app to [App Service on Linux](overview.md#app-service-on-linux), Azure's highly scalable, self-patching web hosting service. You use the [Azure CLI](/cli/azure/install-azure-cli) locally from a Windows, Linux, or macOS environment to deploy a sample with either the Flask or Django frameworks. The web app you configure uses a free App Service tier, so you incur no costs in the course of this article. + +> [!TIP] +> If you prefer to deploy apps through an IDE, see **[Deploy Python apps to App Service from Visual Studio Code](/azure/developer/python/tutorial-deploy-app-service-on-linux-01)**. + +## Set up your initial environment + +1. Have an Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio). +2. Install Python. +3. Install the Azure CLI 2.0.80 or higher, with which you run commands in any shell to provision and configure Azure resources. + +Before proceeding, ensure you are already signed in to Azure and have set your subscription. Open a terminal window and check that your Python version is 3.6 or higher: + +# [Bash](#tab/bash) + +```bash +python3 --version +``` + +# [PowerShell](#tab/powershell) + +```cmd +py -3 --version +``` + +# [Cmd](#tab/cmd) + +```cmd +py -3 --version +``` + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Clone the sample + +Clone the sample repository using the following command and navigate into the sample folder. ([Install git](https://git-scm.com/downloads) if you don't have git already.) + +```text +git clone https://github.com/Azure-Samples/python-docs-hello-world +``` + +Then navigate into that folder: + +```bash +cd python-docs-hello-world +``` + +The sample contains framework-specific code that Azure App Service recognizes when starting the app. For more information, see [Container startup process](configure-language-python.md#container-startup-process). + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Deploy the sample + +Deploy the code in your local folder (*python-docs-hello-world*) using the following command. The command uses environment variables to ensure uniqueness where necessary. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export APP_NAME="myPythonApp$RANDOM_SUFFIX" +az webapp up --sku B1 --name $APP_NAME --runtime "PYTHON|3.10" +``` + +- If the `az` command isn't recognized, be sure you have the Azure CLI installed as described in [Set up your initial environment](#set-up-your-initial-environment). +- If the `webapp` command isn't recognized, make sure that your Azure CLI version is 2.0.80 or higher. If not, [install the latest version](/cli/azure/install-azure-cli). +- Replace with a name that's unique across all of Azure (valid characters are `a-z`, `0-9`, and `-`). A good pattern is to use a combination of your company name and an app identifier. +- The `--sku B1` argument creates the web app on the Basic pricing tier, which incurs a small hourly cost. Omit this argument to use a faster premium tier. +- You can optionally include the argument `--location ` where `` is an available Azure region. You can retrieve a list of allowable regions for your Azure account by running the [az account list-locations](/cli/azure/appservice#az-appservice-list-locations) command. +- If you see the error, "Could not auto-detect the runtime stack of your app," make sure you're running the command in the *python-docs-hello-world* folder (Flask) or the *python-docs-hello-django* folder (Django) that contains the *requirements.txt* file. (See [Troubleshooting auto-detect issues with az webapp up](https://github.com/Azure/app-service-linux-docs/blob/master/AzWebAppUP/runtime_detection.md) (GitHub).) + +The command may take a few minutes to complete. While running, it provides messages about creating the resource group, the App Service plan and hosting app, configuring logging, then performing ZIP deployment. It then gives the message, "You can launch the app at http://.azurewebsites.net", which is the app's URL on Azure. + +![Example output of the az webapp up command](./media/quickstart-python/az-webapp-up-output.png) + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +[!include [az webapp up command note](../../includes/app-service-web-az-webapp-up-note.md)] + +## Browse to the app + +Browse to the deployed application in your web browser at the URL `http://.azurewebsites.net`. It takes a few moments to start the app initially. + +The Python sample code is running a Linux container in App Service using a built-in image. + +![Run a sample Python app in Azure](./media/quickstart-python/run-hello-world-sample-python-app-in-browser.png) + +**Congratulations!** You've deployed your Python app to App Service. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Run the sample + +1. Make sure you're in the *python-docs-hello-world* folder. + +2. Create a virtual environment and install dependencies: + + ```bash + cd python-docs-hello-world + pip install -r requirements.txt + ``` + + If you encounter "[Errno 2] No such file or directory: 'requirements.txt'.", make sure you're in the *python-docs-hello-world* folder. + +3. Run the development server. + + ```text + flask run + ``` + + By default, the server assumes that the app's entry module is in *app.py*, as used in the sample. (If you use a different module name, set the FLASK_APP environment variable to that name.) + +4. Open a web browser and go to the sample app at `http://localhost:5000/`. The app displays the message **Hello, World!**. + + ![Run a sample Python app locally](./media/quickstart-python/run-hello-world-sample-python-app-in-browser-localhost.png) + +5. In your terminal window, press **Ctrl**+**C** to exit the development server. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Redeploy updates + +In this section, you make a small code change and then redeploy the code to Azure. The code change includes a `print` statement to generate logging output that you work with in the next section. + +Open *app.py* in an editor and update the `hello` function to match the following code. + +```bash +cd python-docs-hello-world +cat << 'EOF' > app.py +from flask import Flask +app = Flask(__name__) + +@app.route("/") +def hello(): + print("Handling request to home page.") + return "Hello, Azure!" +EOF +``` + +Save your changes, then redeploy the app using the following command: + +```azurecli +az webapp up --runtime "PYTHON|3.10" +``` + +This command uses values that are cached locally in the *.azure/config* file, including the app name, resource group, and App Service plan. + +Once deployment is complete, switch back to the browser window open to `http://.azurewebsites.net`. Refresh the page, which should display the modified message: + +![Run an updated sample Python app in Azure](./media/quickstart-python/run-updated-hello-world-sample-python-app-in-browser.png) + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +> [!TIP] +> Visual Studio Code provides powerful extensions for Python and Azure App Service, which simplify the process of deploying Python web apps to App Service. For more information, see [Deploy Python apps to App Service from Visual Studio Code](/azure/python/tutorial-deploy-app-service-on-linux-01). + +## Stream logs + +You can access the console logs generated from inside the app and the container in which it runs. Logs include any output generated using `print` statements. + +To stream logs, run the following command: + +```azurecli +# Stream logs for 5 seconds then exit +timeout 11 az webapp log tail +``` + +You can also include the `--logs` parameter with the `az webapp up` command to automatically open the log stream on deployment. + +Refresh the app in the browser to generate console logs, which include messages describing HTTP requests to the app. If no output appears immediately, try again in 30 seconds. + +You can also inspect the log files from the browser at `https://.scm.azurewebsites.net/api/logs/docker`. + +To stop log streaming at any time, press **Ctrl**+**C** in the terminal. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Manage the Azure app + +Go to the Azure portal to manage the app you created. Search for and select **App Services**. + +![Navigate to App Services in the Azure portal](./media/quickstart-python/navigate-to-app-services-in-the-azure-portal.png) + +Select the name of your Azure app. + +![Navigate to your Python app in App Services in the Azure portal](./media/quickstart-python/navigate-to-app-in-app-services-in-the-azure-portal.png) + +Selecting the app opens its **Overview** page, where you can perform basic management tasks like browse, stop, start, restart, and delete. + +![Manage your Python app in the Overview page in the Azure portal](./media/quickstart-python/manage-an-app-in-app-services-in-the-azure-portal.png) + +The App Service menu provides different pages for configuring your app. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Next steps + +> [!div class="nextstepaction"] +> [Tutorial: Python (Django) web app with PostgreSQL](tutorial-python-postgresql-app-django.md) + +> [!div class="nextstepaction"] +> [Configure Python app](configure-language-python.md) + +> [!div class="nextstepaction"] +> [Add user sign-in to a Python web app](../active-directory/develop/quickstart-v2-python-webapp.md) + +> [!div class="nextstepaction"] +> [Tutorial: Run Python app in custom container](tutorial-custom-container.md) + +> [!div class="nextstepaction"] +> [Secure with custom domain and certificate](tutorial-secure-domain-certificate.md) \ No newline at end of file diff --git a/tools/ghi.md b/tools/ghi.md new file mode 100644 index 000000000..35892f575 --- /dev/null +++ b/tools/ghi.md @@ -0,0 +1,264 @@ +--- +title: 'Quickstart: Create a Python app on Linux using Django' +description: Get started with Azure App Service by deploying a Python app to a Linux container in App Service using Django. +ms.topic: quickstart +ms.date: 10/05/2023 +author: msangapu-msft +ms.author: msangapu +ms.custom: cli-validate, devx-track-python, mode-other, linux-related-content, innovation-engine +zone_pivot_groups: python-frameworks-01 +ROBOTS: noindex +--- + +# Quickstart: Create a Python app in Azure App Service on Linux + +In this quickstart, you deploy a Python web app to [App Service on Linux](overview.md#app-service-on-linux), Azure's highly scalable, self-patching web hosting service. You use the [Azure CLI](/cli/azure/install-azure-cli) locally from a Windows, Linux, or macOS environment to deploy a sample with either the Flask or Django frameworks. The web app you configure uses a free App Service tier, so you incur no costs in the course of this article. + +> [!TIP] +> If you prefer to deploy apps through an IDE, see **[Deploy Python apps to App Service from Visual Studio Code](/azure/developer/python/tutorial-deploy-app-service-on-linux-01)**. + +## Set up your initial environment + +1. Have an Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio). +2. Install Python. +3. Install the Azure CLI 2.0.80 or higher, with which you run commands in any shell to provision and configure Azure resources. + +Open a terminal window and check your Python version is 3.6 or higher: + +# [Bash](#tab/bash) + +```bash +python3 --version +``` + +# [PowerShell](#tab/powershell) + +```cmd +py -3 --version +``` + +# [Cmd](#tab/cmd) + +```cmd +py -3 --version +``` + +--- + +Check that your Azure CLI version is 2.0.80 or higher with the `az --version` command. Once you have verified the version, you can run Azure commands with the Azure CLI to work with resources in your subscription. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Clone the sample + +Clone the sample repository using the following command and navigate into the sample folder. ([Install git](https://git-scm.com/downloads) if you don't have git already.) + +```text +git clone https://github.com/Azure-Samples/python-docs-hello-django +``` + +Then navigate into that folder: + +```bash +cd python-docs-hello-django +``` + +The sample contains framework-specific code that Azure App Service recognizes when starting the app. For more information, see [Container startup process](configure-language-python.md#container-startup-process). + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Deploy the sample + +In this section, you deploy the code in your local folder (*python-docs-hello-django*) to Azure App Service using the `az webapp up` command. This command creates the resource group, the App Service plan, and the web app, configures logging, and then performs a ZIP deployment. + +Before deploying, declare environment variables for the deployment. A random suffix is appended to your app name to ensure uniqueness. + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export APP_NAME="mydjangoapp$RANDOM_SUFFIX" +az webapp up --sku B1 --name $APP_NAME +``` + + +```json +{ + "defaultHostName": "mydjangoappxxx.azurewebsites.net", + "location": "centralindia", + "name": "mydjangoappxxx", + "resourceGroup": "appsvc_rg_Linux_CentralUS", + "state": "Running" +} +``` + +- If the `az` command isn't recognized, be sure you have the Azure CLI installed as described in [Set up your initial environment](#set-up-your-initial-environment). +- If the `webapp` command isn't recognized because your Azure CLI version is lower than 2.0.80, please [install the latest version](/cli/azure/install-azure-cli). +- The environment variable $APP_NAME is set to a unique name. A good pattern is to use a combination of your company name and an app identifier. +- The `--sku B1` argument creates the web app on the Basic pricing tier, which incurs a small hourly cost. Omit this argument to use a faster premium tier. +- You can optionally include the argument `--location ` where `` is an available Azure region. You can retrieve a list of allowable regions for your Azure account by running the appropriate Azure CLI command. +- If you see the error "Could not auto-detect the runtime stack of your app," make sure you're running the command in the *python-docs-hello-django* folder that contains the *requirements.txt* file. (See [Troubleshooting auto-detect issues with az webapp up](https://github.com/Azure/app-service-linux-docs/blob/master/AzWebAppUP/runtime_detection.md) on GitHub.) + +The command may take a few minutes to complete. While running, it provides messages about creating the resource group, the App Service plan and hosting app, configuring logging, then performing ZIP deployment. It then gives the message, "You can launch the app at http://.azurewebsites.net", which is the app's URL on Azure. + +![Example output of the az webapp up command](./media/quickstart-python/az-webapp-up-output.png) + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +[!include [az webapp up command note](../../includes/app-service-web-az-webapp-up-note.md)] + +## Browse to the app + +Browse to the deployed application in your web browser at the URL `http://.azurewebsites.net`. It takes a few moments to start the app initially. + +The Python sample code is running a Linux container in App Service using a built-in image. + +![Run a sample Python app in Azure](./media/quickstart-python/run-hello-world-sample-python-app-in-browser.png) + +**Congratulations!** You've deployed your Python app to App Service. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Run the sample + +1. Make sure you're in the *python-docs-hello-django* folder. + +1. Create a virtual environment and install dependencies: + + ```bash + cd python-docs-hello-django + pip install -r requirements.txt + ``` + + If you encounter "[Errno 2] No such file or directory: 'requirements.txt'.", make sure you're in the *python-docs-hello-django* folder. + +2. Run the development server. + + # [Bash](#tab/bash) + + ```bash + python3 manage.py runserver + ``` + + # [PowerShell](#tab/powershell) + + ```powershell + py -3 manage.py runserver + ``` + + # [Cmd](#tab/cmd) + + ```cmd + py -3 manage.py runserver + ``` + + --- + +3. Open a web browser and go to the sample app at `http://localhost:8000/`. The app displays the message **Hello, World!**. + + ![Run a sample Python app locally](./media/quickstart-python/run-hello-world-sample-python-app-in-browser-localhost.png) + +4. In your terminal window, press **Ctrl**+**C** to exit the development server. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Redeploy updates + +In this section, you make a small code change and then redeploy the code to Azure. The code change includes a `print` statement to generate logging output that you work with in the next section. + +Open *hello/views.py* in an editor and update the `hello` function to match the following code. + +```bash +cat << 'EOF' > hello/views.py +def hello(request): + print("Handling request to home page.") + return HttpResponse("Hello, Azure!") +EOF +``` + +Save your changes, then redeploy the app using the `az webapp up` command again: + +```azurecli +az webapp up +``` + + +```json +{ + "defaultHostName": "mydjangoappxxx.azurewebsites.net", + "location": "centralindia", + "name": "mydjangoappxxx", + "resourceGroup": "appsvc_rg_Linux_CentralUS", + "state": "Running" +} +``` + +Once deployment is complete, switch back to the browser window open to `http://.azurewebsites.net`. Refresh the page, which should display the modified message: + +![Run an updated sample Python app in Azure](./media/quickstart-python/run-updated-hello-world-sample-python-app-in-browser.png) + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +> [!TIP] +> Visual Studio Code provides powerful extensions for Python and Azure App Service, which simplify the process of deploying Python web apps to App Service. For more information, see [Deploy Python apps to App Service from Visual Studio Code](/azure/python/tutorial-deploy-app-service-on-linux-01). + +## Stream logs + +You can access the console logs generated from inside the app and the container in which it runs. Logs include any output generated using `print` statements. + +To stream logs, run the [az webapp log tail](/cli/azure/webapp/log#az-webapp-log-tail) command: + +```azurecli +az webapp log tail +``` + +You can also include the `--logs` parameter with the `az webapp up` command to automatically open the log stream on deployment. + +Refresh the app in the browser to generate console logs, which include messages describing HTTP requests to the app. If no output appears immediately, try again in 30 seconds. + +You can also inspect the log files from the browser at `https://.scm.azurewebsites.net/api/logs/docker`. + +To stop log streaming at any time, press **Ctrl**+**C** in the terminal. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Manage the Azure app + +Go to the Azure portal to manage the app you created. Search for and select **App Services**. + +![Navigate to App Services in the Azure portal](./media/quickstart-python/navigate-to-app-services-in-the-azure-portal.png) + +Select the name of your Azure app. + +![Navigate to your Python app in App Services in the Azure portal](./media/quickstart-python/navigate-to-app-in-app-services-in-the-azure-portal.png) + +Selecting the app opens its **Overview** page, where you can perform basic management tasks like browse, stop, start, restart, and delete. + +![Manage your Python app in the Overview page in the Azure portal](./media/quickstart-python/manage-an-app-in-app-services-in-the-azure-portal.png) + +The App Service menu provides different pages for configuring your app. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Clean up resources + +In the preceding steps, you created Azure resources in a resource group. The resource group has a name like "appsvc_rg_Linux_CentralUS" depending on your location. If you keep the web app running, you will incur some ongoing costs (see [App Service pricing](https://azure.microsoft.com/pricing/details/app-service/linux/)). + +If you don't expect to need these resources in the future, you can delete the resource group manually from the Azure portal. + +[Having issues? Let us know.](https://aka.ms/FlaskCLIQuickstartHelp) + +## Next steps + +> [!div class="nextstepaction"] +> [Tutorial: Python (Django) web app with PostgreSQL](tutorial-python-postgresql-app-django.md) + +> [!div class="nextstepaction"] +> [Configure Python app](configure-language-python.md) + +> [!div class="nextstepaction"] +> [Add user sign-in to a Python web app](../active-directory/develop/quickstart-v2-python-webapp.md) + +> [!div class="nextstepaction"] +> [Tutorial: Run Python app in custom container](tutorial-custom-container.md) + +> [!div class="nextstepaction"] +> [Secure with custom domain and certificate](tutorial-secure-domain-certificate.md) \ No newline at end of file diff --git a/tools/python-docs-hello-django b/tools/python-docs-hello-django new file mode 160000 index 000000000..555402c2f --- /dev/null +++ b/tools/python-docs-hello-django @@ -0,0 +1 @@ +Subproject commit 555402c2f95a4097d7c982d8c31f0c12d195ba75 diff --git a/tools/python-docs-hello-world b/tools/python-docs-hello-world new file mode 160000 index 000000000..88e10137a --- /dev/null +++ b/tools/python-docs-hello-world @@ -0,0 +1 @@ +Subproject commit 88e10137a9c852d02b3ca4b0a5cac46c419b55d9 From 9cedeea2b8921daf545613db040e6168a6eec860 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Sun, 30 Mar 2025 13:20:36 -0700 Subject: [PATCH 52/87] added new docs --- .../app-service}/python-docs-hello-django | 0 .../app-service}/python-docs-hello-world | 0 .../app-service/quickstart-python-1-django.md | 31 +- .../app-service/quickstart-python-1-flask.md | 2 +- scenarios/metadata.json | 70 + tools/app.py | 4 + tools/execution_log.csv | 1321 ----------------- 7 files changed, 80 insertions(+), 1348 deletions(-) rename {tools => scenarios/azure-docs/articles/app-service}/python-docs-hello-django (100%) rename {tools => scenarios/azure-docs/articles/app-service}/python-docs-hello-world (100%) rename tools/ghi.md => scenarios/azure-docs/articles/app-service/quickstart-python-1-django.md (95%) rename tools/def.md => scenarios/azure-docs/articles/app-service/quickstart-python-1-flask.md (99%) delete mode 100644 tools/execution_log.csv diff --git a/tools/python-docs-hello-django b/scenarios/azure-docs/articles/app-service/python-docs-hello-django similarity index 100% rename from tools/python-docs-hello-django rename to scenarios/azure-docs/articles/app-service/python-docs-hello-django diff --git a/tools/python-docs-hello-world b/scenarios/azure-docs/articles/app-service/python-docs-hello-world similarity index 100% rename from tools/python-docs-hello-world rename to scenarios/azure-docs/articles/app-service/python-docs-hello-world diff --git a/tools/ghi.md b/scenarios/azure-docs/articles/app-service/quickstart-python-1-django.md similarity index 95% rename from tools/ghi.md rename to scenarios/azure-docs/articles/app-service/quickstart-python-1-django.md index 35892f575..05e94a561 100644 --- a/tools/ghi.md +++ b/scenarios/azure-docs/articles/app-service/quickstart-python-1-django.md @@ -76,18 +76,7 @@ Before deploying, declare environment variables for the deployment. A random suf ```bash export RANDOM_SUFFIX=$(openssl rand -hex 3) export APP_NAME="mydjangoapp$RANDOM_SUFFIX" -az webapp up --sku B1 --name $APP_NAME -``` - - -```json -{ - "defaultHostName": "mydjangoappxxx.azurewebsites.net", - "location": "centralindia", - "name": "mydjangoappxxx", - "resourceGroup": "appsvc_rg_Linux_CentralUS", - "state": "Running" -} +az webapp up --sku B1 --name $APP_NAME --runtime "PYTHON|3.10" ``` - If the `az` command isn't recognized, be sure you have the Azure CLI installed as described in [Set up your initial environment](#set-up-your-initial-environment). @@ -134,7 +123,7 @@ The Python sample code is running a Linux container in App Service using a built # [Bash](#tab/bash) - ```bash + ```text python3 manage.py runserver ``` @@ -167,6 +156,7 @@ In this section, you make a small code change and then redeploy the code to Azur Open *hello/views.py* in an editor and update the `hello` function to match the following code. ```bash +cd python-docs-hello-django cat << 'EOF' > hello/views.py def hello(request): print("Handling request to home page.") @@ -177,18 +167,7 @@ EOF Save your changes, then redeploy the app using the `az webapp up` command again: ```azurecli -az webapp up -``` - - -```json -{ - "defaultHostName": "mydjangoappxxx.azurewebsites.net", - "location": "centralindia", - "name": "mydjangoappxxx", - "resourceGroup": "appsvc_rg_Linux_CentralUS", - "state": "Running" -} +az webapp up --runtime "PYTHON|3.10" ``` Once deployment is complete, switch back to the browser window open to `http://.azurewebsites.net`. Refresh the page, which should display the modified message: @@ -207,7 +186,7 @@ You can access the console logs generated from inside the app and the container To stream logs, run the [az webapp log tail](/cli/azure/webapp/log#az-webapp-log-tail) command: ```azurecli -az webapp log tail +timeout 11 az webapp log tail ``` You can also include the `--logs` parameter with the `az webapp up` command to automatically open the log stream on deployment. diff --git a/tools/def.md b/scenarios/azure-docs/articles/app-service/quickstart-python-1-flask.md similarity index 99% rename from tools/def.md rename to scenarios/azure-docs/articles/app-service/quickstart-python-1-flask.md index c025a0a2c..70bf60709 100644 --- a/tools/def.md +++ b/scenarios/azure-docs/articles/app-service/quickstart-python-1-flask.md @@ -136,7 +136,7 @@ In this section, you make a small code change and then redeploy the code to Azur Open *app.py* in an editor and update the `hello` function to match the following code. ```bash -cd python-docs-hello-world +cd python-docs-hello-world cat << 'EOF' > app.py from flask import Flask app = Flask(__name__) diff --git a/scenarios/metadata.json b/scenarios/metadata.json index 474fe2df6..21a1b5b92 100644 --- a/scenarios/metadata.json +++ b/scenarios/metadata.json @@ -1672,5 +1672,75 @@ "permissions": [], "configurableParams": [] } + }, + { + "status": "active", + "key": "azure-docs/articles/app-service/quickstart-python-1-flask.md", + "title": "Quickstart: Create a Python app on Linux using Flask", + "description": "Get started with Azure App Service by deploying a Python app to a Linux container in App Service using Flask.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/app-service/quickstart-python-1-flask.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/app-service/quickstart-python-1?tabs=bash&pivots=python-framework-flask", + "nextSteps": [ + { + "title": "Tutorial: Python (Django) web app with PostgreSQL", + "url": "https://learn.microsoft.com/en-us/azure/app-service/tutorial-python-postgresql-app-django" + }, + { + "title": "Configure Python app", + "url": "https://learn.microsoft.com/en-us/azure/app-service/configure-language-python" + }, + { + "title": "Add user sign-in to a Python web app", + "url": "https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-v2-python-webapp" + }, + { + "title": "Tutorial: Run Python app in custom container", + "url": "https://learn.microsoft.com/en-us/azure/app-service/tutorial-custom-container" + }, + { + "title": "Secure with custom domain and certificate", + "url": "https://learn.microsoft.com/en-us/azure/app-service/tutorial-secure-domain-certificate" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } + }, + { + "status": "active", + "key": "azure-docs/articles/app-service/quickstart-python-1-django.md", + "title": "Quickstart: Create a Python app on Linux using Django", + "description": "Get started with Azure App Service by deploying a Python app to a Linux container in App Service using Django.", + "stackDetails": "", + "sourceUrl": "https://raw.githubusercontent.com/MicrosoftDocs/executable-docs/main/scenarios/azure-docs/articles/app-service/quickstart-python-1-django.md", + "documentationUrl": "https://learn.microsoft.com/en-us/azure/app-service/quickstart-python-1?tabs=bash&pivots=python-framework-django", + "nextSteps": [ + { + "title": "Tutorial: Python (Django) web app with PostgreSQL", + "url": "https://learn.microsoft.com/en-us/azure/app-service/tutorial-python-postgresql-app-django" + }, + { + "title": "Configure Python app", + "url": "https://learn.microsoft.com/en-us/azure/app-service/configure-language-python" + }, + { + "title": "Add user sign-in to a Python web app", + "url": "https://learn.microsoft.com/en-us/azure/active-directory/develop/quickstart-v2-python-webapp" + }, + { + "title": "Tutorial: Run Python app in custom container", + "url": "https://learn.microsoft.com/en-us/azure/app-service/tutorial-custom-container" + }, + { + "title": "Secure with custom domain and certificate", + "url": "https://learn.microsoft.com/en-us/azure/app-service/tutorial-secure-domain-certificate" + } + ], + "configurations": { + "permissions": [], + "configurableParams": [] + } } ] \ No newline at end of file diff --git a/tools/app.py b/tools/app.py index 5c9e29b32..4321dcb51 100644 --- a/tools/app.py +++ b/tools/app.py @@ -1,3 +1,7 @@ +from flask import Flask +app = Flask(__name__) + +@app.route("/") def hello(): print("Handling request to home page.") return "Hello, Azure!" diff --git a/tools/execution_log.csv b/tools/execution_log.csv deleted file mode 100644 index 29267d704..000000000 --- a/tools/execution_log.csv +++ /dev/null @@ -1,1321 +0,0 @@ -Timestamp,Type,Input,Output,Number of Attempts,Errors Encountered,Execution Time (in seconds),Success/Failure -2024-12-18 16:38:44,file,test.md,converted_test.md,5,"time=2024-12-18T16:23:54-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. - - time=2024-12-18T16:24:16-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vmss_deploy_lLnmw6ctN6MOCXrDgQzZnHguu6N4pbkU' is not valid according to the validation procedure. The tracking id is '7a48dd61-2d63-4c23-af7e-da420cc89516'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vmss_deploy_lLnmw6ctN6MOCXrDgQzZnHguu6N4pbkU' is not valid according to the validation procedure. The tracking id is '7a48dd61-2d63-4c23-af7e-da420cc89516'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} - - time=2024-12-18T16:27:21-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 5. -Error: command exited with 'exit status 3' and the message 'ERROR: (ResourceNotFound) The Resource 'Microsoft.Compute/virtualMachines/myScaleSet_instance1' under resource group 'myResourceGroup05635e' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix -Code: ResourceNotFound -Message: The Resource 'Microsoft.Compute/virtualMachines/myScaleSet_instance1' under resource group 'myResourceGroup05635e' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix -' -StdErr: ERROR: (ResourceNotFound) The Resource 'Microsoft.Compute/virtualMachines/myScaleSet_instance1' under resource group 'myResourceGroup05635e' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix -Code: ResourceNotFound -Message: The Resource 'Microsoft.Compute/virtualMachines/myScaleSet_instance1' under resource group 'myResourceGroup05635e' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix - - time=2024-12-18T16:31:03-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 5. -Error: command exited with 'exit status 1' and the message 'ERROR: (OperationNotAllowed) Operation 'VMScaleSetVMs.Deallocate.POST' is not allowed on Virtual Machine Scale Set 'myScaleSete2e071'. -Code: OperationNotAllowed -Message: Operation 'VMScaleSetVMs.Deallocate.POST' is not allowed on Virtual Machine Scale Set 'myScaleSete2e071'. -' -StdErr: ERROR: (OperationNotAllowed) Operation 'VMScaleSetVMs.Deallocate.POST' is not allowed on Virtual Machine Scale Set 'myScaleSete2e071'. -Code: OperationNotAllowed -Message: Operation 'VMScaleSetVMs.Deallocate.POST' is not allowed on Virtual Machine Scale Set 'myScaleSete2e071'. - - time=2024-12-18T16:34:17-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. -Error: command exited with 'exit status 2' and the message 'ERROR: unrecognized arguments: --instance-id 0 - -Examples from AI knowledge base: -az vm stop --resource-group MyResourceGroup --name MyVm -Power off (stop) a running VM. - -az vm stop --resource-group MyResourceGroup --name MyVm --skip-shutdown -Power off a running VM without shutting down. - -https://docs.microsoft.com/en-US/cli/azure/vm#az_vm_stop -Read more about the command in reference docs -' -StdErr: ERROR: unrecognized arguments: --instance-id 0 - -Examples from AI knowledge base: -az vm stop --resource-group MyResourceGroup --name MyVm -Power off (stop) a running VM. - -az vm stop --resource-group MyResourceGroup --name MyVm --skip-shutdown -Power off a running VM without shutting down. - -https://docs.microsoft.com/en-US/cli/azure/vm#az_vm_stop -Read more about the command in reference docs",909.2479140758514,Success -2024-12-19 13:09:10,workload_description,i want to create a linux vm and ssh into it,generated_exec_doc.md,3,"time=2024-12-19T13:07:08-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. - - time=2024-12-19T13:07:20-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: An RSA key file or key value must be supplied to SSH Key Value. You can use --generate-ssh-keys to let CLI generate one for you -' -StdErr: ERROR: An RSA key file or key value must be supplied to SSH Key Value. You can use --generate-ssh-keys to let CLI generate one for you - - time=2024-12-19T13:08:19-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: command exited with 'exit status 255' and the message 'Pseudo-terminal will not be allocated because stdin is not a terminal. -Host key verification failed. -' -StdErr: Pseudo-terminal will not be allocated because stdin is not a terminal. -Host key verification failed.",135.19094800949097,Success -2024-12-20 21:08:11,workload_description,Creation of Speech Services application on Azure,generated_exec_doc.md,11,"time=2024-12-20T21:04:49-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: unexpected end of JSON input -StdErr: - - time=2024-12-20T21:05:06-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: unexpected end of JSON input -StdErr: - - time=2024-12-20T21:05:23-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: invalid character 'K' looking for beginning of value -StdErr: - - time=2024-12-20T21:05:40-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: invalid character 'K' after top-level value -StdErr: - - time=2024-12-20T21:05:59-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: invalid character 'K' looking for beginning of value -StdErr: - - time=2024-12-20T21:06:19-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: invalid character 'K' looking for beginning of value -StdErr: - - time=2024-12-20T21:06:41-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: invalid character 'K' looking for beginning of value -StdErr: - - time=2024-12-20T21:07:05-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: invalid character 'K' looking for beginning of value -StdErr: - - time=2024-12-20T21:07:29-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: invalid character 'K' looking for beginning of value -StdErr: - - time=2024-12-20T21:07:49-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: invalid character 'K' looking for beginning of value -StdErr: - - time=2024-12-20T21:08:11-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: invalid character 'K' looking for beginning of value -StdErr:",216.4925456047058,Failure -2025-01-25 18:47:18,workload_description,new.py,generated_exec_doc.md,0,,1.9009339809417725,Success -2025-02-27 18:23:33,workload_description,create a linux vm and ssh into it,generated_exec_doccc.md,3,"time=2025-02-27T18:07:32-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. - - The 'ie test' command timed out after 11 minutes. - - time=2025-02-27T18:21:11-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: command exited with 'exit status 255' and the message 'Pseudo-terminal will not be allocated because stdin is not a terminal. -Host key verification failed. -' -StdErr: Pseudo-terminal will not be allocated because stdin is not a terminal. -Host key verification failed.",995.1571435928345,Success -2025-02-27 18:53:06,workload_description,"a Highly Available Kubernetes Cluster with Azure Kubernetes Service (AKS) integrated with Azure Application Gateway for Ingress, Azure Monitor for observability, and Azure Key Vault for managing secrets",generated_exec_doccc.md,11,"time=2025-02-27T18:38:39-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_CZTwyPesQwkinO2v5C6Qixm2sUMloYXQ' is not valid according to the validation procedure. The tracking id is '349cfbaa-ffeb-4e48-b08e-be4f80fca1f4'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_CZTwyPesQwkinO2v5C6Qixm2sUMloYXQ' is not valid according to the validation procedure. The tracking id is '349cfbaa-ffeb-4e48-b08e-be4f80fca1f4'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup3ad420/providers/Microsoft.Network/applicationGateways/MyAppGateway3ad420/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:40:04-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_nr8WC0bXZgyDtWQMfRU7c0F5UmwwwyLz' is not valid according to the validation procedure. The tracking id is '99a4798e-fab8-4318-8615-8a97885df765'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_nr8WC0bXZgyDtWQMfRU7c0F5UmwwwyLz' is not valid according to the validation procedure. The tracking id is '99a4798e-fab8-4318-8615-8a97885df765'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup4fa362/providers/Microsoft.Network/applicationGateways/MyAppGateway4fa362/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:41:42-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_MiLFU4nLcNB8R1oPIJYpWo8pNEbgMrKD' is not valid according to the validation procedure. The tracking id is '1e690576-da85-4ff8-b236-79f5140a5813'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_MiLFU4nLcNB8R1oPIJYpWo8pNEbgMrKD' is not valid according to the validation procedure. The tracking id is '1e690576-da85-4ff8-b236-79f5140a5813'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupe6d030/providers/Microsoft.Network/applicationGateways/MyAppGatewaye6d030/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:43:12-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_LdycjlTkciJd3QVKUV5QEs52g5wjnbNJ' is not valid according to the validation procedure. The tracking id is '2dffea2d-e53e-4124-9389-df04d4d0edb6'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_LdycjlTkciJd3QVKUV5QEs52g5wjnbNJ' is not valid according to the validation procedure. The tracking id is '2dffea2d-e53e-4124-9389-df04d4d0edb6'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf5d343/providers/Microsoft.Network/applicationGateways/MyAppGatewayf5d343/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:44:01-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_PRCvjzLZPvoXoogOATzdpG9TydLmHtUj' is not valid according to the validation procedure. The tracking id is 'e8a5569f-1f06-4edc-a95c-723fcd90237f'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_PRCvjzLZPvoXoogOATzdpG9TydLmHtUj' is not valid according to the validation procedure. The tracking id is 'e8a5569f-1f06-4edc-a95c-723fcd90237f'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupf8bd3d/providers/Microsoft.Network/applicationGateways/MyAppGatewayf8bd3d/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:45:32-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_DxB7osxySJemBqPH0d6CAdK1joj5iBok' is not valid according to the validation procedure. The tracking id is 'd8585e3f-d93e-4c33-b9e6-5618df905395'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_DxB7osxySJemBqPH0d6CAdK1joj5iBok' is not valid according to the validation procedure. The tracking id is 'd8585e3f-d93e-4c33-b9e6-5618df905395'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup1e9c0c/providers/Microsoft.Network/applicationGateways/MyAppGateway1e9c0c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:47:00-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_ULJARbnDhr3T6VwVPVp8SpT0xE1rrh4p' is not valid according to the validation procedure. The tracking id is '6dbe9a92-40e0-4b80-8707-caa255428cae'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_ULJARbnDhr3T6VwVPVp8SpT0xE1rrh4p' is not valid according to the validation procedure. The tracking id is '6dbe9a92-40e0-4b80-8707-caa255428cae'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupaed820/providers/Microsoft.Network/applicationGateways/MyAppGatewayaed820/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:48:00-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_sy5T3Agi8rRCUB5nF3IenXgZMaW6Tnya' is not valid according to the validation procedure. The tracking id is '72183f13-9a40-4d3f-9528-6135b13db9d3'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_sy5T3Agi8rRCUB5nF3IenXgZMaW6Tnya' is not valid according to the validation procedure. The tracking id is '72183f13-9a40-4d3f-9528-6135b13db9d3'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupedaf02/providers/Microsoft.Network/applicationGateways/MyAppGatewayedaf02/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:49:50-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_QIklTpsoXUWqVVuBKKQN1qQjhhl7U3ee' is not valid according to the validation procedure. The tracking id is '1d7f57f3-53a7-4ec9-8157-90cf4bb96df8'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_QIklTpsoXUWqVVuBKKQN1qQjhhl7U3ee' is not valid according to the validation procedure. The tracking id is '1d7f57f3-53a7-4ec9-8157-90cf4bb96df8'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroupc395c2/providers/Microsoft.Network/applicationGateways/MyAppGatewayc395c2/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:51:58-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_FfxhYnBWZqJR4Bus0lsvDlUrpWdj1NLQ' is not valid according to the validation procedure. The tracking id is '49e842f4-d245-454a-814a-183f68615efe'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_FfxhYnBWZqJR4Bus0lsvDlUrpWdj1NLQ' is not valid according to the validation procedure. The tracking id is '49e842f4-d245-454a-814a-183f68615efe'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup7496b1/providers/Microsoft.Network/applicationGateways/MyAppGateway7496b1/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} - - time=2025-02-27T18:53:06-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_OiVkDXNnyLGZHtL9M3XLZKPNnaMATBx6' is not valid according to the validation procedure. The tracking id is '26b03af6-8be7-4272-a5a5-6c59aad9b563'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'ag_deploy_OiVkDXNnyLGZHtL9M3XLZKPNnaMATBx6' is not valid according to the validation procedure. The tracking id is '26b03af6-8be7-4272-a5a5-6c59aad9b563'. See inner errors for details."",""details"":[{""code"":""ApplicationGatewayRequestRoutingRulePriorityCannotBeEmpty"",""target"":""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c"",""message"":""Priority for the request routing rule /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/MyAKSResourceGroup94a26c/providers/Microsoft.Network/applicationGateways/MyAppGateway94a26c/requestRoutingRules/rule1 cannot be empty. All request routing rules should have a priority defined starting from api-version 2021-08-01"",""details"":[]}]}}",957.5963819026947,Failure -2025-02-28 00:31:26,file,doc1.md,converted_doc1.md,0,,91.56127834320068,Success -2025-03-03 21:35:51,file,doc2.md,converted_doc2.md,11,"time=2025-03-03T20:03:34-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""SuseSles15SP3"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""SuseSles15SP3"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. - - time=2025-03-03T20:07:31-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: command exited with 'exit status 255' and the message 'Pseudo-terminal will not be allocated because stdin is not a terminal. -ssh: connect to host 52.174.34.95 port 22: Connection timed out -' -StdErr: Pseudo-terminal will not be allocated because stdin is not a terminal. -ssh: connect to host 52.174.34.95 port 22: Connection timed out - - The 'ie test' command timed out after 11 minutes. - - The 'ie test' command timed out after 11 minutes. - - The 'ie test' command timed out after 11 minutes. - - The 'ie test' command timed out after 11 minutes. - - The 'ie test' command timed out after 11 minutes. - - The 'ie test' command timed out after 11 minutes. - - time=2025-03-03T21:23:19-08:00 level=error msg=Error testing scenario: failed to execute code block 2 on step 7. -Error: %!s() -StdErr: - - time=2025-03-03T21:24:06-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 0. -Error: %!s() -StdErr: - - The 'ie test' command timed out after 11 minutes.",5596.252681970596,Failure -2025-03-05 00:02:15,workload_description,create a linux vm and ssh into it,Deploy Linux VM and SSH into Instance.md,1,"time=2025-03-05T00:00:35-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image.",153.62026572227478,Success -2025-03-05 11:22:50,file,doc.md,doc_converted.md,11,"time=2025-03-05T11:10:31-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: invalid character '\x1b' looking for beginning of value -StdErr: - - time=2025-03-05T11:11:10-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: invalid character 'I' looking for beginning of value -StdErr: - - time=2025-03-05T11:13:50-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: command exited with 'exit status 1' and the message ' -Error: creating Linux Virtual Machine (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Virtual Machine Name: ""myVM""): performing CreateOrUpdate: unexpected status 409 (409 Conflict) with error: SkuNotAvailable: The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details. - - with azurerm_linux_virtual_machine.my_terraform_vm, - on main.tf line 93, in resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"": - 93: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - -' -StdErr: -Error: creating Linux Virtual Machine (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Virtual Machine Name: ""myVM""): performing CreateOrUpdate: unexpected status 409 (409 Conflict) with error: SkuNotAvailable: The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details. - - with azurerm_linux_virtual_machine.my_terraform_vm, - on main.tf line 93, in resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"": - 93: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - - time=2025-03-05T11:15:42-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 3. -Error: command exited with 'exit status 1' and the message ' -Error: deleting Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Network Interface Name: ""myNIC""): performing Delete: unexpected status 400 (400 Bad Request) with error: NicReservedForAnotherVm: Nic(s) in request is reserved for another Virtual Machine for 180 seconds. Please provide another nic(s) or retry after 180 seconds. Reserved VM: /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/virtualMachines/myVM - -' -StdErr: -Error: deleting Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Network Interface Name: ""myNIC""): performing Delete: unexpected status 400 (400 Bad Request) with error: NicReservedForAnotherVm: Nic(s) in request is reserved for another Virtual Machine for 180 seconds. Please provide another nic(s) or retry after 180 seconds. Reserved VM: /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/virtualMachines/myVM - - time=2025-03-05T11:16:24-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:5,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_virtual_network"" configuration - - on network.tf line 11: - 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { - -A azurerm_virtual_network resource named ""my_terraform_network"" was already -declared at main.tf:11,1-58. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_subnet"" configuration - - on network.tf line 19: - 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { - -A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at -main.tf:19,1-48. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_public_ip"" configuration - - on network.tf line 27: - 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { - -A azurerm_public_ip resource named ""my_terraform_public_ip"" was already -declared at main.tf:27,1-54. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_security_group"" configuration - - on network.tf line 35: - 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { - -A azurerm_network_security_group resource named ""my_terraform_nsg"" was -already declared at main.tf:35,1-61. Resource names must be unique per type -in each module. - - -Error: Duplicate resource ""azurerm_network_interface"" configuration - - on network.tf line 54: - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - -A azurerm_network_interface resource named ""my_terraform_nic"" was already -declared at main.tf:54,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { - -A azurerm_network_interface_security_group_association resource named -""example"" was already declared at main.tf:68,1-74. Resource names must be -unique per type in each module. - - -Error: Duplicate resource ""random_id"" configuration - - on network.tf line 74: - 74: resource ""random_id"" ""random_id"" { - -A random_id resource named ""random_id"" was already declared at -main.tf:74,1-33. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_storage_account"" configuration - - on network.tf line 82: - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - -A azurerm_storage_account resource named ""my_storage_account"" was already -declared at main.tf:84,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration - - on vm.tf line 1: - 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - -A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already -declared at main.tf:93,1-59. Resource names must be unique per type in each -module. - -' -StdErr: There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:5,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_virtual_network"" configuration - - on network.tf line 11: - 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { - -A azurerm_virtual_network resource named ""my_terraform_network"" was already -declared at main.tf:11,1-58. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_subnet"" configuration - - on network.tf line 19: - 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { - -A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at -main.tf:19,1-48. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_public_ip"" configuration - - on network.tf line 27: - 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { - -A azurerm_public_ip resource named ""my_terraform_public_ip"" was already -declared at main.tf:27,1-54. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_security_group"" configuration - - on network.tf line 35: - 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { - -A azurerm_network_security_group resource named ""my_terraform_nsg"" was -already declared at main.tf:35,1-61. Resource names must be unique per type -in each module. - - -Error: Duplicate resource ""azurerm_network_interface"" configuration - - on network.tf line 54: - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - -A azurerm_network_interface resource named ""my_terraform_nic"" was already -declared at main.tf:54,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { - -A azurerm_network_interface_security_group_association resource named -""example"" was already declared at main.tf:68,1-74. Resource names must be -unique per type in each module. - - -Error: Duplicate resource ""random_id"" configuration - - on network.tf line 74: - 74: resource ""random_id"" ""random_id"" { - -A random_id resource named ""random_id"" was already declared at -main.tf:74,1-33. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_storage_account"" configuration - - on network.tf line 82: - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - -A azurerm_storage_account resource named ""my_storage_account"" was already -declared at main.tf:84,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration - - on vm.tf line 1: - 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - -A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already -declared at main.tf:93,1-59. Resource names must be unique per type in each -module. - - time=2025-03-05T11:16:58-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:5,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_virtual_network"" configuration - - on network.tf line 11: - 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { - -A azurerm_virtual_network resource named ""my_terraform_network"" was already -declared at main.tf:11,1-58. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_subnet"" configuration - - on network.tf line 19: - 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { - -A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at -main.tf:19,1-48. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_public_ip"" configuration - - on network.tf line 27: - 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { - -A azurerm_public_ip resource named ""my_terraform_public_ip"" was already -declared at main.tf:27,1-54. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_security_group"" configuration - - on network.tf line 35: - 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { - -A azurerm_network_security_group resource named ""my_terraform_nsg"" was -already declared at main.tf:35,1-61. Resource names must be unique per type -in each module. - - -Error: Duplicate resource ""azurerm_network_interface"" configuration - - on network.tf line 54: - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - -A azurerm_network_interface resource named ""my_terraform_nic"" was already -declared at main.tf:54,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { - -A azurerm_network_interface_security_group_association resource named -""example"" was already declared at main.tf:68,1-74. Resource names must be -unique per type in each module. - - -Error: Duplicate resource ""random_id"" configuration - - on network.tf line 74: - 74: resource ""random_id"" ""random_id"" { - -A random_id resource named ""random_id"" was already declared at -main.tf:74,1-33. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_storage_account"" configuration - - on network.tf line 82: - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - -A azurerm_storage_account resource named ""my_storage_account"" was already -declared at main.tf:84,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration - - on vm.tf line 1: - 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - -A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already -declared at main.tf:93,1-59. Resource names must be unique per type in each -module. - -' -StdErr: There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:5,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_virtual_network"" configuration - - on network.tf line 11: - 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { - -A azurerm_virtual_network resource named ""my_terraform_network"" was already -declared at main.tf:11,1-58. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_subnet"" configuration - - on network.tf line 19: - 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { - -A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at -main.tf:19,1-48. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_public_ip"" configuration - - on network.tf line 27: - 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { - -A azurerm_public_ip resource named ""my_terraform_public_ip"" was already -declared at main.tf:27,1-54. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_security_group"" configuration - - on network.tf line 35: - 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { - -A azurerm_network_security_group resource named ""my_terraform_nsg"" was -already declared at main.tf:35,1-61. Resource names must be unique per type -in each module. - - -Error: Duplicate resource ""azurerm_network_interface"" configuration - - on network.tf line 54: - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - -A azurerm_network_interface resource named ""my_terraform_nic"" was already -declared at main.tf:54,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { - -A azurerm_network_interface_security_group_association resource named -""example"" was already declared at main.tf:68,1-74. Resource names must be -unique per type in each module. - - -Error: Duplicate resource ""random_id"" configuration - - on network.tf line 74: - 74: resource ""random_id"" ""random_id"" { - -A random_id resource named ""random_id"" was already declared at -main.tf:74,1-33. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_storage_account"" configuration - - on network.tf line 82: - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - -A azurerm_storage_account resource named ""my_storage_account"" was already -declared at main.tf:84,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration - - on vm.tf line 1: - 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - -A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already -declared at main.tf:93,1-59. Resource names must be unique per type in each -module. - - time=2025-03-05T11:17:48-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:5,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_virtual_network"" configuration - - on network.tf line 11: - 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { - -A azurerm_virtual_network resource named ""my_terraform_network"" was already -declared at main.tf:11,1-58. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_subnet"" configuration - - on network.tf line 19: - 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { - -A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at -main.tf:19,1-48. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_public_ip"" configuration - - on network.tf line 27: - 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { - -A azurerm_public_ip resource named ""my_terraform_public_ip"" was already -declared at main.tf:27,1-54. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_security_group"" configuration - - on network.tf line 35: - 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { - -A azurerm_network_security_group resource named ""my_terraform_nsg"" was -already declared at main.tf:35,1-61. Resource names must be unique per type -in each module. - - -Error: Duplicate resource ""azurerm_network_interface"" configuration - - on network.tf line 54: - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - -A azurerm_network_interface resource named ""my_terraform_nic"" was already -declared at main.tf:54,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { - -A azurerm_network_interface_security_group_association resource named -""example"" was already declared at main.tf:68,1-74. Resource names must be -unique per type in each module. - - -Error: Duplicate resource ""random_id"" configuration - - on network.tf line 74: - 74: resource ""random_id"" ""random_id"" { - -A random_id resource named ""random_id"" was already declared at -main.tf:74,1-33. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_storage_account"" configuration - - on network.tf line 82: - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - -A azurerm_storage_account resource named ""my_storage_account"" was already -declared at main.tf:84,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration - - on vm.tf line 1: - 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - -A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already -declared at main.tf:93,1-59. Resource names must be unique per type in each -module. - -' -StdErr: There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:1,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:5,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_virtual_network"" configuration - - on network.tf line 11: - 11: resource ""azurerm_virtual_network"" ""my_terraform_network"" { - -A azurerm_virtual_network resource named ""my_terraform_network"" was already -declared at main.tf:11,1-58. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_subnet"" configuration - - on network.tf line 19: - 19: resource ""azurerm_subnet"" ""my_terraform_subnet"" { - -A azurerm_subnet resource named ""my_terraform_subnet"" was already declared at -main.tf:19,1-48. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_public_ip"" configuration - - on network.tf line 27: - 27: resource ""azurerm_public_ip"" ""my_terraform_public_ip"" { - -A azurerm_public_ip resource named ""my_terraform_public_ip"" was already -declared at main.tf:27,1-54. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_security_group"" configuration - - on network.tf line 35: - 35: resource ""azurerm_network_security_group"" ""my_terraform_nsg"" { - -A azurerm_network_security_group resource named ""my_terraform_nsg"" was -already declared at main.tf:35,1-61. Resource names must be unique per type -in each module. - - -Error: Duplicate resource ""azurerm_network_interface"" configuration - - on network.tf line 54: - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - -A azurerm_network_interface resource named ""my_terraform_nic"" was already -declared at main.tf:54,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""example"" { - -A azurerm_network_interface_security_group_association resource named -""example"" was already declared at main.tf:68,1-74. Resource names must be -unique per type in each module. - - -Error: Duplicate resource ""random_id"" configuration - - on network.tf line 74: - 74: resource ""random_id"" ""random_id"" { - -A random_id resource named ""random_id"" was already declared at -main.tf:74,1-33. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_storage_account"" configuration - - on network.tf line 82: - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - -A azurerm_storage_account resource named ""my_storage_account"" was already -declared at main.tf:84,1-56. Resource names must be unique per type in each -module. - - -Error: Duplicate resource ""azurerm_linux_virtual_machine"" configuration - - on vm.tf line 1: - 1: resource ""azurerm_linux_virtual_machine"" ""my_terraform_vm"" { - -A azurerm_linux_virtual_machine resource named ""my_terraform_vm"" was already -declared at main.tf:93,1-59. Resource names must be unique per type in each -module. - - time=2025-03-05T11:20:29-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. -Error: command exited with 'exit status 1' and the message ' -Error: Provider produced inconsistent result after apply - -When applying changes to azurerm_network_security_group.my_terraform_nsg, -provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an -unexpected new value: Root resource was present, but now absent. - -This is a bug in the provider, which should be reported in the provider's own -issue tracker. - -Error: Provider produced inconsistent result after apply - -When applying changes to azurerm_virtual_network.my_terraform_network, -provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an -unexpected new value: Root resource was present, but now absent. - -This is a bug in the provider, which should be reported in the provider's own -issue tracker. - -Error: creating Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Network Interface Name: ""myNIC""): performing CreateOrUpdate: unexpected status 400 (400 Bad Request) with error: InvalidResourceReference: Resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/virtualNetworks/myVnet/subnets/mySubnet referenced by resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/networkInterfaces/myNIC was not found. Please make sure that the referenced resource exists, and that both resources are in the same region. - - with azurerm_network_interface.my_terraform_nic, - on network.tf line 54, in resource ""azurerm_network_interface"" ""my_terraform_nic"": - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - - -Error: retrieving Storage Account (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Storage Account Name: ""diag0bdcb34b14495a71""): unexpected status 404 (404 Not Found) with error: ResourceNotFound: The Resource 'Microsoft.Storage/storageAccounts/diag0bdcb34b14495a71' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix - - with azurerm_storage_account.my_storage_account, - on network.tf line 82, in resource ""azurerm_storage_account"" ""my_storage_account"": - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - - -Error: Failed to create/update resource - - with azapi_resource.ssh_public_key, - on ssh.tf line 15, in resource ""azapi_resource"" ""ssh_public_key"": - 15: resource ""azapi_resource"" ""ssh_public_key"" { - -creating/updating Resource: (ResourceId -""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar"" -/ Api Version ""2022-11-01""): GET -https://management.azure.com/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar --------------------------------------------------------------------------------- -RESPONSE 404: 404 Not Found -ERROR CODE: ResourceNotFound --------------------------------------------------------------------------------- -{ - ""error"": { - ""code"": ""ResourceNotFound"", - ""message"": ""The Resource 'Microsoft.Compute/sshPublicKeys/sshevidentjaguar' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix"" - } -} --------------------------------------------------------------------------------- - -' -StdErr: -Error: Provider produced inconsistent result after apply - -When applying changes to azurerm_network_security_group.my_terraform_nsg, -provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an -unexpected new value: Root resource was present, but now absent. - -This is a bug in the provider, which should be reported in the provider's own -issue tracker. - -Error: Provider produced inconsistent result after apply - -When applying changes to azurerm_virtual_network.my_terraform_network, -provider ""provider[\""registry.terraform.io/hashicorp/azurerm\""]"" produced an -unexpected new value: Root resource was present, but now absent. - -This is a bug in the provider, which should be reported in the provider's own -issue tracker. - -Error: creating Network Interface (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Network Interface Name: ""myNIC""): performing CreateOrUpdate: unexpected status 400 (400 Bad Request) with error: InvalidResourceReference: Resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/virtualNetworks/myVnet/subnets/mySubnet referenced by resource /subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Network/networkInterfaces/myNIC was not found. Please make sure that the referenced resource exists, and that both resources are in the same region. - - with azurerm_network_interface.my_terraform_nic, - on network.tf line 54, in resource ""azurerm_network_interface"" ""my_terraform_nic"": - 54: resource ""azurerm_network_interface"" ""my_terraform_nic"" { - - -Error: retrieving Storage Account (Subscription: ""325e7c34-99fb-4190-aa87-1df746c67705"" -Resource Group Name: ""rg-bold-caiman"" -Storage Account Name: ""diag0bdcb34b14495a71""): unexpected status 404 (404 Not Found) with error: ResourceNotFound: The Resource 'Microsoft.Storage/storageAccounts/diag0bdcb34b14495a71' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix - - with azurerm_storage_account.my_storage_account, - on network.tf line 82, in resource ""azurerm_storage_account"" ""my_storage_account"": - 82: resource ""azurerm_storage_account"" ""my_storage_account"" { - - -Error: Failed to create/update resource - - with azapi_resource.ssh_public_key, - on ssh.tf line 15, in resource ""azapi_resource"" ""ssh_public_key"": - 15: resource ""azapi_resource"" ""ssh_public_key"" { - -creating/updating Resource: (ResourceId -""/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar"" -/ Api Version ""2022-11-01""): GET -https://management.azure.com/subscriptions/325e7c34-99fb-4190-aa87-1df746c67705/resourceGroups/rg-bold-caiman/providers/Microsoft.Compute/sshPublicKeys/sshevidentjaguar --------------------------------------------------------------------------------- -RESPONSE 404: 404 Not Found -ERROR CODE: ResourceNotFound --------------------------------------------------------------------------------- -{ - ""error"": { - ""code"": ""ResourceNotFound"", - ""message"": ""The Resource 'Microsoft.Compute/sshPublicKeys/sshevidentjaguar' under resource group 'rg-bold-caiman' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix"" - } -} --------------------------------------------------------------------------------- - - time=2025-03-05T11:21:26-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message ' -Error: Failed to query available provider packages - -Could not retrieve the list of available versions for provider -hashicorp/azapi: provider registry registry.terraform.io does not have a -provider named registry.terraform.io/hashicorp/azapi - -Did you intend to use azure/azapi? If so, you must specify that source -address in each module which requires that provider. To see which modules are -currently depending on hashicorp/azapi, run the following command: - terraform providers - -' -StdErr: -Error: Failed to query available provider packages - -Could not retrieve the list of available versions for provider -hashicorp/azapi: provider registry registry.terraform.io does not have a -provider named registry.terraform.io/hashicorp/azapi - -Did you intend to use azure/azapi? If so, you must specify that source -address in each module which requires that provider. To see which modules are -currently depending on hashicorp/azapi, run the following command: - terraform providers - - time=2025-03-05T11:22:09-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. -Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:12,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { - -A azurerm_network_interface_security_group_association resource named -""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be -unique per type in each module. - -' -StdErr: There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:12,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { - -A azurerm_network_interface_security_group_association resource named -""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be -unique per type in each module. - - time=2025-03-05T11:22:50-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. -Error: command exited with 'exit status 1' and the message 'There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:12,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { - -A azurerm_network_interface_security_group_association resource named -""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be -unique per type in each module. - -' -StdErr: There are some problems with the configuration, described below. - -The Terraform configuration must be valid before initialization so that -Terraform can determine which modules and providers need to be installed. - -Error: Duplicate resource ""random_pet"" configuration - - on network.tf line 1: - 1: resource ""random_pet"" ""rg_name"" { - -A random_pet resource named ""rg_name"" was already declared at main.tf:2,1-32. -Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_resource_group"" configuration - - on network.tf line 5: - 5: resource ""azurerm_resource_group"" ""rg"" { - -A azurerm_resource_group resource named ""rg"" was already declared at -main.tf:12,1-39. Resource names must be unique per type in each module. - - -Error: Duplicate resource ""azurerm_network_interface_security_group_association"" configuration - - on network.tf line 68: - 68: resource ""azurerm_network_interface_security_group_association"" ""nsg_assoc"" { - -A azurerm_network_interface_security_group_association resource named -""nsg_assoc"" was already declared at main.tf:75,1-76. Resource names must be -unique per type in each module.",873.9581248760223,Failure -2025-03-05 12:00:05,workload_description,create a linux vm and ssh into it,Deploy a Linux VM and Connect via SSH_ai_generated.md,1,"time=2025-03-05T11:58:41-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image.",109.34437084197998,Success -2025-03-05 15:01:21,workload_description,create a linux vm and ssh into it ,Create Linux VM with SSH Access on Azure_ai_generated.md,1,"time=2025-03-05T15:00:22-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image.",87.87348413467407,Success -2025-03-05 22:12:36,workload_description,create a linux vm and ssh into it,Deploy Linux VM with SSH Access in Azure_ai_generated.md,1,"time=2025-03-05T22:10:23-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image.",160.46278858184814,Success -2025-03-06 00:09:24,file,doc.md,doc_converted.md,2,"time=2025-03-06T00:04:08-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 5. -Error: command exited with 'exit status 127' and the message 'bash: line 2: Get-AzVm: command not found -' -StdErr: bash: line 2: Get-AzVm: command not found - - time=2025-03-06T00:06:37-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 5. -Error: command exited with 'exit status 127' and the message 'bash: line 2: pwsh: command not found -' -StdErr: bash: line 2: pwsh: command not found",578.4860949516296,Success -2025-03-06 12:42:03,workload_description,create a linux vm and ssh into it using terraform,Deploy Linux VM and SSH using Terraform_ai_generated.md,2,"time=2025-03-06T12:36:09-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 4. -Error: Expected output does not match actual output. -Got: -"""" - -Expected: -x.x.x.x - -Expected Score:0.300000 -Actual Score:0.000000 -StdErr: - - time=2025-03-06T12:38:45-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 5. -Error: command exited with 'exit status 255' and the message 'Pseudo-terminal will not be allocated because stdin is not a terminal. -ssh: Could not resolve hostname x.x.x.x: Name or service not known -' -StdErr: Pseudo-terminal will not be allocated because stdin is not a terminal. -ssh: Could not resolve hostname x.x.x.x: Name or service not known",490.6871666908264,Success -2025-03-06 13:11:31,file,convert.md,convert_converted.md,0,,97.25097727775574,Success -2025-03-06 13:17:04,workload_description,create a linux vm and ssh into it,Deploy Linux VM with SSH Access_ai_generated.md,2,"time=2025-03-06T13:14:15-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: command exited with 'exit status 1' and the message 'ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. -' -StdErr: ERROR: Invalid image ""UbuntuLTS"". Use a valid image URN, custom image name, custom image id, VHD blob URI, or pick an image from ['CentOS85Gen2', 'Debian11', 'OpenSuseLeap154Gen2', 'RHELRaw8LVMGen2', 'SuseSles15SP5', 'Ubuntu2204', 'Ubuntu2404', 'Ubuntu2404Pro', 'FlatcarLinuxFreeGen2', 'Win2022Datacenter', 'Win2022AzureEditionCore', 'Win2019Datacenter', 'Win2016Datacenter', 'Win2012R2Datacenter', 'Win2012Datacenter']. -See vm create -h for more information on specifying an image. - - time=2025-03-06T13:14:46-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 1. -Error: command exited with 'exit status 1' and the message 'ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_sx95BEHUMfLmMWthesw8MpVq7FOIx45d' is not valid according to the validation procedure. The tracking id is '44b14b15-e2ea-4ac8-b5db-a9415338882f'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} -' -StdErr: ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_sx95BEHUMfLmMWthesw8MpVq7FOIx45d' is not valid according to the validation procedure. The tracking id is '44b14b15-e2ea-4ac8-b5db-a9415338882f'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}}",245.48310685157776,Success -2025-03-06 18:43:21,file,convert.md,convert_converted.md,11,"time=2025-03-06T18:01:49-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 0. -Error: unexpected end of JSON input -StdErr: - - time=2025-03-06T18:03:00-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. -Error: command exited with 'exit status 1' and the message 'WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. -ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_uwCOm6AsMFaPq38JK3iUeOr5GzysgPPQ' is not valid according to the validation procedure. The tracking id is '4b44146b-f9ec-45c0-b06e-4547a098c85d'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} -' -StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. -ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_uwCOm6AsMFaPq38JK3iUeOr5GzysgPPQ' is not valid according to the validation procedure. The tracking id is '4b44146b-f9ec-45c0-b06e-4547a098c85d'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_DS1_v2' is currently not available in location 'eastus'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} - - time=2025-03-06T18:04:04-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. -Error: command exited with 'exit status 1' and the message 'WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. -ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_NH1l0tnhOQitW1xREka8tnjlo6i9gBYS' is not valid according to the validation procedure. The tracking id is '53aa2916-7335-490c-bcf1-69953a136620'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} -' -StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. -ERROR: {""error"":{""code"":""InvalidTemplateDeployment"",""message"":""The template deployment 'vm_deploy_NH1l0tnhOQitW1xREka8tnjlo6i9gBYS' is not valid according to the validation procedure. The tracking id is '53aa2916-7335-490c-bcf1-69953a136620'. See inner errors for details."",""details"":[{""code"":""SkuNotAvailable"",""message"":""The requested VM size for resource 'Following SKUs have failed for Capacity Restrictions: Standard_B1s' is currently not available in location 'westus2'. Please try another size or deploy to a different location or different zone. See https://aka.ms/azureskunotavailable for details.""}]}} - - time=2025-03-06T18:09:03-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 127' and the message 'bash: line 2: =: command not found -' -StdErr: bash: line 2: =: command not found - - time=2025-03-06T18:13:52-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 2. -Error: command exited with 'exit status 127' and the message 'bash: line 2: =: command not found -' -StdErr: bash: line 2: =: command not found - - time=2025-03-06T18:18:58-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. -Error: invalid character '{' after top-level value -StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. - - time=2025-03-06T18:23:55-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. -Error: invalid character '{' after top-level value -StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. - - time=2025-03-06T18:28:29-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 1. -Error: invalid character '{' after top-level value -StdErr: WARNING: Consider upgrading security for your workloads using Azure Trusted Launch VMs. To know more about Trusted Launch, please visit https://aka.ms/TrustedLaunch. - - time=2025-03-06T18:34:32-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 2. -Error: command exited with 'exit status 127' and the message 'bash: line 2: ansible-inventory: command not found -' -StdErr: bash: line 2: ansible-inventory: command not found - - time=2025-03-06T18:35:22-08:00 level=error msg=Error testing scenario: failed to execute code block 0 on step 0. -Error: command exited with 'exit status 1' and the message '' -StdErr: - - time=2025-03-06T18:43:21-08:00 level=error msg=Error testing scenario: failed to execute code block 1 on step 2. -Error: command exited with 'exit status 127' and the message 'bash: line 2: ansible-inventory: command not found -' -StdErr: bash: line 2: ansible-inventory: command not found",2563.7570362091064,Failure From 387002bcbc22bb9700e2d80ef6f14d55f2112337 Mon Sep 17 00:00:00 2001 From: naman-msft Date: Sun, 30 Mar 2025 22:05:44 -0700 Subject: [PATCH 53/87] added aks doc on windows node --- .../quick-windows-container-deploy-cli.md | 386 ++++++++++++++++++ .../articles/aks/learn/sample.yaml | 40 ++ scenarios/metadata.json | 23 ++ 3 files changed, 449 insertions(+) create mode 100644 scenarios/azure-aks-docs/articles/aks/learn/quick-windows-container-deploy-cli.md create mode 100644 scenarios/azure-aks-docs/articles/aks/learn/sample.yaml diff --git a/scenarios/azure-aks-docs/articles/aks/learn/quick-windows-container-deploy-cli.md b/scenarios/azure-aks-docs/articles/aks/learn/quick-windows-container-deploy-cli.md new file mode 100644 index 000000000..15690ff4d --- /dev/null +++ b/scenarios/azure-aks-docs/articles/aks/learn/quick-windows-container-deploy-cli.md @@ -0,0 +1,386 @@ +--- +title: Deploy a Windows Server container on an Azure Kubernetes Service (AKS) cluster using Azure CLI +description: Learn how to quickly deploy a Kubernetes cluster and deploy an application in a Windows Server container in Azure Kubernetes Service (AKS) using Azure CLI. +ms.topic: quickstart +ms.custom: devx-track-azurecli, innovation-engine +ms.date: 01/11/2024 +author: schaffererin +ms.author: schaffererin +--- + +# Deploy a Windows Server container on an Azure Kubernetes Service (AKS) cluster using Azure CLI + +Azure Kubernetes Service (AKS) is a managed Kubernetes service that lets you quickly deploy and manage clusters. In this article, you use Azure CLI to deploy an AKS cluster that runs Windows Server containers. You also deploy an ASP.NET sample application in a Windows Server container to the cluster. + +> [!NOTE] +> To get started with quickly provisioning an AKS cluster, this article includes steps to deploy a cluster with default settings for evaluation purposes only. Before deploying a production-ready cluster, we recommend that you familiarize yourself with our [baseline reference architecture][baseline-reference-architecture] to consider how it aligns with your business requirements. + +## Before you begin + +This quickstart assumes a basic understanding of Kubernetes concepts. For more information, see [Kubernetes core concepts for Azure Kubernetes Service (AKS)](../concepts-clusters-workloads.md). + +- [!INCLUDE [quickstarts-free-trial-note](~/reusable-content/ce-skilling/azure/includes/quickstarts-free-trial-note.md)] + +[!INCLUDE [azure-cli-prepare-your-environment-no-header.md](~/reusable-content/azure-cli/azure-cli-prepare-your-environment-no-header.md)] + +- This article requires version 2.0.64 or later of the Azure CLI. If you're using Azure Cloud Shell, the latest version is already installed there. +- Make sure that the identity you're using to create your cluster has the appropriate minimum permissions. For more details on access and identity for AKS, see [Access and identity options for Azure Kubernetes Service (AKS)](../concepts-identity.md). +- If you have multiple Azure subscriptions, select the appropriate subscription ID in which the resources should be billed using the [az account set](/cli/azure/account#az-account-set) command. For more information, see [How to manage Azure subscriptions – Azure CLI](/cli/azure/manage-azure-subscriptions-azure-cli?tabs=bash#change-the-active-subscription). + +## Create a resource group + +An [Azure resource group](/azure/azure-resource-manager/management/overview) is a logical group in which Azure resources are deployed and managed. When you create a resource group, you're asked to specify a location. This location is where resource group metadata is stored and where your resources run in Azure if you don't specify another region during resource creation. + +- Create a resource group using the [az group create][az-group-create] command. The following example creates a resource group named *myResourceGroup* in the *WestUS2* location. Enter this command and other commands in this article into a BASH shell: + +```bash +export RANDOM_SUFFIX=$(openssl rand -hex 3) +export REGION="eastus2" +export MY_RESOURCE_GROUP_NAME="myAKSResourceGroup$RANDOM_SUFFIX" +az group create --name $MY_RESOURCE_GROUP_NAME --location $REGION +``` + +Results: + + + +```JSON +{ + "id": "/subscriptions/xxxxx-xxxxx-xxxxx-xxxxx/resourceGroups/myResourceGroupxxxxx", + "location": "WestUS2", + "managedBy": null, + "name": "myResourceGroupxxxxx", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null, + "type": "Microsoft.Resources/resourceGroups" +} +``` + +## Create an AKS cluster + +In this section, we create an AKS cluster with the following configuration: + +- The cluster is configured with two nodes to ensure it operates reliably. A [node](../concepts-clusters-workloads.md#nodes) is an Azure virtual machine (VM) that runs the Kubernetes node components and container runtime. +- The `--windows-admin-password` and `--windows-admin-username` parameters set the administrator credentials for any Windows Server nodes on the cluster and must meet [Windows Server password requirements][windows-server-password]. +- The node pool uses `VirtualMachineScaleSets`. + +To create the AKS cluster with Azure CLI, follow these steps: + +1. Create a username to use as administrator credentials for the Windows Server nodes on your cluster. (The original example prompted for input; in this Exec Doc, the environment variable is set non-interactively.) + +```bash +export WINDOWS_USERNAME="winadmin" +``` + +2. Create a password for the administrator username you created in the previous step. The password must be a minimum of 14 characters and meet the [Windows Server password complexity requirements][windows-server-password]. + +```bash +export WINDOWS_PASSWORD="$(openssl rand -base64 32 | tr -d '=+/' | cut -c1-14)" +``` + +3. Create your cluster using the [az aks create][az-aks-create] command and specify the `--windows-admin-username` and `--windows-admin-password` parameters. The following example command creates a cluster using the values from *WINDOWS_USERNAME* and *WINDOWS_PASSWORD* you set in the previous commands. A random suffix is appended to the cluster name for uniqueness. + +```bash +export MY_AKS_CLUSTER="myAKSCluster$RANDOM_SUFFIX" +``` + +```bash +export MY_AKS_CLUSTER="myAKSCluster$RANDOM_SUFFIX" +az aks create \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --name $MY_AKS_CLUSTER \ + --node-count 2 \ + --enable-addons monitoring \ + --generate-ssh-keys \ + --windows-admin-username $WINDOWS_USERNAME \ + --windows-admin-password $WINDOWS_PASSWORD \ + --vm-set-type VirtualMachineScaleSets \ + --network-plugin azure +``` + +After a few minutes, the command completes and returns JSON-formatted information about the cluster. Occasionally, the cluster can take longer than a few minutes to provision. Allow up to 10 minutes for provisioning. + +If you get a password validation error, and the password that you set meets the length and complexity requirements, try creating your resource group in another region. Then try creating the cluster with the new resource group. + +If you don't specify an administrator username and password when creating the node pool, the username is set to *azureuser* and the password is set to a random value. For more information, see the [Windows Server FAQ](../windows-faq.yml) + +The administrator username can't be changed, but you can change the administrator password that your AKS cluster uses for Windows Server nodes using `az aks update`. For more information, see [Windows Server FAQ](../windows-faq.yml). + +To run an AKS cluster that supports node pools for Windows Server containers, your cluster needs to use a network policy that uses [Azure CNI (advanced)][azure-cni] network plugin. The `--network-plugin azure` parameter specifies Azure CNI. + +## Add a node pool + +By default, an AKS cluster is created with a node pool that can run Linux containers. You must add another node pool that can run Windows Server containers alongside the Linux node pool. + +Windows Server 2022 is the default operating system for Kubernetes versions 1.25.0 and higher. Windows Server 2019 is the default OS for earlier versions. If you don't specify a particular OS SKU, Azure creates the new node pool with the default SKU for the version of Kubernetes used by the cluster. + +### [Windows node pool (default SKU)](#tab/add-windows-node-pool) + +To use the default OS SKU, create the node pool without specifying an OS SKU. The node pool is configured for the default operating system based on the Kubernetes version of the cluster. + +Add a Windows node pool using the `az aks nodepool add` command. The following command creates a new node pool named *npwin* and adds it to *myAKSCluster*. The command also uses the default subnet in the default virtual network created when running `az aks create`. An OS SKU isn't specified, so the node pool is set to the default operating system based on the Kubernetes version of the cluster: + +```text +az aks nodepool add \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --cluster-name $MY_AKS_CLUSTER \ + --os-type Windows \ + --name npwin \ + --node-count 1 +``` + +### [Windows Server 2022 node pool](#tab/add-windows-server-2022-node-pool) + +To use Windows Server 2022, specify the following parameters: + +- `os-type` set to `Windows` +- `os-sku` set to `Windows2022` + +> [!NOTE] +> Windows Server 2022 requires Kubernetes version 1.23.0 or higher. Windows Server 2022 is being retired after Kubernetes version 1.34 reaches its end of support. Windows Server 2022 will not be supported in Kubernetes version 1.35 and above. For more information about this retirement, see the [AKS release notes][aks-release-notes]. + +Add a Windows Server 2022 node pool using the `az aks nodepool add` command: + +```text +az aks nodepool add \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --cluster-name $MY_AKS_CLUSTER \ + --os-type Windows \ + --os-sku Windows2022 \ + --name npwin \ + --node-count 1 +``` + +### [Windows Server 2019 node pool](#tab/add-windows-server-2019-node-pool) + +To use Windows Server 2019, specify the following parameters: + +- `os-type` set to `Windows` +- `os-sku` set to `Windows2019` + +> [!NOTE] +> Windows Server 2019 is being retired after Kubernetes version 1.32 reaches end of support. Windows Server 2019 will not be supported in Kubernetes version 1.33 and above. For more information about this retirement, see the [AKS release notes][aks-release-notes]. + +Add a Windows Server 2019 node pool using the `az aks nodepool add` command: + +```text +az aks nodepool add \ + --resource-group $MY_RESOURCE_GROUP_NAME \ + --cluster-name $MY_AKS_CLUSTER \ + --os-type Windows \ + --os-sku Windows2019 \ + --name npwin \ + --node-count 1 +``` + +## Connect to the cluster + +You use [kubectl][kubectl], the Kubernetes command-line client, to manage your Kubernetes clusters. If you use Azure Cloud Shell, `kubectl` is already installed. If you want to install and run `kubectl` locally, call the [az aks install-cli][az-aks-install-cli] command. + +1. Configure `kubectl` to connect to your Kubernetes cluster using the [az aks get-credentials][az-aks-get-credentials] command. This command downloads credentials and configures the Kubernetes CLI to use them. + +```bash +az aks get-credentials --resource-group $MY_RESOURCE_GROUP_NAME --name $MY_AKS_CLUSTER +``` + +2. Verify the connection to your cluster using the [kubectl get][kubectl-get] command, which returns a list of the cluster nodes. + +```bash +kubectl get nodes -o wide +``` + +The following sample output shows all nodes in the cluster. Make sure the status of all nodes is *Ready*: + + + +```text +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +aks-nodepool1-20786768-vmss000000 Ready agent 22h v1.27.7 10.224.0.4 Ubuntu 22.04.3 LTS 5.15.0-1052-azure containerd://1.7.5-1 +aks-nodepool1-20786768-vmss000001 Ready agent 22h v1.27.7 10.224.0.33 Ubuntu 22.04.3 LTS 5.15.0-1052-azure containerd://1.7.5-1 +aksnpwin000000 Ready agent 20h v1.27.7 10.224.0.62 Windows Server 2022 Datacenter 10.0.20348.2159 containerd://1.6.21+azure +``` + +> [!NOTE] +> The container runtime for each node pool is shown under *CONTAINER-RUNTIME*. The container runtime values begin with `containerd://`, which means that they each use `containerd` for the container runtime. + +## Deploy the application + +A Kubernetes manifest file defines a desired state for the cluster, such as what container images to run. In this article, you use a manifest to create all objects needed to run the ASP.NET sample application in a Windows Server container. This manifest includes a [Kubernetes deployment][kubernetes-deployment] for the ASP.NET sample application and an external [Kubernetes service][kubernetes-service] to access the application from the internet. + +The ASP.NET sample application is provided as part of the [.NET Framework Samples][dotnet-samples] and runs in a Windows Server container. AKS requires Windows Server containers to be based on images of *Windows Server 2019* or greater. The Kubernetes manifest file must also define a [node selector][node-selector] to tell your AKS cluster to run your ASP.NET sample application's pod on a node that can run Windows Server containers. + +1. Create a file named `sample.yaml` and copy in the following YAML definition. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sample + labels: + app: sample +spec: + replicas: 1 + template: + metadata: + name: sample + labels: + app: sample + spec: + nodeSelector: + "kubernetes.io/os": windows + containers: + - name: sample + image: mcr.microsoft.com/dotnet/framework/samples:aspnetapp + resources: + limits: + cpu: 1 + memory: 800M + ports: + - containerPort: 80 + selector: + matchLabels: + app: sample +--- +apiVersion: v1 +kind: Service +metadata: + name: sample +spec: + type: LoadBalancer + ports: + - protocol: TCP + port: 80 + selector: + app: sample +``` + +For a breakdown of YAML manifest files, see [Deployments and YAML manifests](../concepts-clusters-workloads.md#deployments-and-yaml-manifests). + +If you create and save the YAML file locally, then you can upload the manifest file to your default directory in CloudShell by selecting the **Upload/Download files** button and selecting the file from your local file system. + +2. Deploy the application using the [kubectl apply][kubectl-apply] command and specify the name of your YAML manifest. + +```bash +kubectl apply -f sample.yaml +``` + +The following sample output shows the deployment and service created successfully: + + + +```text +{ + "deployment.apps/sample": "created", + "service/sample": "created" +} +``` + +## Test the application + +When the application runs, a Kubernetes service exposes the application front end to the internet. This process can take a few minutes to complete. Occasionally, the service can take longer than a few minutes to provision. Allow up to 10 minutes for provisioning. + +1. Check the status of the deployed pods using the [kubectl get pods][kubectl-get] command. Make sure all pods are `Running` before proceeding. + +```bash +kubectl get pods +``` + +2. Monitor progress using the [kubectl get service][kubectl-get] command with the `--watch` argument. + +```bash +while true; do + export EXTERNAL_IP=$(kubectl get service sample -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 2>/dev/null) + if [[ -n "$EXTERNAL_IP" && "$EXTERNAL_IP" != "" ]]; then + kubectl get service sample + break + fi + echo "Still waiting for external IP assignment..." + sleep 5 +done +``` + +Initially, the output shows the *EXTERNAL-IP* for the sample service as *pending*: + + + +```text +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +sample LoadBalancer xx.xx.xx.xx pending xx:xxxx/TCP 2m +``` + +When the *EXTERNAL-IP* address changes from *pending* to an actual public IP address, use `CTRL-C` to stop the `kubectl` watch process. The following sample output shows a valid public IP address assigned to the service: + +```JSON +{ + "NAME": "sample", + "TYPE": "LoadBalancer", + "CLUSTER-IP": "10.0.37.27", + "EXTERNAL-IP": "52.179.23.131", + "PORT(S)": "80:30572/TCP", + "AGE": "2m" +} +``` + +3. See the sample app in action by opening a web browser to the external IP address of your service. + +```bash +curl -s $EXTERNAL_IP | head -n 20 +``` + +The following sample output shows the HTML content returned by the ASP.NET sample application: + + + +```text + + + + + + Home Page - ASP.NET Application + + + + + + +
+ + + - - - - - - - - -