From 2e07b26e6d3fea1d44e2250dee33fc5179b2056e Mon Sep 17 00:00:00 2001 From: Tomi Paananen Date: Thu, 2 Sep 2021 17:22:34 +0300 Subject: [PATCH 1/7] Adds NSG for Bastion subnet --- templates/core/terraform/main.tf | 36 ++-- templates/core/terraform/network/network.tf | 35 ---- .../network/network_security_groups.tf | 162 ++++++++++++++++++ templates/core/terraform/network/output.tf | 20 +-- 4 files changed, 188 insertions(+), 65 deletions(-) create mode 100644 templates/core/terraform/network/network_security_groups.tf diff --git a/templates/core/terraform/main.tf b/templates/core/terraform/main.tf index 4ca8e8d421..c3cda51b4c 100644 --- a/templates/core/terraform/main.tf +++ b/templates/core/terraform/main.tf @@ -63,8 +63,8 @@ module "storage" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - shared_subnet = module.network.shared - core_vnet = module.network.core + shared_subnet = module.network.shared_subnet_id + core_vnet = module.network.core_vnet_id } module "appgateway" { @@ -72,8 +72,8 @@ module "appgateway" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - app_gw_subnet = module.network.app_gw - shared_subnet = module.network.shared + app_gw_subnet = module.network.app_gw_subnet_id + shared_subnet = module.network.shared_subnet_id management_api_fqdn = module.api-webapp.management_api_fqdn keyvault_id = module.keyvault.keyvault_id static_web_dns_zone_id = module.network.static_web_dns_zone_id @@ -85,10 +85,10 @@ module "api-webapp" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - web_app_subnet = module.network.web_app - shared_subnet = module.network.shared - app_gw_subnet = module.network.app_gw - core_vnet = module.network.core + web_app_subnet = module.network.web_app_subnet_id + shared_subnet = module.network.shared_subnet_id + app_gw_subnet = module.network.app_gw_subnet_id + core_vnet = module.network.core_vnet_id app_insights_connection_string = azurerm_application_insights.core.connection_string app_insights_instrumentation_key = azurerm_application_insights.core.instrumentation_key log_analytics_workspace_id = azurerm_log_analytics_workspace.core.id @@ -125,7 +125,7 @@ module "resource_processor_vmss_porter" { resource_group_name = azurerm_resource_group.core.name acr_id = data.azurerm_container_registry.mgmt_acr.id app_insights_connection_string = azurerm_application_insights.core.connection_string - resource_processor_subnet_id = module.network.resource_processor + resource_processor_subnet_id = module.network.resource_processor_subnet_id docker_registry_server = var.docker_registry_server resource_processor_vmss_porter_image_repository = var.resource_processor_vmss_porter_image_repository resource_processor_vmss_porter_image_tag = var.resource_processor_vmss_porter_image_tag @@ -148,8 +148,8 @@ module "servicebus" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - core_vnet = module.network.core - resource_processor_subnet_id = module.network.resource_processor + core_vnet = module.network.core_vnet_id + resource_processor_subnet_id = module.network.resource_processor_subnet_id } module "keyvault" { @@ -157,8 +157,8 @@ module "keyvault" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - shared_subnet = module.network.shared - core_vnet = module.network.core + shared_subnet = module.network.shared_subnet_id + core_vnet = module.network.core_vnet_id tenant_id = data.azurerm_client_config.current.tenant_id managed_identity_tenant_id = module.identity.managed_identity.tenant_id managed_identity_object_id = module.identity.managed_identity.principal_id @@ -186,8 +186,8 @@ module "routetable" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - shared_subnet_id = module.network.shared - resource_processor_subnet_id = module.network.resource_processor + shared_subnet_id = module.network.shared_subnet_id + resource_processor_subnet_id = module.network.resource_processor_subnet_id firewall_private_ip_address = module.firewall.firewall_private_ip_address } @@ -196,8 +196,8 @@ module "state-store" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - shared_subnet = module.network.shared - core_vnet = module.network.core + shared_subnet = module.network.shared_subnet_id + core_vnet = module.network.core_vnet_id } module "bastion" { @@ -205,7 +205,7 @@ module "bastion" { tre_id = var.tre_id location = var.location resource_group_name = azurerm_resource_group.core.name - bastion_subnet = module.network.bastion + bastion_subnet = module.network.bastion_subnet_id } module "gitea" { diff --git a/templates/core/terraform/network/network.tf b/templates/core/terraform/network/network.tf index 4151720edd..75e74e4283 100644 --- a/templates/core/terraform/network/network.tf +++ b/templates/core/terraform/network/network.tf @@ -48,41 +48,6 @@ resource "azurerm_subnet" "web_app" { } } -resource "azurerm_subnet" "aci" { - name = "AciSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.aci_subnet_address_prefix] - enforce_private_link_endpoint_network_policies = true - enforce_private_link_service_network_policies = true - - delegation { - name = "acidelegationservice" - - service_delegation { - name = "Microsoft.ContainerInstance/containerGroups" - actions = ["Microsoft.Network/virtualNetworks/subnets/action"] - } - } -} - -resource "azurerm_network_profile" "aciprofile" { - name = "aciprofile" - location = var.location - resource_group_name = var.resource_group_name - - container_network_interface { - name = "acr-frontal-nic" - - ip_configuration { - name = "acrfrontal" - subnet_id = azurerm_subnet.aci.id - } - } - - lifecycle { ignore_changes = [tags] } -} - resource "azurerm_subnet" "shared" { name = "SharedSubnet" virtual_network_name = azurerm_virtual_network.core.name diff --git a/templates/core/terraform/network/network_security_groups.tf b/templates/core/terraform/network/network_security_groups.tf new file mode 100644 index 0000000000..fec1eb5e62 --- /dev/null +++ b/templates/core/terraform/network/network_security_groups.tf @@ -0,0 +1,162 @@ +# Network Security Group for Azure Bastion +# See https://docs.microsoft.com/en-us/azure/bastion/bastion-nsg +resource "azurerm_network_security_group" "bastion" { + name = "nsg-bastion-subnet" + location = var.location + resource_group_name = var.resource_group_name + + security_rule { + name = "BastionInboundAllowInternet" + priority = 4000 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "Internet" + destination_address_prefix = "*" + } + + security_rule { + name = "BastionInboundAllowGatewayManager" + priority = 4001 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "GatewayManager" + destination_address_prefix = "*" + } + + security_rule { + name = "BastionInboundAllowAzureLoadBalancer" + priority = 4002 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "AzureLoadBalancer" + destination_address_prefix = "*" + } + + security_rule { + name = "BastionInboundAllowHostCommunication" + priority = 4003 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_ranges = ["5701", "8080"] + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "VirtualNetwork" + } + + #security_rule { + # name = "bastion-inbound-deny-all" + # priority = 4010 + # direction = "Inbound" + # access = "Deny" + # protocol = "*" + # source_port_range = "*" + # destination_port_range = "*" + # source_address_prefix = "*" + # destination_address_prefix = "*" + #} + + security_rule { + name = "BastionOutboundAllowSshRdp" + priority = 4020 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_ranges = ["22", "3389"] + source_address_prefix = "*" + destination_address_prefix = "VirtualNetwork" + } + + security_rule { + name = "BastionOutboundAllowAzureCloud" + priority = 4021 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "*" + destination_address_prefix = "AzureCloud" + } + + security_rule { + name = "BastionOutboundAllowHostCommunication" + priority = 4022 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_ranges = ["5701", "8080"] + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "VirtualNetwork" + } + + security_rule { + name = "BastionOutboundAllowGetSessionInformation" + priority = 4023 + direction = "Outbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "80" + source_address_prefix = "*" + destination_address_prefix = "Internet" + } + + #security_rule { + # name = "bastion-outbound-deny-all" + # priority = 4030 + # direction = "Outbound" + # access = "Deny" + # protocol = "*" + # source_port_range = "*" + # destination_port_range = "*" + # source_address_prefix = "*" + # destination_address_prefix = "*" + #} +} + +resource "azurerm_subnet_network_security_group_association" "bastion" { + subnet_id = azurerm_subnet.bastion.id + network_security_group_id = azurerm_network_security_group.bastion.id +} + +resource "azurerm_network_security_group" "web_app" { + name = "nsg-web-app-subnet" + location = var.location + resource_group_name = var.resource_group_name + + # TODO: The real rules - this is just a test rule + security_rule { + name = "webapp-inbound-allow-virtualnetwork" + priority = 3900 + direction = "Inbound" + access = "Allow" + protocol = "*" + source_port_range = "*" + destination_port_range = "*" + source_address_prefix = "VirtualNetwork" + destination_address_prefix = "VirtualNetwork" + } +} + +resource "azurerm_subnet_network_security_group_association" "web_app" { + subnet_id = azurerm_subnet.web_app.id + network_security_group_id = azurerm_network_security_group.web_app.id +} + +# Test multi-association +resource "azurerm_subnet_network_security_group_association" "resource_processor" { + subnet_id = azurerm_subnet.resource_processor.id + network_security_group_id = azurerm_network_security_group.web_app.id +} diff --git a/templates/core/terraform/network/output.tf b/templates/core/terraform/network/output.tf index 4668830427..edd64a5f34 100644 --- a/templates/core/terraform/network/output.tf +++ b/templates/core/terraform/network/output.tf @@ -1,32 +1,28 @@ -output "core" { +output "core_vnet_id" { value = azurerm_virtual_network.core.id } -output "bastion" { +output "bastion_subnet_id" { value = azurerm_subnet.bastion.id } -output "azure_firewall" { +output "azure_firewall_subnet_id" { value = azurerm_subnet.azure_firewall.id } -output "app_gw" { +output "app_gw_subnet_id" { value = azurerm_subnet.app_gw.id } -output "web_app" { +output "web_app_subnet_id" { value = azurerm_subnet.web_app.id } -output "shared" { +output "shared_subnet_id" { value = azurerm_subnet.shared.id } -output "aci" { - value = azurerm_subnet.aci.id -} - -output "resource_processor" { +output "resource_processor_subnet_id" { value = azurerm_subnet.resource_processor.id } @@ -36,4 +32,4 @@ output "azurewebsites_dns_zone_id" { output "static_web_dns_zone_id" { value = azurerm_private_dns_zone.static_web.id -} \ No newline at end of file +} From 838bd4e609d62db330c68c9db2ad4697a417b5d2 Mon Sep 17 00:00:00 2001 From: Tomi Paananen Date: Mon, 6 Sep 2021 09:40:46 +0300 Subject: [PATCH 2/7] Updates docs --- docs/architecture.md | 37 ---------- docs/index.md | 1 + docs/networking.md | 60 +++++++++++++++ .../network/network_security_groups.tf | 73 ++++++------------- 4 files changed, 82 insertions(+), 89 deletions(-) create mode 100644 docs/networking.md diff --git a/docs/architecture.md b/docs/architecture.md index 7f0c3c3383..c39539e368 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,40 +93,3 @@ The flow to provision a Workspace is as follows (the flow is the same for all ki 1. The Resource Processor sends events to the `deploymentstatus` queue on state changes and informs if the deployment succeeded or failed. 1. The status of a Porter bundle execution is received. 1. The status of a Porter bundle execution is updated in the Configuration Store. - -## Network Architecture - -The network topology is based on [hub-spoke](https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/hybrid-networking/hub-spoke). The TRE Management VNET ([Azure Virtual Network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview)) is the central hub and each Workspace are spokes. - -> Note: TRE Management is referred to as Core in scripts and code. - -![Network Architecture](./assets/network-architecture.png) - -Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The rules are managed in the `nsg-ws` Network Security Group (NSG): - -- Inbound traffic from TRE Management VNET to Workspace allowed for [Azure Bastion](https://docs.microsoft.com/en-us/azure/bastion/bastion-overview) (22, 3389) - All other inbound traffic from Core to Workspace denied. -- Outbound traffic to `SharedSubnet` from Workspace allowed. -- Outbound traffic to Internet allowed on HTTPS port 443 (next hop Azure Firewall). -- All other outbound traffic denied. - -> In Azure traffic between subnets are allowed except explicitly denied. - -Each of these rules can be managed per Workspace. - -Each Workspace has a default route routing all egress traffic through the Azure Firewall, to ensure only explicitly allowed destinations on the Internet to be accessed. It is planned that all other subnet will use the same pattern (Issue [#421](https://github.com/microsoft/AzureTRE/issues/421)) - -The Azure Firewall rules are: - -- No default inbound rules – block all. -- No default outbound rules – block all. - -Inbound traffic from the Internet is only allowed through the Application Gateway, which forwards HTTPS (port 443) call to the TRE Management API in the `WebAppSubnet`. - -| Subnet | Description | -| -------| ----------- | -| `AzureBastionSubnet` | A dedicated subnet for Azure Bastion hosts. | -| `AppGwSubnet` | Subnet for Azure Application Gateway controlling ingress traffic. | -| `AzureFirewallSubnet` | Subnet for Azure Firewall controlling egress traffic. | -| `ResourceProcessorSubnet` | Subnet for VMSS used by the Composition Service to host Docker containers to execute Porter bundles that deploys Workspaces. | -| `WebAppSubnet` | Subnet for Management API. | -| `SharedSubnet` | Shared Services subnet for all things shared by TRE Management and Workspaces. Future Shared Services are Firewall Shared Service, Source Mirror Shared Service and Package Mirror Shared Service. | diff --git a/docs/index.md b/docs/index.md index c0df164f24..3829a617e8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,6 +4,7 @@ * [Concepts](./concepts.md) * [User roles](./user-roles.md) * [Architecture](./architecture.md) + * [Networking](./networking.md) * [Logical data model](./logical-data-model.md) * Getting started * [Dev environment](./dev-environment.md) diff --git a/docs/networking.md b/docs/networking.md new file mode 100644 index 0000000000..a00fce11f4 --- /dev/null +++ b/docs/networking.md @@ -0,0 +1,60 @@ +# Networking + +Trusted Research Environment (TRE) network topology is based on [hub-spoke](https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/hybrid-networking/hub-spoke). The TRE Management VNET ([Azure Virtual Network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview)) is the central hub and each workspace is a spoke. + +> Note: TRE Management is referred to as **core** in scripts and code. + +![Network architecture](./assets/network-architecture.png) + +Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The rules are managed in the `nsg-ws` Network Security Group (NSG): + +- Inbound traffic from TRE Management VNET to workspace allowed for [Azure Bastion](https://docs.microsoft.com/en-us/azure/bastion/bastion-overview) (22, 3389) - All other inbound traffic from Core to workspace denied. +- Outbound traffic to `SharedSubnet` from Workspace allowed. +- Outbound traffic to Internet allowed on HTTPS port 443 (next hop Azure Firewall). +- All other outbound traffic denied. + +> In Azure traffic between subnets are allowed except explicitly denied. + +Each of these rules can be managed per workspace. + +Each workspace has a default route routing all egress traffic through the Azure Firewall, to ensure only explicitly allowed destinations on the Internet to be accessed. It is planned that all other subnet will use the same pattern (Issue [#421](https://github.com/microsoft/AzureTRE/issues/421)) + +The Azure Firewall rules are: + +- No default inbound rules – block all. +- No default outbound rules – block all. + +Inbound traffic from the Internet is only allowed through the Application Gateway, which forwards HTTPS (port 443) call to the TRE Management API in the `WebAppSubnet`. + +| Subnet | Description | +| -------| ----------- | +| `AzureBastionSubnet` | A dedicated subnet for Azure Bastion hosts. | +| `AppGwSubnet` | Subnet for Azure Application Gateway controlling ingress traffic. | +| `AzureFirewallSubnet` | Subnet for Azure Firewall controlling egress traffic. | +| `ResourceProcessorSubnet` | Subnet for VMSS used by the Composition Service to host Docker containers to execute Porter bundles that deploys Workspaces. | +| `WebAppSubnet` | Subnet for Management API. | +| `SharedSubnet` | Shared Services subnet for all things shared by TRE Management and Workspaces. Future Shared Services are Firewall Shared Service, Source Mirror Shared Service and Package Mirror Shared Service. | + +## Network security groups + +### TRE management/core + +Network security groups (NSG) for TRE core resources are defined in [`/templates/core/terraform/network/network_security_groups.tf`](../templates/core/terraform/network/network_security_groups.tf). + +| Network security group | Associated subnets | +| ---------------------- | ------------------ | +| `nsg-default-rules` | | +| `nsg-bastion-subnet` | `AzureBastionSubnet` | + +### Workspaces + +Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The rules are managed by the `nsg-ws` network security group: + +- Inbound traffic from TRE Management VNET to workspace allowed for [Azure Bastion](https://docs.microsoft.com/en-us/azure/bastion/bastion-overview) (22, 3389) - All other inbound traffic from Core to workspace denied. +- Outbound traffic to `SharedSubnet` from Workspace allowed. +- Outbound traffic to Internet allowed on HTTPS port 443 (next hop Azure Firewall). +- All other outbound traffic denied. + +> In Azure traffic between subnets are allowed except explicitly denied. + +Each of these rules can be managed per workspace. diff --git a/templates/core/terraform/network/network_security_groups.tf b/templates/core/terraform/network/network_security_groups.tf index fec1eb5e62..4eaaa4a55f 100644 --- a/templates/core/terraform/network/network_security_groups.tf +++ b/templates/core/terraform/network/network_security_groups.tf @@ -1,12 +1,12 @@ -# Network Security Group for Azure Bastion -# See https://docs.microsoft.com/en-us/azure/bastion/bastion-nsg +# Network security group for Azure Bastion subnet +# See https://docs.microsoft.com/azure/bastion/bastion-nsg resource "azurerm_network_security_group" "bastion" { name = "nsg-bastion-subnet" location = var.location resource_group_name = var.resource_group_name security_rule { - name = "BastionInboundAllowInternet" + name = "AllowInboundInternet" priority = 4000 direction = "Inbound" access = "Allow" @@ -18,7 +18,7 @@ resource "azurerm_network_security_group" "bastion" { } security_rule { - name = "BastionInboundAllowGatewayManager" + name = "AllowInboundGatewayManager" priority = 4001 direction = "Inbound" access = "Allow" @@ -30,7 +30,7 @@ resource "azurerm_network_security_group" "bastion" { } security_rule { - name = "BastionInboundAllowAzureLoadBalancer" + name = "AllowInboundAzureLoadBalancer" priority = 4002 direction = "Inbound" access = "Allow" @@ -42,7 +42,7 @@ resource "azurerm_network_security_group" "bastion" { } security_rule { - name = "BastionInboundAllowHostCommunication" + name = "AllowInboundHostCommunication" priority = 4003 direction = "Inbound" access = "Allow" @@ -53,20 +53,8 @@ resource "azurerm_network_security_group" "bastion" { destination_address_prefix = "VirtualNetwork" } - #security_rule { - # name = "bastion-inbound-deny-all" - # priority = 4010 - # direction = "Inbound" - # access = "Deny" - # protocol = "*" - # source_port_range = "*" - # destination_port_range = "*" - # source_address_prefix = "*" - # destination_address_prefix = "*" - #} - security_rule { - name = "BastionOutboundAllowSshRdp" + name = "AllowOutboundSshRdp" priority = 4020 direction = "Outbound" access = "Allow" @@ -78,7 +66,7 @@ resource "azurerm_network_security_group" "bastion" { } security_rule { - name = "BastionOutboundAllowAzureCloud" + name = "AllowOutboundAzureCloud" priority = 4021 direction = "Outbound" access = "Allow" @@ -90,7 +78,7 @@ resource "azurerm_network_security_group" "bastion" { } security_rule { - name = "BastionOutboundAllowHostCommunication" + name = "AllowOutboundHostCommunication" priority = 4022 direction = "Outbound" access = "Allow" @@ -102,7 +90,7 @@ resource "azurerm_network_security_group" "bastion" { } security_rule { - name = "BastionOutboundAllowGetSessionInformation" + name = "AllowOutboundGetSessionInformation" priority = 4023 direction = "Outbound" access = "Allow" @@ -112,18 +100,6 @@ resource "azurerm_network_security_group" "bastion" { source_address_prefix = "*" destination_address_prefix = "Internet" } - - #security_rule { - # name = "bastion-outbound-deny-all" - # priority = 4030 - # direction = "Outbound" - # access = "Deny" - # protocol = "*" - # source_port_range = "*" - # destination_port_range = "*" - # source_address_prefix = "*" - # destination_address_prefix = "*" - #} } resource "azurerm_subnet_network_security_group_association" "bastion" { @@ -131,32 +107,25 @@ resource "azurerm_subnet_network_security_group_association" "bastion" { network_security_group_id = azurerm_network_security_group.bastion.id } -resource "azurerm_network_security_group" "web_app" { - name = "nsg-web-app-subnet" +# Network security group with only default security rules +# See https://docs.microsoft.com/azure/virtual-network/network-security-groups-overview#default-security-rules +resource "azurerm_network_security_group" "default_rules" { + name = "nsg-default-rules" location = var.location resource_group_name = var.resource_group_name - - # TODO: The real rules - this is just a test rule - security_rule { - name = "webapp-inbound-allow-virtualnetwork" - priority = 3900 - direction = "Inbound" - access = "Allow" - protocol = "*" - source_port_range = "*" - destination_port_range = "*" - source_address_prefix = "VirtualNetwork" - destination_address_prefix = "VirtualNetwork" - } } resource "azurerm_subnet_network_security_group_association" "web_app" { subnet_id = azurerm_subnet.web_app.id - network_security_group_id = azurerm_network_security_group.web_app.id + network_security_group_id = azurerm_network_security_group.default_rules.id } -# Test multi-association resource "azurerm_subnet_network_security_group_association" "resource_processor" { subnet_id = azurerm_subnet.resource_processor.id - network_security_group_id = azurerm_network_security_group.web_app.id + network_security_group_id = azurerm_network_security_group.default_rules.id +} + +resource "azurerm_subnet_network_security_group_association" "shared" { + subnet_id = azurerm_subnet.shared.id + network_security_group_id = azurerm_network_security_group.default_rules.id } From cacb566cd874e81c24e3d66adb393f87e59e81b1 Mon Sep 17 00:00:00 2001 From: Tomi Paananen Date: Tue, 7 Sep 2021 12:41:56 +0300 Subject: [PATCH 3/7] Updating NSGs and fixing the devcontainer --- .devcontainer/Dockerfile | 2 +- .../network/network_security_groups.tf | 35 ++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f54312a58d..f118d362f5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,7 +2,7 @@ # [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6 ARG VARIANT="3" -FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} +FROM mcr.microsoft.com/vscode/devcontainers/python:dev-${VARIANT}-buster # [Option] Install Node.js ARG INSTALL_NODE="true" diff --git a/templates/core/terraform/network/network_security_groups.tf b/templates/core/terraform/network/network_security_groups.tf index 4eaaa4a55f..8c1b4263ed 100644 --- a/templates/core/terraform/network/network_security_groups.tf +++ b/templates/core/terraform/network/network_security_groups.tf @@ -107,6 +107,31 @@ resource "azurerm_subnet_network_security_group_association" "bastion" { network_security_group_id = azurerm_network_security_group.bastion.id } +# Network security group for Application Gateway +# See https://docs.microsoft.com/azure/application-gateway/configuration-infrastructure#network-security-groups +resource "azurerm_network_security_group" "app_gw" { + name = "nsg-app-gw" + location = var.location + resource_group_name = var.resource_group_name + + security_rule { + name = "AllowInboundGatewayManager" + priority = 3800 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "65200-65535" + source_address_prefix = "GatewayManager" + destination_address_prefix = "*" + } +} + +resource "azurerm_subnet_network_security_group_association" "app_gw" { + subnet_id = azurerm_subnet.app_gw.id + network_security_group_id = azurerm_network_security_group.app_gw.id +} + # Network security group with only default security rules # See https://docs.microsoft.com/azure/virtual-network/network-security-groups-overview#default-security-rules resource "azurerm_network_security_group" "default_rules" { @@ -115,6 +140,11 @@ resource "azurerm_network_security_group" "default_rules" { resource_group_name = var.resource_group_name } +resource "azurerm_subnet_network_security_group_association" "shared" { + subnet_id = azurerm_subnet.shared.id + network_security_group_id = azurerm_network_security_group.default_rules.id +} + resource "azurerm_subnet_network_security_group_association" "web_app" { subnet_id = azurerm_subnet.web_app.id network_security_group_id = azurerm_network_security_group.default_rules.id @@ -124,8 +154,3 @@ resource "azurerm_subnet_network_security_group_association" "resource_processor subnet_id = azurerm_subnet.resource_processor.id network_security_group_id = azurerm_network_security_group.default_rules.id } - -resource "azurerm_subnet_network_security_group_association" "shared" { - subnet_id = azurerm_subnet.shared.id - network_security_group_id = azurerm_network_security_group.default_rules.id -} From 2b4e9f36ce208782e2f654dab9024e1799b4d348 Mon Sep 17 00:00:00 2001 From: Tomi Paananen Date: Tue, 7 Sep 2021 16:16:05 +0300 Subject: [PATCH 4/7] Updates docs --- docs/architecture.md | 37 ++----------------------------------- docs/networking.md | 28 ++++++++++------------------ 2 files changed, 12 insertions(+), 53 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index edf2e4ff8a..4d8c90ed0f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -94,39 +94,6 @@ The flow to provision a Workspace is as follows (the flow is the same for all ki 1. The status of a Porter bundle execution is received. 1. The status of a Porter bundle execution is updated in the Configuration Store. -## Network Architecture +## Network architecture -The network topology is based on [hub-spoke](https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/hybrid-networking/hub-spoke). The TRE Management VNET ([Azure Virtual Network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview)) is the central hub and each Workspace are spokes. - -> **Note:** TRE Management is referred to as Core in scripts and code. - -![Network Architecture](./assets/network-architecture.png) - -Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The rules are managed in the `nsg-ws` Network Security Group (NSG): - -- Inbound traffic from TRE Management VNET to Workspace allowed for [Azure Bastion](https://docs.microsoft.com/en-us/azure/bastion/bastion-overview) (22, 3389) - All other inbound traffic from Core to Workspace denied. -- Outbound traffic to `SharedSubnet` from Workspace allowed. -- Outbound traffic to Internet allowed on HTTPS port 443 (next hop Azure Firewall). -- All other outbound traffic denied. - -> In Azure traffic between subnets are allowed except explicitly denied. - -Each of these rules can be managed per Workspace. - -Each Workspace has a default route routing all egress traffic through the Azure Firewall, to ensure only explicitly allowed destinations on the Internet to be accessed. It is planned that all other subnet will use the same pattern (Issue [#421](https://github.com/microsoft/AzureTRE/issues/421)) - -The Azure Firewall rules are: - -- No default inbound rules – block all. -- No default outbound rules – block all. - -Inbound traffic from the Internet is only allowed through the Application Gateway, which forwards HTTPS (port 443) call to the TRE API in the `WebAppSubnet`. - -| Subnet | Description | -| -------| ----------- | -| `AzureBastionSubnet` | A dedicated subnet for Azure Bastion hosts. | -| `AppGwSubnet` | Subnet for Azure Application Gateway controlling ingress traffic. | -| `AzureFirewallSubnet` | Subnet for Azure Firewall controlling egress traffic. | -| `ResourceProcessorSubnet` | Subnet for VMSS used by the Composition Service to host Docker containers to execute Porter bundles that deploys Workspaces. | -| `WebAppSubnet` | Subnet for TRE API. | -| `SharedSubnet` | Shared Services subnet for all things shared by TRE Management and Workspaces. Future Shared Services are Firewall Shared Service, Source Mirror Shared Service and Package Mirror Shared Service. | +See [networking](./networking.md). diff --git a/docs/networking.md b/docs/networking.md index a00fce11f4..1c57c312f6 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -6,16 +6,7 @@ Trusted Research Environment (TRE) network topology is based on [hub-spoke](http ![Network architecture](./assets/network-architecture.png) -Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The rules are managed in the `nsg-ws` Network Security Group (NSG): - -- Inbound traffic from TRE Management VNET to workspace allowed for [Azure Bastion](https://docs.microsoft.com/en-us/azure/bastion/bastion-overview) (22, 3389) - All other inbound traffic from Core to workspace denied. -- Outbound traffic to `SharedSubnet` from Workspace allowed. -- Outbound traffic to Internet allowed on HTTPS port 443 (next hop Azure Firewall). -- All other outbound traffic denied. - -> In Azure traffic between subnets are allowed except explicitly denied. - -Each of these rules can be managed per workspace. +Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The security rules are managed by `nsg-ws` network security group. See [workspace network security groups (NSG)](#workspaces) further down. Each workspace has a default route routing all egress traffic through the Azure Firewall, to ensure only explicitly allowed destinations on the Internet to be accessed. It is planned that all other subnet will use the same pattern (Issue [#421](https://github.com/microsoft/AzureTRE/issues/421)) @@ -24,7 +15,7 @@ The Azure Firewall rules are: - No default inbound rules – block all. - No default outbound rules – block all. -Inbound traffic from the Internet is only allowed through the Application Gateway, which forwards HTTPS (port 443) call to the TRE Management API in the `WebAppSubnet`. +Inbound traffic from the Internet is only allowed through the Application Gateway, which forwards HTTPS (port 443) call to the TRE API in the `WebAppSubnet`. | Subnet | Description | | -------| ----------- | @@ -32,23 +23,24 @@ Inbound traffic from the Internet is only allowed through the Application Gatewa | `AppGwSubnet` | Subnet for Azure Application Gateway controlling ingress traffic. | | `AzureFirewallSubnet` | Subnet for Azure Firewall controlling egress traffic. | | `ResourceProcessorSubnet` | Subnet for VMSS used by the Composition Service to host Docker containers to execute Porter bundles that deploys Workspaces. | -| `WebAppSubnet` | Subnet for Management API. | +| `WebAppSubnet` | Subnet for TRE API. | | `SharedSubnet` | Shared Services subnet for all things shared by TRE Management and Workspaces. Future Shared Services are Firewall Shared Service, Source Mirror Shared Service and Package Mirror Shared Service. | ## Network security groups -### TRE management/core +### TRE Management/core -Network security groups (NSG) for TRE core resources are defined in [`/templates/core/terraform/network/network_security_groups.tf`](../templates/core/terraform/network/network_security_groups.tf). +Network security groups (NSG) and their security rules for TRE core resources are defined in [`/templates/core/terraform/network/network_security_groups.tf`](../templates/core/terraform/network/network_security_groups.tf). -| Network security group | Associated subnets | -| ---------------------- | ------------------ | -| `nsg-default-rules` | | +| Network security group | Associated subnet(s) | +| ---------------------- | -------------------- | | `nsg-bastion-subnet` | `AzureBastionSubnet` | +| `nsg-app-gw` | `AppGwSubnet` | +| `nsg-default-rules` | `ResourceProcessorSubnet`, `SharedSubnet`, `WebAppSubnet` | ### Workspaces -Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The rules are managed by the `nsg-ws` network security group: +Azure TRE VNETs are segregated allowing limited traffic between the TRE Management VNET and Workspace VNETs. The rules to manage and limit the traffic between the TRE Management VNET and Workspace VNETs are defined by the `nsg-ws` network security group: - Inbound traffic from TRE Management VNET to workspace allowed for [Azure Bastion](https://docs.microsoft.com/en-us/azure/bastion/bastion-overview) (22, 3389) - All other inbound traffic from Core to workspace denied. - Outbound traffic to `SharedSubnet` from Workspace allowed. From 218f2018d1ad03e9bc31d6938379d729c34450e1 Mon Sep 17 00:00:00 2001 From: Tomi Paananen Date: Wed, 8 Sep 2021 11:26:56 +0000 Subject: [PATCH 5/7] Adds inbound internet rule to App Gateway NSG --- .../terraform/network/network_security_groups.tf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/templates/core/terraform/network/network_security_groups.tf b/templates/core/terraform/network/network_security_groups.tf index 8c1b4263ed..2473e1f6f3 100644 --- a/templates/core/terraform/network/network_security_groups.tf +++ b/templates/core/terraform/network/network_security_groups.tf @@ -125,6 +125,18 @@ resource "azurerm_network_security_group" "app_gw" { source_address_prefix = "GatewayManager" destination_address_prefix = "*" } + + security_rule { + name = "AllowInboundInternet" + priority = 3801 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefix = "Internet" + destination_address_prefix = "*" + } } resource "azurerm_subnet_network_security_group_association" "app_gw" { From 9ce8a5e449dfe646c7bc215756f774467df93698 Mon Sep 17 00:00:00 2001 From: Tomi Paananen Date: Thu, 9 Sep 2021 16:35:25 +0300 Subject: [PATCH 6/7] Update docs/networking.md Co-authored-by: Tess Ferrandez --- docs/networking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/networking.md b/docs/networking.md index 1c57c312f6..78e73d5a53 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -47,6 +47,6 @@ Azure TRE VNETs are segregated allowing limited traffic between the TRE Manageme - Outbound traffic to Internet allowed on HTTPS port 443 (next hop Azure Firewall). - All other outbound traffic denied. -> In Azure traffic between subnets are allowed except explicitly denied. +> In Azure, traffic between subnets are allowed except explicitly denied. Each of these rules can be managed per workspace. From 0855b039deec851a8e370bf0e94333c40600a00a Mon Sep 17 00:00:00 2001 From: Tomi Paananen Date: Thu, 9 Sep 2021 16:35:34 +0300 Subject: [PATCH 7/7] Update docs/networking.md Co-authored-by: Tess Ferrandez --- docs/networking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/networking.md b/docs/networking.md index 78e73d5a53..1ec0780582 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -1,6 +1,6 @@ # Networking -Trusted Research Environment (TRE) network topology is based on [hub-spoke](https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/hybrid-networking/hub-spoke). The TRE Management VNET ([Azure Virtual Network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview)) is the central hub and each workspace is a spoke. +The Trusted Research Environment (TRE) network topology is based on [hub-spoke](https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/hybrid-networking/hub-spoke). The TRE Management VNET ([Azure Virtual Network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview)) is the central hub and each workspace is a spoke. > Note: TRE Management is referred to as **core** in scripts and code.