Move log analytics to SRE

data_safe_haven/commands/sre.py

@@ -76,36 +76,11 @@ def deploy(
         )
         stack.add_option("azure-native:tenantId", config.azure.tenant_id, replace=False)
         # Load SHM stack outputs
-        stack.add_option(
-            "shm-monitoring-log_analytics_workspace_id",
-            shm_stack.output("monitoring")["log_analytics_workspace_id"],
-            replace=True,
-        )
-        stack.add_secret(
-            "shm-monitoring-log_analytics_workspace_key",
-            shm_stack.output("monitoring")["log_analytics_workspace_key"],
-            replace=True,
-        )
-        stack.add_option(
-            "shm-networking-private_dns_zone_base_id",
-            shm_stack.output("networking")["private_dns_zone_base_id"],
-            replace=True,
-        )
         stack.add_option(
             "shm-networking-resource_group_name",
             shm_stack.output("networking")["resource_group_name"],
             replace=True,
         )
-        stack.add_option(
-            "shm-networking-subnet_subnet_monitoring_prefix",
-            shm_stack.output("networking")["subnet_monitoring_prefix"],
-            replace=True,
-        )
-        stack.add_option(
-            "shm-networking-virtual_network_name",
-            shm_stack.output("networking")["virtual_network_name"],
-            replace=True,
-        )
 
         # Deploy Azure infrastructure with Pulumi
         if force is None:

data_safe_haven/functions/strings.py

@@ -87,7 +87,7 @@ def password(length: int) -> str:
 
 
 def replace_separators(input_string: str, separator: str = "") -> str:
-    """Return a string using underscores as a separator"""
+    """Return a string replacing all instances of [ _-.] with the desired separator."""
     return (
         input_string.replace(" ", separator)
         .replace("_", separator)

data_safe_haven/infrastructure/common/ip_ranges.py

@@ -24,6 +24,7 @@ def __init__(self, index: int) -> None:
         self.guacamole_containers = self.vnet.next_subnet(8)
         self.guacamole_containers_support = self.vnet.next_subnet(8)
         self.identity_containers = self.vnet.next_subnet(8)
+        self.monitoring = self.vnet.next_subnet(32)
         self.user_services_containers = self.vnet.next_subnet(8)
         self.user_services_containers_support = self.vnet.next_subnet(8)
         self.user_services_databases = self.vnet.next_subnet(8)

data_safe_haven/infrastructure/components/composite/virtual_machine.py

@@ -4,25 +4,24 @@
 from typing import Any
 
 from pulumi import ComponentResource, Input, Output, ResourceOptions
-from pulumi_azure_native import compute, maintenance, network
+from pulumi_azure_native import compute, insights, maintenance, network
 
 from data_safe_haven.functions import replace_separators
-from data_safe_haven.infrastructure.components.wrapped import (
-    WrappedLogAnalyticsWorkspace,
-)
 
 
 class VMComponentProps:
     """Properties for WindowsVMComponent"""
 
     image_reference_args: compute.ImageReferenceArgs | None
-    log_analytics_extension_name: str
-    log_analytics_extension_version: str
+    azure_monitor_extension_name: str
+    azure_monitor_extension_version: str
     os_profile_args: compute.OSProfileArgs | None
 
     def __init__(
         self,
         admin_password: Input[str],
+        data_collection_rule_id: Input[str],
+        data_collection_endpoint_id: Input[str],
         ip_address_private: Input[str],
         location: Input[str],
         resource_group_name: Input[str],
@@ -33,40 +32,20 @@ def __init__(
         vm_size: Input[str],
         admin_username: Input[str] | None = None,
         ip_address_public: Input[bool] | None = None,
-        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace] | None = None,
-        log_analytics_workspace_id: Input[str] | None = None,
-        log_analytics_workspace_key: Input[str] | None = None,
         maintenance_configuration_id: Input[str] | None = None,
     ) -> None:
         self.admin_password = admin_password
         self.admin_username = admin_username if admin_username else "dshvmadmin"
+        self.data_collection_rule_id = data_collection_rule_id
+        self.data_collection_rule_name = Output.from_input(
+            data_collection_rule_id
+        ).apply(lambda rule_id: str(rule_id).split("/")[-1])
+        self.data_collection_endpoint_id = data_collection_endpoint_id
         self.image_reference_args = None
         self.ip_address_private = ip_address_private
         self.ip_address_public = ip_address_public
         self.location = location
         self.maintenance_configuration_id = maintenance_configuration_id
-        if log_analytics_workspace:
-            self.log_analytics_workspace_id = Output.from_input(
-                log_analytics_workspace
-            ).apply(lambda workspace: workspace.workspace_id)
-        elif log_analytics_workspace_id:
-            self.log_analytics_workspace_id = Output.from_input(
-                log_analytics_workspace_id
-            )
-        else:
-            msg = "No value provided for 'log_analytics_workspace_id'"
-            raise ValueError(msg)
-        if log_analytics_workspace:
-            self.log_analytics_workspace_key = Output.from_input(
-                log_analytics_workspace
-            ).apply(lambda workspace: workspace.workspace_key)
-        elif log_analytics_workspace_key:
-            self.log_analytics_workspace_key = Output.from_input(
-                log_analytics_workspace_key
-            )
-        else:
-            msg = "No value provided for 'log_analytics_workspace_key'"
-            raise ValueError(msg)
         self.os_profile_args = None
         self.resource_group_name = resource_group_name
         self.subnet_name = subnet_name
@@ -114,8 +93,8 @@ def __init__(
                 provision_vm_agent=True,
             ),
         )
-        self.log_analytics_extension_name = "MicrosoftMonitoringAgent"
-        self.log_analytics_extension_version = "1.0"
+        self.azure_monitor_extension_name = "AzureMonitorWindowsAgent"
+        self.azure_monitor_extension_version = "1.0"
 
 
 class LinuxVMComponentProps(VMComponentProps):
@@ -151,8 +130,8 @@ def __init__(
                 provision_vm_agent=True,
             ),
         )
-        self.log_analytics_extension_name = "OmsAgentForLinux"
-        self.log_analytics_extension_version = "1.0"
+        self.azure_monitor_extension_name = "AzureMonitorLinuxAgent"
+        self.azure_monitor_extension_version = "1.0"
 
 
 class VMComponent(ComponentResource):
@@ -259,22 +238,16 @@ def __init__(
         )
 
         # Register with Log Analytics workspace
-        log_analytics_extension = compute.VirtualMachineExtension(
-            f"{name_underscored}_log_analytics_extension",
+        compute.VirtualMachineExtension(
+            f"{name_underscored}_azure_monitor_extension",
             auto_upgrade_minor_version=True,
-            enable_automatic_upgrade=False,
+            enable_automatic_upgrade=True,
             location=props.location,
-            publisher="Microsoft.EnterpriseCloud.Monitoring",
-            protected_settings=props.log_analytics_workspace_key.apply(
-                lambda key: {"workspaceKey": key}
-            ),
+            publisher="Microsoft.Azure.Monitor",
             resource_group_name=props.resource_group_name,
-            settings=props.log_analytics_workspace_id.apply(
-                lambda id_: {"workspaceId": id_}
-            ),
-            type=props.log_analytics_extension_name,
-            type_handler_version=props.log_analytics_extension_version,
-            vm_extension_name=props.log_analytics_extension_name,
+            type=props.azure_monitor_extension_name,
+            type_handler_version=props.azure_monitor_extension_version,
+            vm_extension_name=props.azure_monitor_extension_name,
             vm_name=virtual_machine.name,
             opts=ResourceOptions.merge(
                 child_opts,
@@ -285,7 +258,10 @@ def __init__(
 
         # Register with maintenance configuration
         maintenance.ConfigurationAssignment(
-            f"{name_underscored}_configurationAssignment",
+            f"{name_underscored}_configuration_assignment",
+            configuration_assignment_name=Output.concat(
+                props.vm_name, "-maintenance-configuration"
+            ),
             location=props.location,
             maintenance_configuration_id=props.maintenance_configuration_id,
             provider_name="Microsoft.Compute",
@@ -298,11 +274,36 @@ def __init__(
             ),
         )
 
+        # Register with data collection rule
+        insights.DataCollectionRuleAssociation(
+            f"{name_underscored}_dcra_to_dcr",
+            association_name=Output.concat(
+                props.data_collection_rule_name, "-association"  # this name is required
+            ),
+            data_collection_rule_id=props.data_collection_rule_id,
+            resource_uri=virtual_machine.id,
+            opts=ResourceOptions.merge(
+                child_opts,
+                ResourceOptions(parent=virtual_machine),
+            ),
+        )
+
+        # Register with data collection endpoint
+        insights.DataCollectionRuleAssociation(
+            f"{name_underscored}_dcra_to_dce",
+            association_name="configurationAccessEndpoint",  # this name is required
+            data_collection_endpoint_id=props.data_collection_endpoint_id,
+            resource_uri=virtual_machine.id,
+            opts=ResourceOptions.merge(
+                child_opts,
+                ResourceOptions(parent=virtual_machine),
+            ),
+        )
+
         # Register outputs
         self.ip_address_private: Output[str] = Output.from_input(
             props.ip_address_private
         )
-        self.log_analytics_extension = log_analytics_extension
         self.resource_group_name: Output[str] = Output.from_input(
             props.resource_group_name
         )

data_safe_haven/infrastructure/programs/declarative_shm.py

@@ -5,7 +5,6 @@
 from data_safe_haven.config import Config
 from data_safe_haven.context import Context
 
-from .shm.monitoring import SHMMonitoringComponent, SHMMonitoringProps
 from .shm.networking import SHMNetworkingComponent, SHMNetworkingProps
 
 
@@ -39,20 +38,5 @@ def __call__(self) -> None:
             tags=self.tags,
         )
 
-        # Deploy automated monitoring
-        monitoring = SHMMonitoringComponent(
-            "shm_monitoring",
-            self.stack_name,
-            SHMMonitoringProps(
-                dns_resource_group_name=networking.resource_group_name,
-                location=self.context.location,
-                private_dns_zone_base_id=networking.private_dns_zone_base_id,
-                subnet_monitoring=networking.subnet_monitoring,
-                timezone=self.cfg.shm.timezone,
-            ),
-            tags=self.tags,
-        )
-
         # Export values for later use
-        pulumi.export("monitoring", monitoring.exports)
         pulumi.export("networking", networking.exports)

data_safe_haven/infrastructure/programs/declarative_sre.py

@@ -4,10 +4,6 @@
 
 from data_safe_haven.config import Config
 from data_safe_haven.context import Context
-from data_safe_haven.infrastructure.programs.sre.maintenance import (
-    SREMaintenanceComponent,
-    SREMaintenanceProps,
-)
 
 from .sre.application_gateway import (
     SREApplicationGatewayComponent,
@@ -34,6 +30,10 @@
     SREIdentityComponent,
     SREIdentityProps,
 )
+from .sre.monitoring import (
+    SREMonitoringComponent,
+    SREMonitoringProps,
+)
 from .sre.networking import (
     SRENetworkingComponent,
     SRENetworkingProps,
@@ -143,6 +143,7 @@ def __call__(self) -> None:
             "sre_networking",
             self.stack_name,
             SRENetworkingProps(
+                dns_private_zones=dns.private_zones,
                 dns_resource_group_name=dns.resource_group.name,
                 dns_server_ip=dns.ip_address,
                 dns_virtual_network=dns.virtual_network,
@@ -151,12 +152,6 @@ def __call__(self) -> None:
                 shm_networking_resource_group_name=self.pulumi_opts.require(
                     "shm-networking-resource_group_name"
                 ),
-                shm_subnet_monitoring_prefix=self.pulumi_opts.require(
-                    "shm-networking-subnet_subnet_monitoring_prefix",
-                ),
-                shm_virtual_network_name=self.pulumi_opts.require(
-                    "shm-networking-virtual_network_name"
-                ),
                 shm_zone_name=self.cfg.shm.fqdn,
                 sre_index=self.cfg.sre(self.sre_name).index,
                 sre_name=self.sre_name,
@@ -197,11 +192,11 @@ def __call__(self) -> None:
                 data_provider_ip_addresses=self.cfg.sre(
                     self.sre_name
                 ).data_provider_ip_addresses,
+                dns_private_zones=dns.private_zones,
                 dns_record=networking.shm_ns_record,
                 dns_server_admin_password=dns.password_admin,
                 location=self.context.location,
                 networking_resource_group=networking.resource_group,
-                pulumi_opts=self.pulumi_opts,
                 sre_fqdn=networking.sre_fqdn,
                 subnet_data_configuration=networking.subnet_data_configuration,
                 subnet_data_private=networking.subnet_data_private,
@@ -317,7 +312,6 @@ def __call__(self) -> None:
                 nexus_admin_password=data.password_nexus_admin,
                 software_packages=self.cfg.sre(self.sre_name).software_packages,
                 sre_fqdn=networking.sre_fqdn,
-                sre_private_dns_zone_id=networking.sre_private_dns_zone_id,
                 storage_account_key=data.storage_account_data_configuration_key,
                 storage_account_name=data.storage_account_data_configuration_name,
                 storage_account_resource_group_name=data.resource_group_name,
@@ -329,13 +323,14 @@ def __call__(self) -> None:
             tags=self.tags,
         )
 
-        # Deploy maintenance configuration
-        maintenance = SREMaintenanceComponent(
-            "sre_maintenance",
+        # Deploy monitoring
+        monitoring = SREMonitoringComponent(
+            "sre_monitoring",
             self.stack_name,
-            SREMaintenanceProps(
+            SREMonitoringProps(
+                dns_private_zones=dns.private_zones,
                 location=self.context.location,
-                resource_group_name=data.resource_group_name,
+                subnet=networking.subnet_monitoring,
                 timezone=self.cfg.shm.timezone,
             ),
             tags=self.tags,
@@ -348,20 +343,16 @@ def __call__(self) -> None:
             SREWorkspacesProps(
                 admin_password=data.password_workspace_admin,
                 apt_proxy_server_hostname=apt_proxy_server.hostname,
+                data_collection_rule_id=monitoring.data_collection_rule_vms.id,
+                data_collection_endpoint_id=monitoring.data_collection_endpoint.id,
                 ldap_group_filter=ldap_group_filter,
                 ldap_group_search_base=ldap_group_search_base,
                 ldap_server_hostname=identity.hostname,
                 ldap_server_port=identity.server_port,
                 ldap_user_filter=ldap_user_filter,
                 ldap_user_search_base=ldap_user_search_base,
                 location=self.context.location,
-                log_analytics_workspace_id=self.pulumi_opts.require(
-                    "shm-monitoring-log_analytics_workspace_id"
-                ),
-                log_analytics_workspace_key=self.pulumi_opts.require(
-                    "shm-monitoring-log_analytics_workspace_key"
-                ),
-                maintenance_configuration_id=maintenance.configuration_id,
+                maintenance_configuration_id=monitoring.maintenance_configuration.id,
                 software_repository_hostname=user_services.software_repositories.hostname,
                 sre_name=self.sre_name,
                 storage_account_data_private_user_name=data.storage_account_data_private_user_name,