Fix eventhub diagnostics profiles

.github/workflows/master-standalone-tf13.yaml

@@ -57,6 +57,7 @@ jobs:
             "cosmos_db/100-simple-cosmos-db-table",
             "databricks/100-simple-databricks",
             "datalake/101-datalake-storage",
+            "diagnostics_profiles/200-diagnostics-eventhub-namespaces",
             "eventhub/100-simple-eventhub-namespace",
             "eventhub/101-evh-namespace-with-private-endpoint",
             "eventhub/102-namespace-and-evh-with-auth-rules",

examples/diagnostics_profiles/200-diagnostics-eventhub-namespaces/configuration.tfvars

@@ -0,0 +1,13 @@
+global_settings = {
+  default_region = "region1"
+  regions = {
+    region1 = "southeastasia"
+  }
+}
+
+
+resource_groups = {
+  ops = {
+    name = "operations"
+  }
+}

examples/diagnostics_profiles/200-diagnostics-eventhub-namespaces/diagnostic_event_hub_namespaces.tfvars

@@ -0,0 +1,20 @@
+
+# Event hub diagnostics
+diagnostic_event_hub_namespaces = {
+  central_logs_region1 = {
+    name               = "logs"
+    resource_group_key = "ops"
+    sku                = "Standard"
+    region             = "region1"
+
+    diagnostic_profiles = {
+      central_logs_region1 = {
+        definition_key   = "event_hub_namespace"
+        destination_type = "event_hub"
+        destination_key  = "central_logs"
+      }
+    }
+  }
+}
+
+

examples/diagnostics_profiles/200-diagnostics-eventhub-namespaces/diagnostics_definition.tfvars

@@ -0,0 +1,274 @@
+
+#
+# Define a set of settings for the various type of Azure resources
+#
+
+diagnostics_definition = {
+  log_analytics = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["Audit", true, false, 7],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+
+  }
+
+  default_all = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AuditEvent", true, false, 7],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+
+  }
+
+  bastion_host = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["BastionAuditLogs", true, false, 7],
+      ]
+    }
+
+  }
+
+  networking_all = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["VMProtectionAlerts", true, false, 7],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+
+  }
+
+  public_ip_address = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["DDoSProtectionNotifications", true, false, 7],
+        ["DDoSMitigationFlowLogs", true, false, 7],
+        ["DDoSMitigationReports", true, false, 7],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+
+  }
+
+  network_security_group = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["NetworkSecurityGroupEvent", true, false, 7],
+        ["NetworkSecurityGroupRuleCounter", true, false, 7],
+      ]
+    }
+
+  }
+
+  network_interface_card = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      # log = [
+      #   # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+      #   ["AuditEvent", true, false, 7],
+      # ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+
+  }
+
+  azure_container_registry = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["ContainerRegistryRepositoryEvents", true, false, 7],
+        ["ContainerRegistryLoginEvents", true, false, 7],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+  }
+
+  azure_kubernetes_cluster = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["kube-apiserver", true, false, 7],
+        ["kube-audit", true, false, 7],
+        ["kube-audit-admin", true, false, 7],
+        ["kube-controller-manager", true, false, 7],
+        ["kube-scheduler", true, false, 7],
+        ["cluster-autoscaler", true, false, 7],
+        ["guard", true, false, 7],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+  }
+
+  azure_site_recovery = {
+    name                           = "operational_logs_and_metrics"
+    log_analytics_destination_type = "Dedicated"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AzureBackupReport", true, true, 7],
+        ["CoreAzureBackup", true, true, 7],
+        ["AddonAzureBackupAlerts", true, true, 7],
+        ["AddonAzureBackupJobs", true, true, 7],
+        ["AddonAzureBackupPolicy", true, true, 7],
+        ["AddonAzureBackupProtectedInstance", true, true, 7],
+        ["AddonAzureBackupStorage", true, true, 7],
+        ["AzureSiteRecoveryJobs", true, true, 7],
+        ["AzureSiteRecoveryEvents", true, true, 7],
+        ["AzureSiteRecoveryReplicatedItems", true, true, 7],
+        ["AzureSiteRecoveryReplicationStats", true, true, 7],
+        ["AzureSiteRecoveryRecoveryPoints", true, true, 7],
+        ["AzureSiteRecoveryReplicationDataUploadRate", true, true, 7],
+        ["AzureSiteRecoveryProtectedDiskDataChurn", true, true, 30],
+      ]
+      metric = [
+        #["AllMetrics", 60, True],
+      ]
+    }
+
+  }
+
+  azure_automation = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["JobLogs", true, true, 30],
+        ["JobStreams", true, true, 30],
+        ["DscNodeStatus", true, true, 30],
+      ]
+      metric = [
+        # ["Category name",  "Metric Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, true, 30],
+      ]
+    }
+
+  }
+
+  event_hub_namespace = {
+    name = "operational_logs_and_metrics"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["ArchiveLogs", true, false, 7],
+        ["OperationalLogs", true, false, 7],
+        ["AutoScaleLogs", true, false, 7],
+        ["KafkaCoordinatorLogs", true, false, 7],
+        ["KafkaUserErrorLogs", true, false, 7],
+        ["EventHubVNetConnectionEvent", true, false, 7],
+        ["CustomerManagedKeyUserLogs", true, false, 7],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", true, false, 7],
+      ]
+    }
+
+  }
+
+  compliance_all = {
+    name = "compliance_logs"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AuditEvent", true, true, 365],
+      ]
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", false, false, 7],
+      ]
+    }
+
+  }
+
+  siem_all = {
+    name = "siem"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AuditEvent", true, true, 0],
+      ]
+
+      metric = [
+        #["Category name",  "Diagnostics Enabled(true/false)", "Retention Enabled(true/false)", Retention_period]
+        ["AllMetrics", false, false, 0],
+      ]
+    }
+
+  }
+
+  subscription_operations = {
+    name = "subscription_operations"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)"]
+        ["Administrative", true],
+        ["Security", true],
+        ["ServiceHealth", true],
+        ["Alert", true],
+        ["Policy", true],
+        ["Autoscale", true],
+        ["ResourceHealth", true],
+        ["Recommendation", true],
+      ]
+    }
+  }
+
+  subscription_siem = {
+    name = "activity_logs_for_siem"
+    categories = {
+      log = [
+        # ["Category name",  "Diagnostics Enabled(true/false)"]
+        ["Administrative", false],
+        ["Security", true],
+        ["ServiceHealth", false],
+        ["Alert", false],
+        ["Policy", true],
+        ["Autoscale", false],
+        ["ResourceHealth", false],
+        ["Recommendation", false],
+      ]
+    }
+
+  }
+
+}

examples/diagnostics_profiles/200-diagnostics-eventhub-namespaces/diagnostics_destinations.tfvars

@@ -0,0 +1,15 @@
+# Defines the different destination for the different log profiles
+# Different profiles to target different operational teams
+
+diagnostics_destinations = {
+  # Storage keys must reference the azure region name
+  # For storage, reference "all_regions" and we will send the logs to the storage account
+  # in the region of the deployment
+
+
+  event_hub_namespaces = {
+    central_logs = {
+      event_hub_namespace_key = "central_logs_region1"
+    }
+  }
+}

examples/diagnostics_profiles/200-diagnostics-eventhub-namespaces/standalone/ci.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+current_folder=$(pwd)
+cd standalone
+
+terraform init
+
+terraform apply \
+  -var-file ../configuration.tfvars \
+  -var-file  ../diagnostic_event_hub_namespaces.tfvars \
+  -var-file  ../diagnostics_definition.tfvars \
+  -var-file  ../diagnostics_destinations.tfvars \
+  -var tags='{testing_job_id='"${1}"'}' \
+  -var var_folder_path=${current_folder} \
+  -input=false \
+  -auto-approve
+
+
+terraform destroy \
+  -var-file ../configuration.tfvars \
+  -var-file  ../diagnostic_event_hub_namespaces.tfvars \
+  -var-file  ../diagnostics_definition.tfvars \
+  -var-file  ../diagnostics_destinations.tfvars \
+  -var tags='{testing_job_id='"${1}"'}' \
+  -var var_folder_path=${current_folder} \
+  -input=false \
+  -auto-approve
+