diff --git a/.gitignore b/.gitignore index a043b16a3..9496a3682 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ **/*.log **/backend.azurerm.tf public -aztfmod \ No newline at end of file +aztfmod +*output.json \ No newline at end of file diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index 7218da3d5..5b90984a0 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -1,15 +1,10 @@ module "launchpad" { - source = "aztfmod/caf/azurerm" - version = "~>5.3.2" + # source = "aztfmod/caf/azurerm" + # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" - # azuread = var.azuread - azuread_api_permissions = var.azuread_api_permissions - azuread_apps = var.azuread_apps - azuread_groups = var.azuread_groups - azuread_roles = var.azuread_roles - azuread_users = var.azuread_users current_landingzone_key = var.landingzone.key custom_role_definitions = var.custom_role_definitions enable = var.enable @@ -22,6 +17,7 @@ module "launchpad" { logged_aad_app_objectId = var.logged_aad_app_objectId logged_user_objectId = var.logged_user_objectId managed_identities = var.managed_identities + remote_objects = local.remote resource_groups = var.resource_groups role_mapping = var.role_mapping storage_accounts = var.storage_accounts @@ -30,6 +26,18 @@ module "launchpad" { tenant_id = var.tenant_id user_type = var.user_type + azuread = { + azuread_api_permissions = var.azuread_api_permissions + azuread_applications = var.azuread_applications + azuread_apps = var.azuread_apps + azuread_credential_policies = var.azuread_credential_policies + azuread_groups = var.azuread_groups + azuread_roles = var.azuread_roles + azuread_service_principal_passwords = var.azuread_service_principal_passwords + azuread_service_principals = var.azuread_service_principals + azuread_users = var.azuread_users + } + diagnostics = { diagnostics_definition = try(var.diagnostics.diagnostics_definition, var.diagnostics_definition) diagnostics_destinations = try(var.diagnostics.diagnostics_destinations, var.diagnostics_destinations) diff --git a/caf_launchpad/local.remote.tf b/caf_launchpad/local.remote.tf new file mode 100644 index 000000000..bb78a9300 --- /dev/null +++ b/caf_launchpad/local.remote.tf @@ -0,0 +1,7 @@ +locals { + remote = { + azuread_service_principals = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_service_principals, {})) + } + } +} \ No newline at end of file diff --git a/caf_launchpad/locals.remote_tfstates.tf b/caf_launchpad/locals.remote_tfstates.tf new file mode 100644 index 000000000..88c96e810 --- /dev/null +++ b/caf_launchpad/locals.remote_tfstates.tf @@ -0,0 +1,36 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] +} + +locals { + + remote_state = { + + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = value.workspace + key = value.tfstate + resource_group_name = value.resource_group_name + storage_account_name = value.storage_account_name + subscription_id = value.subscription_id + tenant_id = value.tenant_id + sas_token = try(value.sas_token, null) != null ? var.sas_token : null + } + } + + } + +} \ No newline at end of file diff --git a/caf_launchpad/variables.tf b/caf_launchpad/variables.tf index 847b0df2c..55482506c 100644 --- a/caf_launchpad/variables.tf +++ b/caf_launchpad/variables.tf @@ -16,6 +16,10 @@ variable "tenant_id" {} variable "landingzone" { description = "The landing zone name is used to reference the tfstate in configuration files. Therefore while set it is recommended not to change" } +variable "sas_token" { + description = "SAS Token to access the remote state in another Azure AD tenant." + default = null +} variable "passthrough" { default = false @@ -100,6 +104,15 @@ variable "azuread_users" { variable "azuread_roles" { default = {} } +variable "azuread_credential_policies" { + default = {} +} +variable "azuread_service_principals" { + default = {} +} +variable "azuread_service_principal_passwords" { + default = {} +} variable "managed_identities" { default = {} } @@ -152,6 +165,10 @@ variable "azuread_api_permissions" { default = {} } +variable "azuread_applications" { + default = {} +} + variable "environment" { type = string description = "This variable is set by the rover during the deployment based on the -env or -environment flags. Default to sandpit" diff --git a/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml b/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml index c430b2385..c8e5a083f 100644 --- a/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml +++ b/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml @@ -1,3 +1,5 @@ +# https://github.com/Azure/aad-pod-identity/blob/b3ee1d07209f26c47a96abf3ba20749932763de6/website/content/en/docs/Concepts/azureidentity.md + apiVersion: aadpodidentity.k8s.io/v1 kind: AzureIdentity metadata: @@ -13,4 +15,5 @@ metadata: name: podmi-gitlab-runner-binding spec: azureIdentity: podmi-caf-rover-platform-level0 - selector: podmi-caf-rover-platform-level0 \ No newline at end of file + selector: podmi-caf-rover-platform-level0 + diff --git a/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf b/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf index 05f60b4d1..f95d17c5f 100644 --- a/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf +++ b/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf @@ -1,3 +1,5 @@ +# https://github.com/Azure/aad-pod-identity/blob/b3ee1d07209f26c47a96abf3ba20749932763de6/website/content/en/docs/Concepts/azureidentity.md + resource "kubernetes_namespace" "ns" { metadata { name = var.aad_pod_identity.namespace diff --git a/caf_solution/add-ons/aad-pod-identity/variables.tf b/caf_solution/add-ons/aad-pod-identity/variables.tf index 971033737..3e960c0c7 100644 --- a/caf_solution/add-ons/aad-pod-identity/variables.tf +++ b/caf_solution/add-ons/aad-pod-identity/variables.tf @@ -4,7 +4,7 @@ variable "lower_container_name" {} variable "lower_resource_group_name" {} variable "tfstate_subscription_id" { - description = "This value is propulated by the rover. subscription id hosting the remote tfstates" + description = "This value is populated by the rover. subscription id hosting the remote tfstates" } variable "tfstate_storage_account_name" {} variable "tfstate_container_name" {} diff --git a/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf index 96c672df0..444eed636 100644 --- a/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf +++ b/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf @@ -56,3 +56,13 @@ locals { ) : format("%s-%s", msi.key, msi.msi_key) => msi } } + +resource "azurerm_key_vault_access_policy" "keyvault_policy" { + # provider = azurerm.launchpad + for_each = var.keyvaults + + key_vault_id = local.remote.keyvaults[each.value.lz_key][each.value.key].id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_clusters[var.aks_cluster_key].key].kubelet_identity[0].object_id + secret_permissions = each.value.secret_permissions +} diff --git a/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf b/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf index d458c18a1..c30b190de 100644 --- a/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf +++ b/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf @@ -43,6 +43,9 @@ locals { aks_clusters = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) } + keyvaults = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) + } managed_identities = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) } diff --git a/caf_solution/add-ons/aks-secure-baseline/providers.tf b/caf_solution/add-ons/aks-secure-baseline/providers.tf index 06e22a736..8d20ad216 100644 --- a/caf_solution/add-ons/aks-secure-baseline/providers.tf +++ b/caf_solution/add-ons/aks-secure-baseline/providers.tf @@ -4,6 +4,13 @@ provider "azurerm" { } } +provider "azurerm" { + alias = "launchpad" + subscription_id = var.tfstate_subscription_id + features { + } +} + provider "kubernetes" { host = local.k8sconfigs[var.aks_cluster_key].host username = local.k8sconfigs[var.aks_cluster_key].username @@ -31,6 +38,8 @@ locals { } } +data "azurerm_client_config" "current" {} + # Get kubeconfig from AKS clusters data "azurerm_kubernetes_cluster" "kubeconfig" { for_each = var.aks_clusters diff --git a/caf_solution/add-ons/aks-secure-baseline/variables.tf b/caf_solution/add-ons/aks-secure-baseline/variables.tf index 971033737..010f75139 100644 --- a/caf_solution/add-ons/aks-secure-baseline/variables.tf +++ b/caf_solution/add-ons/aks-secure-baseline/variables.tf @@ -33,4 +33,7 @@ variable "managed_identities" { description = "Map of the user managed identities." } -variable "aad_pod_identity" {} \ No newline at end of file +variable "aad_pod_identity" {} +variable "keyvaults" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/app/main.tf b/caf_solution/add-ons/aks_applications/app/main.tf index 1191eedce..c45957ba5 100644 --- a/caf_solution/add-ons/aks_applications/app/main.tf +++ b/caf_solution/add-ons/aks_applications/app/main.tf @@ -1,7 +1,13 @@ -provider "kubernetes" { - alias = "k8s" -} - -provider "helm" { - alias = "helm" -} +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + } + helm = { + source = "hashicorp/helm" + } + kustomization = { + source = "kbst/kustomization" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/app/module.tf b/caf_solution/add-ons/aks_applications/app/module.tf index 2d4288893..138110a32 100644 --- a/caf_solution/add-ons/aks_applications/app/module.tf +++ b/caf_solution/add-ons/aks_applications/app/module.tf @@ -6,7 +6,6 @@ resource "kubernetes_namespace" "namespaces" { name = each.value.name } - provider = kubernetes.k8s } # https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release @@ -22,6 +21,7 @@ resource "helm_release" "charts" { timeout = try(each.value.timeout, 900) skip_crds = try(each.value.skip_crds, false) create_namespace = try(each.value.create_namespace, false) + values = try(each.value.values, null) dynamic "set" { for_each = try(each.value.sets, {}) @@ -39,9 +39,8 @@ resource "helm_release" "charts" { } } - provider = helm.helm - depends_on = [kubernetes_namespace.namespaces] + # depends_on = [kubernetes_namespace.namespaces] # values = [ # "${file("values.yaml")}" # ] diff --git a/caf_solution/add-ons/aks_applications/app/variables.tf b/caf_solution/add-ons/aks_applications/app/variables.tf index 4a0225192..cb3a56bfc 100644 --- a/caf_solution/add-ons/aks_applications/app/variables.tf +++ b/caf_solution/add-ons/aks_applications/app/variables.tf @@ -1,5 +1,11 @@ -variable "cluster" {} +variable "namespaces" { + default = {} +} -variable "namespaces" {} +variable "helm_charts" { + default = {} +} -variable "helm_charts" {} \ No newline at end of file +variable "kuztomization_settings" { + default = {} +} diff --git a/caf_solution/add-ons/aks_applications/applications.tf b/caf_solution/add-ons/aks_applications/applications.tf index b1f674491..d72da73f4 100644 --- a/caf_solution/add-ons/aks_applications/applications.tf +++ b/caf_solution/add-ons/aks_applications/applications.tf @@ -1,27 +1,5 @@ -module "app1" { +module "app" { source = "./app" - for_each = try(local.clusters[var.cluster_re1_key], null) != null ? { (var.cluster_re1_key) = local.clusters[var.cluster_re1_key] } : {} - - cluster = each.value namespaces = var.namespaces helm_charts = var.helm_charts - - providers = { - kubernetes.k8s = kubernetes.k8s1 - helm.helm = helm.helm1 - } } - -module "app2" { - source = "./app" - for_each = try(local.clusters[var.cluster_re2_key], null) != null ? { (var.cluster_re2_key) = local.clusters[var.cluster_re2_key] } : {} - - cluster = each.value - namespaces = var.namespaces - helm_charts = var.helm_charts - - providers = { - kubernetes.k8s = kubernetes.k8s2 - helm.helm = helm.helm2 - } -} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/kustomization.tf b/caf_solution/add-ons/aks_applications/kustomization.tf new file mode 100644 index 000000000..478352120 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomization.tf @@ -0,0 +1,31 @@ +module "kustomization" { + source = "./kustomize" + for_each = try(data.kustomization_overlay.manifest, {}) + + settings = each.value + +} + +data "kustomization_overlay" "manifest" { + for_each = var.kustomization_overlays + + resources = each.value.resources + + namespace = each.value.namespace + + dynamic "patches"{ + for_each = try(each.value.patches, {}) + content { + patch = patches.value.patch + target = patches.value.target + } + } + kustomize_options = { + load_restrictor = "none" + } +} + +output "manifests" { + value = data.kustomization_overlay.manifest +} + diff --git a/caf_solution/add-ons/aks_applications/kustomize/kustomization_build.tf b/caf_solution/add-ons/aks_applications/kustomize/kustomization_build.tf new file mode 100644 index 000000000..985bd96f5 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomize/kustomization_build.tf @@ -0,0 +1,16 @@ +resource "kustomization_resource" "p0" { + for_each = var.settings.ids_prio[0] + manifest = var.settings.manifests[each.value] +} + +resource "kustomization_resource" "p1" { + depends_on = [kustomization_resource.p0] + for_each = var.settings.ids_prio[1] + manifest = var.settings.manifests[each.value] +} + +resource "kustomization_resource" "p2" { + depends_on = [kustomization_resource.p1] + for_each = var.settings.ids_prio[2] + manifest = var.settings.manifests[each.value] +} diff --git a/caf_solution/add-ons/aks_applications/kustomize/main.tf b/caf_solution/add-ons/aks_applications/kustomize/main.tf new file mode 100644 index 000000000..e65c6fa22 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomize/main.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + kustomization = { + source = "kbst/kustomization" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/kustomize/variables.tf b/caf_solution/add-ons/aks_applications/kustomize/variables.tf new file mode 100644 index 000000000..f5c321890 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomize/variables.tf @@ -0,0 +1,2 @@ +variable "settings" { +} diff --git a/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf b/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf index c4cf88f8a..3cee0f923 100644 --- a/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf +++ b/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf @@ -19,8 +19,9 @@ data "terraform_remote_state" "remote" { backend = var.landingzone.backend_type config = { storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name - container_name = local.landingzone[try(each.value.level, "current")].container_name + container_name = try(each.value.container, local.landingzone[try(each.value.level, "current")].container_name) resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id key = each.value.tfstate } } @@ -33,22 +34,23 @@ locals { tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics remote = { + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + global_settings = local.global_settings + diagnostics = local.diagnostics + + aks_clusters = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) } - } - - clusters = local.remote.aks_clusters[var.landingzone.global_settings_key] - k8sconfigs = { for key, value in values(local.clusters) : key => { - host = value.enable_rbac ? value.kube_admin_config.0.host : value.kube_config.0.host - username = value.enable_rbac ? value.kube_admin_config.0.username : value.kube_config.0.username - password = value.enable_rbac ? value.kube_admin_config.0.password : value.kube_config.0.password - client_certificate = value.enable_rbac ? base64decode(value.kube_admin_config.0.client_certificate) : base64decode(value.kube_config.0.client_certificate) - client_key = value.enable_rbac ? base64decode(value.kube_admin_config.0.client_key) : base64decode(value.kube_config.0.client_key) - cluster_ca_certificate = value.enable_rbac ? base64decode(value.kube_admin_config.0.cluster_ca_certificate) : base64decode(value.kube_config.0.cluster_ca_certificate) + managed_identities = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) + } + vnets = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) } } -} +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/main.tf b/caf_solution/add-ons/aks_applications/main.tf index 2222bc4d2..6b7201fb4 100644 --- a/caf_solution/add-ons/aks_applications/main.tf +++ b/caf_solution/add-ons/aks_applications/main.tf @@ -2,74 +2,20 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 2.43" + version = "~> 2.55.0" } kubernetes = { source = "hashicorp/kubernetes" - version = "~> 1.13.2" + version = "~> 2.0.2" } helm = { source = "hashicorp/helm" - version = "~> 1.3.0" + version = "~> 2.1.2" } - } - required_version = ">= 0.13" -} - - -provider "azurerm" { - features { - key_vault { - purge_soft_delete_on_destroy = true + kustomization = { + source = "kbst/kustomization" + version = "~> 0.5.0" } } -} - - -provider "kubernetes" { - load_config_file = false - host = try(local.k8sconfigs[0].host, null) - username = try(local.k8sconfigs[0].username, null) - password = try(local.k8sconfigs[0].password, null) - client_certificate = try(local.k8sconfigs[0].client_certificate, null) - client_key = try(local.k8sconfigs[0].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[0].cluster_ca_certificate, null) - alias = "k8s1" -} - -provider "helm" { - kubernetes { - load_config_file = false - host = try(local.k8sconfigs[0].host, null) - username = try(local.k8sconfigs[0].username, null) - password = try(local.k8sconfigs[0].password, null) - client_certificate = try(local.k8sconfigs[0].client_certificate, null) - client_key = try(local.k8sconfigs[0].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[0].cluster_ca_certificate, null) - } - alias = "helm1" -} - -provider "kubernetes" { - load_config_file = false - host = try(local.k8sconfigs[1].host, null) - username = try(local.k8sconfigs[1].username, null) - password = try(local.k8sconfigs[1].password, null) - client_certificate = try(local.k8sconfigs[1].client_certificate, null) - client_key = try(local.k8sconfigs[1].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[1].cluster_ca_certificate, null) - alias = "k8s2" -} - -provider "helm" { - kubernetes { - load_config_file = false - host = try(local.k8sconfigs[1].host, null) - username = try(local.k8sconfigs[1].username, null) - password = try(local.k8sconfigs[1].password, null) - client_certificate = try(local.k8sconfigs[1].client_certificate, null) - client_key = try(local.k8sconfigs[1].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[1].cluster_ca_certificate, null) - } - alias = "helm2" + required_version = ">= 0.13" } \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/providers.tf b/caf_solution/add-ons/aks_applications/providers.tf new file mode 100644 index 000000000..e8b9dcfc5 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/providers.tf @@ -0,0 +1,54 @@ +provider "azurerm" { + features { + } +} + +provider "kubernetes" { + host = try(local.k8sconfigs[var.aks_cluster_key].host, null) + username = try(local.k8sconfigs[var.aks_cluster_key].username, null) + password = try(local.k8sconfigs[var.aks_cluster_key].password, null) + client_certificate = try(local.k8sconfigs[var.aks_cluster_key].client_certificate, null) + client_key = try(local.k8sconfigs[var.aks_cluster_key].client_key, null) + cluster_ca_certificate = try(local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate, null) +} + +provider "helm" { + kubernetes { + host = local.k8sconfigs[var.aks_cluster_key].host + username = local.k8sconfigs[var.aks_cluster_key].username + password = local.k8sconfigs[var.aks_cluster_key].password + client_certificate = local.k8sconfigs[var.aks_cluster_key].client_certificate + client_key = local.k8sconfigs[var.aks_cluster_key].client_key + cluster_ca_certificate = local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate + } +} + +provider "kustomization" { + kubeconfig_raw = local.k8sconfigs[var.aks_cluster_key].kube_admin_config_raw +} + +locals { + aks_clusters = { + for key, value in var.aks_clusters : key => + local.remote.aks_clusters[value.lz_key][value.key] + } + k8sconfigs = { + for key, value in var.aks_clusters : key => { + kube_admin_config_raw = data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config_raw + host = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.host : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.host + username = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.username : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.username + password = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.password : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.password + client_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_certificate) + client_key = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_key) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_key) + cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.cluster_ca_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.cluster_ca_certificate) + } + } +} + +# Get kubeconfig from AKS clusters +data "azurerm_kubernetes_cluster" "kubeconfig" { + for_each = var.aks_clusters + + name = local.remote.aks_clusters[each.value.lz_key][each.value.key].cluster_name + resource_group_name = local.remote.aks_clusters[each.value.lz_key][each.value.key].resource_group_name +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/variables.tf b/caf_solution/add-ons/aks_applications/variables.tf index 7ad77ea93..a24be3ab1 100644 --- a/caf_solution/add-ons/aks_applications/variables.tf +++ b/caf_solution/add-ons/aks_applications/variables.tf @@ -3,34 +3,42 @@ variable "lower_storage_account_name" {} variable "lower_container_name" {} variable "lower_resource_group_name" {} +variable "tfstate_subscription_id" { + description = "This value is populated by the rover. subscription id hosting the remote tfstates" +} variable "tfstate_storage_account_name" {} variable "tfstate_container_name" {} +variable "tfstate_key" {} variable "tfstate_resource_group_name" {} -# variable tfstate_key {} variable "global_settings" { default = {} } -# variable tenant_id {} -variable "landingzone" {} - -variable "namespaces" {} -variable "tags" { +variable "landingzone" {} +variable "rover_version" { default = null - type = map(any) +} +variable "tags" { + default = {} +} +variable "namespaces" { + default = {} } -variable "helm_charts" {} +variable "helm_charts" { + default = {} +} +variable "aks_clusters" {} -variable "rover_version" { - default = null +variable "aks_cluster_key" { } -variable "cluster_re1_key" { - default = null +variable "kustomization_overlays" { + default = {} } -variable "cluster_re2_key" { - default = null + +variable "kustomization_builds" { + default = {} } \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/applications.tf b/caf_solution/add-ons/aks_azure_devops_agents/applications.tf new file mode 100644 index 000000000..49c6951c1 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/applications.tf @@ -0,0 +1,5 @@ +module "app" { + source = "../aks_applications/app" + namespaces = var.namespaces + helm_charts = var.helm_charts +} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/backend.azurerm b/caf_solution/add-ons/aks_azure_devops_agents/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf new file mode 100644 index 000000000..88033dae2 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf @@ -0,0 +1,245 @@ +data "kustomization_overlay" "manifest" { + for_each = var.kustomization_overlays + + resources = each.value.resources + + namespace = each.value.namespace + + dynamic "patches"{ + for_each = try(each.value.patches, {}) + content { + patch = patches.value.patch + target = patches.value.target + } + } + # kustomize_options = { + # load_restrictor = "none" + # } +} + +output "manifests" { + value = data.kustomization_overlay.manifest +} + + +# module "kustomization_azdopat-secret" { +# source = "./kustomize" + +# settings = data.kustomization_overlay.azdopat-secret + +# } + +# data "kustomization_overlay" "azdopat-secret" { +# resources = [ +# "yamls/azdopat-secret.yaml", +# ] + +# namespace = var.agent_pools.namespace + +# patches { +# patch = <<-EOF +# - op: replace +# path: /data/personalAccessToken +# value: "dDVjYmljc2R0Y3Juc2RlZmh1cnU2bHBueHdzZ2hxbjdhc2JnMjVkZ2E0bW16dGdldHgzYQ==" +# EOF +# target = { +# kind = "Secret" +# name = "azdopat-secret" +# } +# } +# } + +module "kustomization_azdopat-secret" { + source = "../aks_applications/kustomize" + + settings = data.kustomization_overlay.azdopat-secret + +} + +data "kustomization_overlay" "azdopat-secret" { + resources = [ + "yamls/akvs-secret.yaml", + ] + + namespace = var.agent_pools.namespace + + patches { + patch = <<-EOF + - op: replace + path: /spec/vault/name + value: "${local.remote.keyvaults[var.keyvault.lz_key][var.keyvault.key].name}" + EOF + target = { + kind = "AzureKeyVaultSecret" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/vault/object/name + value: "${var.keyvault.secret_name}" + EOF + target = { + kind = "AzureKeyVaultSecret" + } + } + +} + +module "kustomization" { + source = "../aks_applications/kustomize" + for_each = try(data.kustomization_overlay.roverjob, {}) + + settings = each.value + +} +data "kustomization_overlay" "roverjob" { + for_each = local.remote.agent_pools[var.agent_pools.lz_key] + + resources = [ + "yamls/roverjob.yaml", + ] + + namespace = var.agent_pools.namespace + + patches { + patch = <<-EOF + - op: replace + path: /metadata/name + value: "azdevops-${replace(each.key,"_","-")}" + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/metadata/labels/aadpodidbinding + value: ${each.value.name} + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/spec/containers/0/env/0/value + value: ${var.agent_pools.org_url} + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/spec/containers/0/env/2/value + value: ${each.value.name} + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/spec/containers/0/image + value: "${var.agent_pools.image}" + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/triggers/0/metadata/poolID + value: "${each.value.id}" + EOF + target = { + kind = "ScaledJob" + } + } +} + + +module "kustomization_placeholderagent" { + source = "../aks_applications/kustomize" + for_each = try(data.kustomization_overlay.placeholderjob, {}) + + settings = each.value + +} + +data "kustomization_overlay" "placeholderjob" { + for_each = local.remote.agent_pools[var.agent_pools.lz_key] + + resources = [ + "yamls/placeholderjob.yaml", + ] + + namespace = var.agent_pools.namespace + + patches { + patch = <<-EOF + - op: replace + path: /metadata/name + value: "placeholder-job-${replace(each.key,"_","-")}" + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/spec/containers/0/env/0/value + value: ${var.agent_pools.org_url} + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/metadata/labels/aadpodidbinding + value: ${each.value.name} + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/spec/containers/0/env/2/value + value: ${each.value.name} + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/spec/containers/0/image + value: "${var.agent_pools.image}" + EOF + target = { + kind = "Job" + } + } +} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/locals.remote_tfstates.tf b/caf_solution/add-ons/aks_azure_devops_agents/locals.remote_tfstates.tf new file mode 100644 index 000000000..46b41e90a --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/locals.remote_tfstates.tf @@ -0,0 +1,63 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name + container_name = try(each.value.container, local.landingzone[try(each.value.level, "current")].container_name) + resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id + key = each.value.tfstate + } +} + +locals { + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + + remote = { + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + global_settings = local.global_settings + diagnostics = local.diagnostics + + + aks_clusters = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) + } + keyvaults = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) + } + azure_devops = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.azure_devops, {})) + } + agent_pools = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs[var.agent_pools.key], {})) + } + } + +} +output "keyvaults" { + value = local.remote.keyvaults +} +output "azure_devops" { + value = local.remote.azure_devops +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/main.tf b/caf_solution/add-ons/aks_azure_devops_agents/main.tf new file mode 100644 index 000000000..6b7201fb4 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/main.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.55.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0.2" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.1.2" + } + kustomization = { + source = "kbst/kustomization" + version = "~> 0.5.0" + } + } + required_version = ">= 0.13" +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/output.tf b/caf_solution/add-ons/aks_azure_devops_agents/output.tf new file mode 100644 index 000000000..e69de29bb diff --git a/caf_solution/add-ons/aks_azure_devops_agents/providers.tf b/caf_solution/add-ons/aks_azure_devops_agents/providers.tf new file mode 100644 index 000000000..e8b9dcfc5 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/providers.tf @@ -0,0 +1,54 @@ +provider "azurerm" { + features { + } +} + +provider "kubernetes" { + host = try(local.k8sconfigs[var.aks_cluster_key].host, null) + username = try(local.k8sconfigs[var.aks_cluster_key].username, null) + password = try(local.k8sconfigs[var.aks_cluster_key].password, null) + client_certificate = try(local.k8sconfigs[var.aks_cluster_key].client_certificate, null) + client_key = try(local.k8sconfigs[var.aks_cluster_key].client_key, null) + cluster_ca_certificate = try(local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate, null) +} + +provider "helm" { + kubernetes { + host = local.k8sconfigs[var.aks_cluster_key].host + username = local.k8sconfigs[var.aks_cluster_key].username + password = local.k8sconfigs[var.aks_cluster_key].password + client_certificate = local.k8sconfigs[var.aks_cluster_key].client_certificate + client_key = local.k8sconfigs[var.aks_cluster_key].client_key + cluster_ca_certificate = local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate + } +} + +provider "kustomization" { + kubeconfig_raw = local.k8sconfigs[var.aks_cluster_key].kube_admin_config_raw +} + +locals { + aks_clusters = { + for key, value in var.aks_clusters : key => + local.remote.aks_clusters[value.lz_key][value.key] + } + k8sconfigs = { + for key, value in var.aks_clusters : key => { + kube_admin_config_raw = data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config_raw + host = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.host : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.host + username = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.username : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.username + password = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.password : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.password + client_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_certificate) + client_key = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_key) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_key) + cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.cluster_ca_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.cluster_ca_certificate) + } + } +} + +# Get kubeconfig from AKS clusters +data "azurerm_kubernetes_cluster" "kubeconfig" { + for_each = var.aks_clusters + + name = local.remote.aks_clusters[each.value.lz_key][each.value.key].cluster_name + resource_group_name = local.remote.aks_clusters[each.value.lz_key][each.value.key].resource_group_name +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/variables.tf b/caf_solution/add-ons/aks_azure_devops_agents/variables.tf new file mode 100644 index 000000000..2789ae8f4 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/variables.tf @@ -0,0 +1,52 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is populated by the rover. subscription id hosting the remote tfstates" +} +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "global_settings" { + default = {} +} + + +variable "landingzone" {} +variable "rover_version" { + default = null +} +variable "tags" { + default = {} +} +variable "namespaces" { + default = {} +} + +variable "helm_charts" { + default = {} +} +variable "aks_clusters" {} + +variable "aks_cluster_key" { +} + +variable "kustomization_overlays" { + default = {} +} + +variable "kustomization_builds" { + default = {} +} + +variable "agent_pools" { + +} + +variable "keyvault" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/akvs-secret.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/akvs-secret.yaml new file mode 100644 index 000000000..83c9cefa5 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/akvs-secret.yaml @@ -0,0 +1,14 @@ +apiVersion: spv.no/v2beta1 +kind: AzureKeyVaultSecret +metadata: + name: pat-secret-sync +spec: + vault: + name: afops-kv-afopssre-djo # name of key vault + object: + name: azdo-pat-admin # name of the akv object + type: secret # akv object type + output: + secret: + name: pat-secret-sync # kubernetes secret name + dataKey: personalAccessToken # key to store object value in kubernetes secret \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/azdopat-secret.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/azdopat-secret.yaml new file mode 100644 index 000000000..c3bd660ef --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/azdopat-secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: azdopat-secret +data: + personalAccessToken: ${pat} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderagent.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderagent.yaml new file mode 100644 index 000000000..b9c5d8ae4 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderagent.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rover-deployment + labels: + aadpodidbinding: podmi-caf-rover-platform-level0 + app: rover-agent +spec: + replicas: 1 + selector: + matchLabels: + app: rover-agent + template: + metadata: + labels: + aadpodidbinding: podmi-caf-rover-platform-level0 + app: rover-agent + spec: + containers: + - name: rover-agent + image: aztfmod/rover-agent:0.15.1-2105.041640-preview-azdo + env: + - name: VSTS_AGENT_INPUT_URL + value: "https://dev.azure.com/afopssre" + - name: VSTS_AGENT_INPUT_TOKEN + valueFrom: + secretKeyRef: + name: azdopat-secret + key: personalAccessToken + - name: VSTS_AGENT_INPUT_POOL + value: "aks-agents" + - name: VSTS_AGENT_INPUT_AUTH + value: "pat" + - name: VSTS_AGENT_INPUT_RUN_ONCE + value: "--once" + volumeMounts: + - name: secrets-store01-inline + mountPath: "/mnt/secrets-store" + readOnly: true + lifecycle: + preStop: + exec: + # SIGTERM triggers a quick exit; gracefully terminate instead + command: ["/home/vscode/agent/config.sh","remove","--unattended"] + volumes: + - name: secrets-store01-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "azure-sync" + + # terminationGracePeriodSeconds: 60 \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderjob.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderjob.yaml new file mode 100644 index 000000000..2c9df8b05 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderjob.yaml @@ -0,0 +1,38 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: placeholder-agent + labels: + app: placeholder-agent +spec: + activeDeadlineSeconds: 180 + template: + metadata: + labels: + aadpodidbinding: ${podmi} + app: placeholder-agent + spec: + containers: + - name: azdevops-agent-job + image: ${image} + imagePullPolicy: Always + env: + - name: VSTS_AGENT_INPUT_URL + value: ${VSTS_AGENT_INPUT_URL} + - name: VSTS_AGENT_INPUT_TOKEN + valueFrom: + secretKeyRef: + name: pat-secret-sync + key: personalAccessToken + - name: VSTS_AGENT_INPUT_POOL + value: ${VSTS_AGENT_INPUT_POOL} + - name: VSTS_AGENT_INPUT_AUTH + value: "pat" + - name: VSTS_AGENT_INPUT_RUN_ARGS + value: "--once" + # lifecycle: + # preStop: + # exec: + # # SIGTERM triggers a quick exit; gracefully terminate instead + # command: ["/home/vscode/agent/config.sh","remove","--unattended"] + restartPolicy: Never \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/roverjob.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/roverjob.yaml new file mode 100644 index 000000000..83de01aef --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/roverjob.yaml @@ -0,0 +1,48 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: ${azdevops-scaledjob} +spec: + jobTargetRef: + ttlSecondsAfterFinished: 100 + template: + metadata: + labels: + aadpodidbinding: podmi-caf-rover-platform-level0 + app: rover-agent + spec: + containers: + - name: azdevops-agent-job + image: ${image} + imagePullPolicy: Always + env: + - name: VSTS_AGENT_INPUT_URL + value: ${VSTS_AGENT_INPUT_URL} + - name: VSTS_AGENT_INPUT_TOKEN + valueFrom: + secretKeyRef: + name: pat-secret-sync + key: personalAccessToken + - name: VSTS_AGENT_INPUT_POOL + value: ${VSTS_AGENT_INPUT_POOL} + - name: VSTS_AGENT_INPUT_AUTH + value: "pat" + - name: VSTS_AGENT_INPUT_RUN_ARGS + value: "--once" + # lifecycle: + # preStop: + # exec: + # # SIGTERM triggers a quick exit; gracefully terminate instead + # command: ["/home/vscode/agent/config.sh","remove","--unattended"] + pollingInterval: 30 + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 + maxReplicaCount: 10 + scalingStrategy: + strategy: "default" + triggers: + - type: azure-pipelines + metadata: + poolID: ${poolID} + organizationURLFromEnv: "VSTS_AGENT_INPUT_URL" + personalAccessTokenFromEnv: "VSTS_AGENT_INPUT_TOKEN" diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/secretstorecsi.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/secretstorecsi.yaml new file mode 100644 index 000000000..cb084bc0a --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/secretstorecsi.yaml @@ -0,0 +1,22 @@ +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: azdo-pat-sync +spec: + provider: azure + secretObjects: # secretObjects defines the desired state of synced K8s secret objects + - secretName: azdo-pat-secret + type: Opaque + data: + - objectName: azdo-pat-secret + key: personalAccessToken + parameters: + usePodIdentity: "true" + keyvaultName: afops-kv-afopssre-djo + objects: | + array: + - | + objectName: azdo-pat-admin + objectAlias: azdo-pat-secret + objectType: secret + tenantId: gitopsprod.onmicrosoft.com \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf new file mode 100644 index 000000000..eab82971f --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf @@ -0,0 +1,67 @@ + +# Get the details of the node pool's resource group created by AKS +data "azurerm_resource_group" "noderg" { + for_each = var.aks_clusters + name = local.remote.aks_clusters[each.value.lz_key][each.value.key].node_resource_group +} + +# +# Set permissions to the kubelet and cluster identity +# +resource "azurerm_role_assignment" "kubelet_noderg_miop" { + for_each = var.aks_clusters + + scope = data.azurerm_resource_group.noderg[each.key].id + role_definition_name = "Managed Identity Operator" + principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +} + +resource "azurerm_role_assignment" "kubelet_noderg_vmcontrib" { + for_each = var.aks_clusters + + scope = data.azurerm_resource_group.noderg[each.key].id + role_definition_name = "Virtual Machine Contributor" + principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +} + +# Separate subnet +resource "azurerm_role_assignment" "kubelet_subnets_networkcontrib" { + for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",{vnet=true}) + + scope = try(each.value==true, false) ? local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id : local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].subnets[each.value].id + role_definition_name = "Network Contributor" + principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id +} + +# # Whole vnet +# resource "azurerm_role_assignment" "kubelet_vnet_networkcontrib" { +# for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",null) == null ? var.vnets : {} + +# scope = local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id +# role_definition_name = "Network Contributor" +# principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id +# } + +resource "azurerm_role_assignment" "kubelet_user_msi" { + for_each = local.msi_to_grant_permissions + + scope = each.value.id + role_definition_name = "Managed Identity Operator" + principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].kubelet_identity[0].object_id +} + +locals { + msi_to_grant_permissions = { + for msi in flatten( + [ + for key, value in var.managed_identities : [ + for msi_key in value.msi_keys : { + key = key + msi_key = msi_key + id = local.remote.managed_identities[value.lz_key][msi_key].id + } + ] + ] + ) : format("%s-%s", msi.key, msi.msi_key) => msi + } +} diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/backend.azurerm b/caf_solution/add-ons/aks_secure_baseline_v2/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf new file mode 100644 index 000000000..102b9c5d2 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf @@ -0,0 +1,72 @@ +data "flux_install" "main" { + target_path = var.setting.target_path + namespace = var.setting.namespace +} + +data "flux_sync" "main" { + target_path = var.setting.target_path + url = var.setting.url + branch = var.setting.branch + secret = try(var.setting.flux_auth_secret, null) + namespace = var.setting.namespace +} + +data "kubectl_file_documents" "install" { + content = data.flux_install.main.content +} + +data "kubectl_file_documents" "sync" { + content = data.flux_sync.main.content +} + +resource "kubernetes_namespace" "flux_system" { + metadata { + name = var.setting.namespace + } + + lifecycle { + ignore_changes = [ + metadata[0].labels, metadata[0].annotations + ] + } +} + +locals { + flux_install_yaml_documents_without_namespace = [for x in data.kubectl_file_documents.install.documents: x if length(regexall("kind: Namespace", x)) == 0] + install = [for v in local.flux_install_yaml_documents_without_namespace : { + data : yamldecode(v) + content : v + } + ] + sync = [for v in data.kubectl_file_documents.sync.documents : { + data : yamldecode(v) + content : v + } + ] +} + +resource "kubectl_manifest" "install" { + for_each = { for v in local.install : lower(join("/", compact([v.data.apiVersion, v.data.kind, lookup(v.data.metadata, "namespace", ""), v.data.metadata.name]))) => v.content } + depends_on = [kubernetes_namespace.flux_system] + yaml_body = each.value +} + +resource "kubectl_manifest" "sync" { + for_each = { for v in local.sync : lower(join("/", compact([v.data.apiVersion, v.data.kind, lookup(v.data.metadata, "namespace", ""), v.data.metadata.name]))) => v.content } + depends_on = [kubernetes_namespace.flux_system, kubectl_manifest.install] + yaml_body = each.value +} + + + +resource "kubernetes_secret" "fluxauth" { + depends_on = [kubectl_manifest.install] + + metadata { + name = try(var.setting.flux_auth_secret, "flux-system") + namespace = var.setting.namespace + } + data = try(var.setting.secret.data, null) + type = try(var.setting.secret.type, null) +} + diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/main.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/main.tf new file mode 100644 index 000000000..b19246abc --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + } + kubectl = { + source = "gavinbunney/kubectl" + } + flux = { + source = "fluxcd/flux" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/variables.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/variables.tf new file mode 100644 index 000000000..53a87763f --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/variables.tf @@ -0,0 +1,3 @@ +variable "setting" { + description = "Flux settings" +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/locals.remote_tfstates.tf b/caf_solution/add-ons/aks_secure_baseline_v2/locals.remote_tfstates.tf new file mode 100644 index 000000000..f8312ab8c --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/locals.remote_tfstates.tf @@ -0,0 +1,53 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name + container_name = try(each.value.container, local.landingzone[try(each.value.level, "current")].container_name) + resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id + key = each.value.tfstate + } +} + +locals { + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + + remote = { + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + global_settings = local.global_settings + diagnostics = local.diagnostics + + + aks_clusters = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) + } + managed_identities = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) + } + vnets = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/main.tf b/caf_solution/add-ons/aks_secure_baseline_v2/main.tf new file mode 100644 index 000000000..a409122ce --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/main.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.0.2" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.11.1" + } + flux = { + source = "fluxcd/flux" + version = ">= 0.0.13" + } + } + required_version = ">= 0.13" +} + + + diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/module.tf b/caf_solution/add-ons/aks_secure_baseline_v2/module.tf new file mode 100644 index 000000000..2f77e8315 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/module.tf @@ -0,0 +1,5 @@ +module "flux" { + source = "./flux" + for_each = var.flux_settings + setting = each.value +} diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/output.tf b/caf_solution/add-ons/aks_secure_baseline_v2/output.tf new file mode 100644 index 000000000..7a40f2a75 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/output.tf @@ -0,0 +1,7 @@ +output "aks_clusters_kubeconfig" { + value = { + aks_kubeconfig_admin_cmd = local.remote.aks_clusters.aks.cluster_re1.aks_kubeconfig_admin_cmd + aks_kubeconfig_cmd = local.remote.aks_clusters.aks.cluster_re1.aks_kubeconfig_cmd + } + sensitive = false +} diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/providers.tf b/caf_solution/add-ons/aks_secure_baseline_v2/providers.tf new file mode 100644 index 000000000..aafa4dd61 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/providers.tf @@ -0,0 +1,29 @@ +provider "azurerm" { + features { + } +} + +provider "kubectl" { + host = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.host, null) + username = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.username, null) + password = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.password, null) + client_key = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_key), null) + client_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_certificate), null) + cluster_ca_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.cluster_ca_certificate), null) + load_config_file = false +} + +provider "kubernetes" { + host = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.host, null) + username = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.username, null) + password = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.password, null) + client_key = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_key), null) + client_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_certificate), null) + cluster_ca_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.cluster_ca_certificate), null) +} + +# Get kubeconfig from AKS clusters +data "azurerm_kubernetes_cluster" "kubeconfig" { + name = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_clusters[var.aks_cluster_key].key].cluster_name + resource_group_name = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_clusters[var.aks_cluster_key].key].resource_group_name +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/variables.tf b/caf_solution/add-ons/aks_secure_baseline_v2/variables.tf new file mode 100644 index 000000000..0b195b73b --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/variables.tf @@ -0,0 +1,44 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is populated by the rover. subscription id hosting the remote tfstates" +} +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "global_settings" { + default = {} +} + + +variable "landingzone" {} +variable "rover_version" { + default = null +} +variable "tags" { + default = {} +} +variable "namespaces" { + default = {} +} + +variable "helm_charts" { + default = {} +} +variable "aks_clusters" {} + +variable "aks_cluster_key" {} + +variable "aks_cluster_vnet_key" {} + +variable "flux_settings" {} + +variable "vnets" {} +variable "managed_identities" { + description = "Map of the user managed identities." +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/agent_pools/agent_pool.tf b/caf_solution/add-ons/azure_devops_v1/agent_pools/agent_pool.tf new file mode 100644 index 000000000..510929311 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/agent_pools/agent_pool.tf @@ -0,0 +1,36 @@ +# Get Agent pools by name +data "azuredevops_agent_pool" "pool" { + for_each = { + for key, value in var.settings : key => value + if try(value.name, null) != null + } + + name = each.value.name +} + +resource "azuredevops_agent_queue" "agent_queue" { + for_each = var.settings + + project_id = var.project_id + agent_pool_id = try(data.azuredevops_agent_pool.pool[each.key].id, var.azuredevops_agent_pools[each.value.key].id) +} + +# +# Grant acccess to queue to all pipelines in the project +# + +resource "azuredevops_resource_authorization" "queue" { + for_each = var.settings + + project_id = var.project_id + resource_id = azuredevops_agent_queue.agent_queue[each.key].id + type = "queue" + authorized = try(each.value.grant_access, false) + + +} + + +output "azuredevops_resource_authorization" { + value = azuredevops_resource_authorization.queue +} diff --git a/caf_solution/add-ons/azure_devops_v1/agent_pools/main.tf b/caf_solution/add-ons/azure_devops_v1/agent_pools/main.tf new file mode 100644 index 000000000..a533fb035 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/agent_pools/main.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + azuredevops = { + source = "microsoft/azuredevops" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/agent_pools/variables.tf b/caf_solution/add-ons/azure_devops_v1/agent_pools/variables.tf new file mode 100644 index 000000000..d92c82c8b --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/agent_pools/variables.tf @@ -0,0 +1,9 @@ +variable "azuredevops_agent_pools" { + description = "Agent pools created at the organization level." +} + +variable "project_id" { + description = "Azure devops project id." +} + +variable "settings" {} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/azdo.tf b/caf_solution/add-ons/azure_devops_v1/azdo.tf new file mode 100644 index 000000000..90094bccf --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo.tf @@ -0,0 +1,17 @@ +# The PAT token must be provisioned in a different deployment +provider "azuredevops" { + org_service_url = var.azure_devops.url + personal_access_token = data.external.pat.result.value +} + +# To support cross subscrpition reference +data "external" "pat" { + program = [ + "bash", "-c", + format( + "az keyvault secret show --id '%s'secrets/'%s' --query '{value: value}' -o json", + local.remote.keyvaults[var.azure_devops.pats["admin"].lz_key][var.azure_devops.pats["admin"].keyvault_key].vault_uri, + var.azure_devops.pats["admin"].secret_name + ) + ] +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf b/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf new file mode 100644 index 000000000..2d5b99a13 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf @@ -0,0 +1,44 @@ + +locals { + organization_agent_pools = try(var.organization_agent_pools, {}) + project_agent_pools = try(var.project_agent_pools, {}) +} + + + +## Agent pools +## Those pools are created in the organization, not the project +resource "azuredevops_agent_pool" "pool" { + for_each = var.organization_agent_pools + + name = each.value.name + auto_provision = try(each.value.auto_provision, false) + pool_type = try(each.value.pool_type, null) +} + +output agent_pools { + value = azuredevops_agent_pool.pool +} + + +# +# add the agent pools into the project +# + + +module "project_agent_pools" { + source = "./agent_pools" + for_each = var.project_agent_pools + + azuredevops_agent_pools = azuredevops_agent_pool.pool + project_id = data.azuredevops_project.project[each.key].id + settings = each.value + + depends_on = [ + azuredevops_agent_pool.pool, + azuredevops_project.project + ] +} + + + diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf new file mode 100644 index 000000000..9d7e2ed70 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf @@ -0,0 +1,47 @@ +data "azuredevops_git_repositories" "repos" { + for_each = var.projects + + project_id = data.azuredevops_project.project[each.key].id +} + +resource "azuredevops_build_definition" "build_definition" { + for_each = var.pipelines + + project_id = data.azuredevops_project.project[each.value.project_key].id + name = each.value.name + path = each.value.folder + + variable_groups = lookup(each.value, "variable_group_keys", null) == null ? null : [ + for key in each.value.variable_group_keys : + azuredevops_variable_group.variable_group[key].id + ] + + dynamic "repository" { + for_each = { + for key, value in try(data.azuredevops_git_repositories.repos[try(each.value.repo_project_key, each.value.project_key)].repositories, {}) : key => value + if value.name == each.value.git_repo_name + } + + content { + repo_id = repository.value.id + repo_type = each.value.repo_type + yml_path = each.value.yaml + branch_name = lookup(each.value, "branch_name", null) + # service_connection_id = lookup(each.value, "repo_type", null) == "github" ? null : azuredevops_serviceendpoint_azurerm.github[each.value.service_connection_key].id + } + } + + ci_trigger { + use_yaml = true + } + + dynamic "variable" { + for_each = try(each.value.variables, {}) + + content { + name = variable.key + value = variable.value + } + } + +} diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf new file mode 100644 index 000000000..1991a5d6d --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf @@ -0,0 +1,40 @@ + +# To support cross subscription +data "external" "client_secret" { + for_each = var.service_endpoints + program = [ + "bash", "-c", + format( + "az keyvault secret show --id '%s'secrets/'%s' --query '{value: value}' -o json", + local.remote.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.key].vault_uri, + each.value.keyvault.secret_name + ) + ] +} + +resource "azuredevops_serviceendpoint_azurerm" "azure" { + for_each = var.service_endpoints + + project_id = data.azuredevops_project.project[each.value.project_key].id + service_endpoint_name = each.value.endpoint_name + credentials { + serviceprincipalid = local.remote.azuread_applications[each.value.azuread_application.lz_key][each.value.azuread_application.key].application_id + serviceprincipalkey = data.external.client_secret[each.key].result.value + } + azurerm_spn_tenantid = local.remote.azuread_applications[each.value.azuread_application.lz_key][each.value.azuread_application.key].tenant_id + azurerm_subscription_id = each.value.subscription.id + azurerm_subscription_name = each.value.subscription.name +} + +# +# Grant acccess to service endpoint to all pipelines in the project +# + +resource "azuredevops_resource_authorization" "endpoint" { + for_each = var.service_endpoints + + project_id = data.azuredevops_project.project[each.value.project_key].id + resource_id = azuredevops_serviceendpoint_azurerm.azure[each.key].id + type = "endpoint" + authorized = true +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf new file mode 100644 index 000000000..2a8099203 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf @@ -0,0 +1,52 @@ +# # +# # permissions required: +# # - vso.variablegroups_manage (create) +# # + vso.buid (update) +# # + vso.build_execute (destroy) +# # +resource "azuredevops_variable_group" "variable_group" { + for_each = var.variable_groups + + project_id = data.azuredevops_project.project[each.value.project_key].id + name = each.value.name + description = try(each.value.description, null) + allow_access = try(each.value.allow_access, false) + + dynamic "key_vault" { + for_each = lookup(each.value, "keyvault", null) == null ? [] : [1] + + content { + name = local.remote.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.keyvault_key].name + service_endpoint_id = azuredevops_serviceendpoint_azurerm.azure[each.value.keyvault.serviceendpoint_key].id + } + } + + dynamic "variable" { + for_each = { + for key, variable in each.value.variables : key => { + name = key == "name" ? variable : key + value = key == "name" ? null : variable + } + if try(each.value.remote_objects, false) == false + } + + content { + # When used with Keyvault, the name must be the keyvault secret name and value must not be set + name = variable.value.name + value = variable.value.value + } + } + + dynamic "variable" { + for_each = { + for key, value in each.value.variables : key => value + if try(each.value.remote_objects, false) == true + } + + content { + name = variable.value.name + value = local.remote[variable.value.output_key][variable.value.lz_key][variable.value.resource_key][variable.value.attribute_key] + } + } + +} diff --git a/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf b/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf new file mode 100644 index 000000000..ebe0afc6c --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf @@ -0,0 +1,38 @@ +resource "azuredevops_project" "project" { + for_each = { + for key, value in var.projects : key => value + if try(value.create, false) + } + + name = each.value.name + description = each.value.description + visibility = try(lower(each.value.visibility), null) + version_control = try(each.value.version_control, null) + work_item_template = try(each.value.work_item_template, null) +} + + +data "azuredevops_project" "project" { + depends_on = [azuredevops_project.project] + + for_each = var.projects + + name = each.value.name +} + +resource "azuredevops_project_features" "project" { + for_each = { + for key, value in var.projects : key => value + if try(value.features, null) != null + } + + project_id = data.azuredevops_project.project[each.key].id + + features = { + "artifacts" = try(lower(each.value.features.artifacts), "disabled") + "boards" = try(lower(each.value.features.boards), "disabled") + "pipelines" = try(lower(each.value.features.pipelines), "disabled") + "repositories" = try(lower(each.value.features.repositories), "disabled") + "testplans" = try(lower(each.value.features.testplans), "disabled") + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/backend.azurerm b/caf_solution/add-ons/azure_devops_v1/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/documentation/images/pat_token.png b/caf_solution/add-ons/azure_devops_v1/documentation/images/pat_token.png new file mode 100644 index 000000000..84fc549c5 Binary files /dev/null and b/caf_solution/add-ons/azure_devops_v1/documentation/images/pat_token.png differ diff --git a/caf_solution/add-ons/azure_devops_v1/local.azuread.tf b/caf_solution/add-ons/azure_devops_v1/local.azuread.tf new file mode 100644 index 000000000..dca335922 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/local.azuread.tf @@ -0,0 +1,10 @@ +locals { + azuread = merge( + var.azuread, + { + azuread_applications = var.azuread_applications + azuread_service_principals = var.azuread_service_principals + azuread_service_principal_passwords = var.azuread_service_principal_passwords + } + ) +} diff --git a/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf new file mode 100644 index 000000000..14904c3bc --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf @@ -0,0 +1,70 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] +} + +locals { + + remote_state = { + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = try(value.workspace, local.landingzone[try(value.level, "current")].container_name) + key = value.tfstate + resource_group_name = try(value.resource_group_name, local.landingzone[try(value.level, "current")].resource_group_name) + storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) + subscription_id = try(value.subscription_id, var.tfstate_subscription_id) + tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + } + } + } + + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + + remote = { + aad_apps = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aad_apps, {})) + } + azuread_applications = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) + } + azuread_groups = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) + } + keyvaults = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) + } + managed_identities = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) + } + vnets = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) + } + subscriptions = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].subscriptions, {})) + } + } +} diff --git a/caf_solution/add-ons/azure_devops_v1/main.tf b/caf_solution/add-ons/azure_devops_v1/main.tf new file mode 100644 index 000000000..5ee46dde9 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/main.tf @@ -0,0 +1,56 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.56.0" + } + azuread = { + source = "hashicorp/azuread" + version = "~> 1.4.0" + } + azuredevops = { + source = "microsoft/azuredevops" + version = "~> 0.1.3" + } + } + required_version = ">= 0.13" +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = true + } + } +} + +data "azurerm_client_config" "current" {} + + +locals { + + # Update the tfstates map + tfstates = merge( + tomap( + { + (var.landingzone.key) = local.backend[var.landingzone.backend_type] + } + ) + , + data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates + ) + + + backend = { + azurerm = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + key = var.tfstate_key + level = var.landingzone.level + tenant_id = var.tenant_id + subscription_id = data.azurerm_client_config.current.subscription_id + } + } + +} diff --git a/caf_solution/add-ons/azure_devops_v1/output.tf b/caf_solution/add-ons/azure_devops_v1/output.tf new file mode 100644 index 000000000..51c08bdf0 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/output.tf @@ -0,0 +1,12 @@ +# output "keyvaults" { +# value = tomap( +# { +# (var.landingzone.key) = module.caf.keyvaults +# } +# ) +# sensitive = true +# } + +output "azure_devops" { + value = var.azure_devops +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/readme.md b/caf_solution/add-ons/azure_devops_v1/readme.md new file mode 100644 index 000000000..1ebbb675f --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/readme.md @@ -0,0 +1,60 @@ +# Cloud Adoption Framework for Azure - Landing zones on Terraform - Azure Devops add-on + +The Azure Devops add-ons allow you to setup you Azure Devops environment as a platform to automate all your subsequent landing zone deployment from level 0 until level 4 through Azure pipelines with self hosted agents. + +* Azure Devops: + - Agent Pools (Organization and Project Level) + - Service Endpoint + - Variables and Variable Groups + - Pipelines + +* Azure (Connection with Azure Devops): + - Azure AD Application + - Custom Role + - Keyvault and access policies for Azure AD App + +Azure Devops add-on landing zone operates at **level 0** + +For a review of the hierarchy approach of Cloud Adoption Framework for Azure landing zones on Terraform, you can refer to [the following documentation](../../documentation/code_architecture/hierarchy.md). + +## Dependencies + +Landing zone: +* CAF Launchpad (Scenario 200 or above) + +Azure Devops (example): +* Organization: https://dev.azure.com/azure-terraform +* Project : contoso_demo (https://dev.azure.com/azure-terraform/contoso_demo) +* Repo : caf-configuration (https://dev.azure.com/azure-terraform/contoso_demo/_git/caf-configuration) + - In order for pipeline to work properly, YAML file should be in this repo and referred accordingly under pipeline section in azure_devops.tfvars + - sample yaml attached [here](./scenario/200-contoso_demo/pipeline/rover.yaml). + +Azure: +* PAT Token : PAT Token should be updated in keyvault secret that deployed by launchpad LZ as below + +![](./documentation/images/pat_token.png) + +## Deployment + +```bash +rover -lz /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops \ + -tfstate azure_devops-contoso_demo.tfstate \ + -var-folder /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops/scenario/200-contoso_demo \ + -parallelism 30 \ + -level level0 \ + -env sandpit \ + -a apply + + +# If the tfstates are stored in a different subscription you need to execute the following command +rover -lz /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops \ + -tfstate_subscription_id \ + -tfstate azure_devops-contoso_demo.tfstate \ + -var-folder /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops/scenario/200-contoso_demo \ + -parallelism 30 \ + -level level0 \ + -env sandpit \ + -a apply +``` + +We are planning to release more examples on how to deploy the Azure Devops Agents. diff --git a/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/azure_devops.tfvars b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/azure_devops.tfvars new file mode 100644 index 000000000..bf0bd6c1f --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/azure_devops.tfvars @@ -0,0 +1,232 @@ + + +azure_devops = { + + url = "https://dev.azure.com/azure-terraform/" + project = "contoso_demo" + + # PAT Token should be updated manually to the keyvault after running launchpad + pats = { + admin = { + secret_name = "azdo-pat-admin" + lz_key = "launchpad" + keyvault_key = "secrets" + } + } + + organization_agent_pools = { + level0 = { + name = "caf-sandpit-level0" + auto_provision = false # When set to false the agent pool is not populated automatically into the devops projects (recommended) + } + level1 = { + name = "caf-sandpit-level1" + auto_provision = false + } + level2 = { + name = "caf-sandpit-level2" + auto_provision = false + } + level3 = { + name = "caf-sandpit-level3" + auto_provision = false + } + level4 = { + name = "caf-sandpit-level4" + auto_provision = false + } + } + + project_agent_pools = { + level0 = { + name = "caf-sandpit-level0" + } + level1 = { + name = "caf-sandpit-level1" + } + level2 = { + name = "caf-sandpit-level2" + } + level3 = { + name = "caf-sandpit-level3" + } + level4 = { + name = "caf-sandpit-level4" + } + } + + service_endpoints = { + contoso_demo = { + endpoint_name = "terraformdev (terraformdev.onmicrosoft.com) - contoso_demo" + subscription_name = "ase-landingzone" + subscription_id = "1d53e782-9f46-4720-b6b3-cff29106e9f6" + aad_app_key = "contoso_demo" + secret_keyvault_key = "devops" + } + } + + variable_groups = { + global = { + name = "release-global" # changing that name requires to change it in the devops agents yaml variables group + allow_access = true + variables = { + HOME_FOLDER_USER = "vsts_azpcontainer" + ROVER_IMAGE = "aztfmod/rover:2010.2808" + TF_CLI_ARGS = "'-no-color'" + TF_CLI_ARGS_init = "" + TF_CLI_ARGS_plan = "'-input=false'" + TF_VAR_ARGS_destroy = "'-auto-approve -refresh=false'" + ENVIRONMENT = "sandpit" + LANDINGZONE_BRANCH = "2010.0.0" + } + } + + level0 = { + name = "release-level0" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level0" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level0" + } + } + + level0_kv = { + name = "release-level0-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level0" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + + level1 = { + name = "release-level1" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level1" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level1" + } + } + + level1_kv = { + name = "release-level1-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level1" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + + level2 = { + name = "release-level2" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level2" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level2" + } + } + + level2_kv = { + name = "release-level2-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level2" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + + level3 = { + name = "release-level3" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level3" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level3" + } + } + + level3_kv = { + name = "release-level3-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level3" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + } + + pipelines = { + + # + # Agent pools + # + + devops_agent_level1_plan = { + name = "devops_agent_level1_plan" + folder = "\\configuration\\level1" + yaml = "configuration/pipeline/rover.yaml" + repo_type = "TfsGit" + git_repo_name = "caf-configuration" + variables = { + landingZoneName = "azdo-agent-level1", + terraformAction = "plan", + tfstateName = "azdo-agent-level1.tfstate" + configPath = "/configuration/level1/azuredevops/agent" + landingZonePath = "/public/landingzones/caf_launchpad/add-ons/azure_devops_agent" + level = "level1" + } + variable_group_keys = ["global", "level0", "level0_kv"] + } + devops_agent_level1_apply = { + name = "devops_agent_level1_apply" + folder = "\\configuration\\level1" + yaml = "configuration/pipeline/rover.yaml" + repo_type = "TfsGit" + git_repo_name = "caf-configuration" + variables = { + landingZoneName = "azdo-agent-level1", + terraformAction = "apply", + tfstateName = "azdo-agent-level1.tfstate" + configPath = "/configuration/level1/azuredevops/agent" + landingZonePath = "/public/landingzones/caf_launchpad/add-ons/azure_devops_agent" + level = "level1" + } + variable_group_keys = ["global", "level0", "level0_kv"] + } + devops_agent_level1_destroy = { + name = "devops_agent_level1_destroy" + folder = "\\configuration\\level1" + yaml = "configuration/pipeline/rover.yaml" + repo_type = "TfsGit" + git_repo_name = "caf-configuration" + variables = { + landingZoneName = "azdo-agent-level1", + terraformAction = "destroy", + tfstateName = "azdo-agent-level1.tfstate" + configPath = "/configuration/level1/azuredevops/agent" + landingZonePath = "/public/landingzones/caf_launchpad/add-ons/azure_devops_agent" + level = "level1" + } + variable_group_keys = ["global", "level0", "level0_kv"] + } + } +} diff --git a/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/configurations.tfvars b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/configurations.tfvars new file mode 100644 index 000000000..e4d8208f3 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/configurations.tfvars @@ -0,0 +1,123 @@ +landingzone = { + backend_type = "azurerm" + global_settings_key = "launchpad" + level = "level0" + key = "azdo-contoso_demo" + tfstates = { + launchpad = { + level = "current" + tfstate = "caf_launchpad.tfstate" + } + } +} + +resource_groups = { + rg1 = { + name = "devops-agents-security" + } +} + + +keyvaults = { + devops = { + name = "devops" + resource_group_key = "rg1" + sku_name = "standard" + + creation_policies = { + keyvault_level0_rw = { + # Reference a key to an azure ad group + lz_key = "launchpad" + azuread_group_key = "keyvault_level0_rw" + secret_permissions = ["Set", "Get", "List", "Delete", "Purge", "Recover"] + } + } + } +} + +keyvault_access_policies_azuread_apps = { + level0 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level1 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level2 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level3 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level4 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } +} + + +azuread_apps = { + + contoso_demo = { + useprefix = true + application_name = "caf-level4-contoso_demo" + password_expire_in_days = 60 + tenant_name = "terraformdev.onmicrosoft.com" + reply_urls = ["https://localhost"] + keyvaults = { + devops = { + secret_prefix = "aadapp-caf-level4-azdo-contoso_demo" + } + } + } + +} + +custom_role_definitions = { + + caf-azdo-to-azure-subscription = { + name = "caf-azure-devops-azure-app-service-improvement-program-TO-azure-subscription" + useprefix = true + description = "CAF Custom role for service principal in Azure Devops to access resources" + permissions = { + actions = [ + "Microsoft.Resources/subscriptions/read", + "Microsoft.KeyVault/vaults/read" + ] + } + } + +} + + +role_mapping = { + custom_role_mapping = { + subscriptions = { + logged_in_subscription = { + "caf-azdo-to-azure-subscription" = { + azuread_apps = { + keys = ["contoso_demo"] + } + } + } + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/pipeline/rover.yaml b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/pipeline/rover.yaml new file mode 100644 index 000000000..156fb9983 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/pipeline/rover.yaml @@ -0,0 +1,70 @@ +parameters: + - name: timeoutInMinutes + displayName: 'Timeout in minutes' + type: number + default: 60 + + variables: + - group: release-global + + resources: + containers: + - container: rover + image: $(ROVER_IMAGE) + options: --user 0:0 -e TF_PLUGIN_CACHE_DIR="/home/$(HOME_FOLDER_USER)/plugin-cache" -e TF_DATA_DIR="/home/$(HOME_FOLDER_USER)" + + trigger: none + + jobs: + - job: CAF_Rover + + displayName: Azure Landing Zone + + pool: $(AGENT_POOL) + + continueOnError: false + + workspace: + clean: all + + container: rover + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + steps: + - checkout: self + + - bash: | + git clone --branch $(LANDINGZONE_BRANCH) https://github.com/Azure/caf-terraform-landingzones.git ${BUILD_REPOSITORY_LOCALPATH}/public 2>/dev/null + + az login --identity -u $(msi-resource-id) + + /tf/rover/rover.sh -lz ${BUILD_REPOSITORY_LOCALPATH}$(landingZonePath) \ + -tfstate $(tfstateName) \ + -var-folder ${BUILD_REPOSITORY_LOCALPATH}$(configPath) \ + -parallelism=30 \ + -level $(level) \ + -a $(terraformAction) \ + -env $(ENVIRONMENT) + + condition: ne(variables['terraformAction'], 'destroy') + failOnStderr: true + displayName: 'Terraform $(terraformAction)' + + - bash: | + git clone --branch $(LANDINGZONE_BRANCH) https://github.com/Azure/caf-terraform-landingzones.git ${BUILD_REPOSITORY_LOCALPATH}/public 2>/dev/null + + az login --identity -u $(msi-resource-id) + + /tf/rover/rover.sh -lz ${BUILD_REPOSITORY_LOCALPATH}$(landingZonePath) \ + -tfstate $(tfstateName) \ + -var-folder ${BUILD_REPOSITORY_LOCALPATH}$(configPath) \ + -parallelism=30 \ + -level $(level) \ + -a $(terraformAction) \ + -auto-approve \ + -env $(ENVIRONMENT) + + condition: eq(variables['terraformAction'], 'destroy') + failOnStderr: true + displayName: 'Terraform destroy' diff --git a/caf_solution/add-ons/azure_devops_v1/variables.azuread.tf b/caf_solution/add-ons/azure_devops_v1/variables.azuread.tf new file mode 100644 index 000000000..3ddca35c4 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/variables.azuread.tf @@ -0,0 +1,12 @@ +variable "azuread" { + default = {} +} +variable "azuread_applications" { + default = {} +} +variable "azuread_service_principals" { + default = {} +} +variable "azuread_service_principal_passwords" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/variables.tf b/caf_solution/add-ons/azure_devops_v1/variables.tf new file mode 100644 index 000000000..7444f09db --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/variables.tf @@ -0,0 +1,123 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is propulated by the rover. subscription id hosting the remote tfstates" +} + +variable "global_settings" { + default = {} +} +variable "tenant_id" {} +variable "landingzone" { +} +variable "rover_version" { + default = null +} + +variable "logged_user_objectId" { + default = null +} +variable "logged_aad_app_objectId" { + default = null +} +variable "tags" { + default = null +} +variable "app_service_environments" { + default = {} +} +variable "app_service_plans" { + default = {} +} +variable "app_services" { + default = {} +} +variable "diagnostics_definition" { + default = {} +} +variable "resource_groups" { + default = {} +} +variable "network_security_group_definition" { + default = {} +} +variable "vnets" { + default = {} +} +variable "azurerm_redis_caches" { + default = {} +} +variable "mssql_servers" { + default = {} +} +variable "storage_accounts" { + default = {} +} +variable "storage_account_blobs" { + default = {} +} +variable "azuread_groups" { + default = {} +} +variable "keyvaults" { + default = {} +} +variable "keyvault_access_policies" { + default = {} +} +variable "keyvault_access_policies_azuread_apps" { + default = {} +} +variable "virtual_machines" { + default = {} +} +variable "diagnostic_storage_accounts" { + default = {} +} +variable "virtual_machine_extension_scripts" { + default = {} +} +variable "azure_devops" { + default = {} +} +variable "role_mapping" { + default = {} +} +variable "custom_role_definitions" { + default = {} +} +variable "azuread_apps" { + default = {} +} +variable "dynamic_keyvault_secrets" { + default = {} +} + +### new +variable "organization_agent_pools" { + default = {} +} +variable "organization_url" {} +variable "projects" { + default = {} +} +variable "project_agent_pools" { + default = {} +} +variable "service_endpoints" { + default = {} +} +variable "variable_groups" { + default = {} +} +variable "pipelines" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf b/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf index 48fc5bc32..7b7b28c28 100644 --- a/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf +++ b/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf @@ -17,35 +17,58 @@ data "terraform_remote_state" "remote" { for_each = try(var.landingzone.tfstates, {}) backend = var.landingzone.backend_type - config = { - storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name - container_name = local.landingzone[try(each.value.level, "current")].container_name - resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name - subscription_id = var.tfstate_subscription_id - key = each.value.tfstate - } + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] } locals { + remote_state = { + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = try(value.workspace, local.landingzone[try(value.level, "current")].container_name) + key = value.tfstate + resource_group_name = try(value.resource_group_name, local.landingzone[try(value.level, "current")].resource_group_name) + storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) + subscription_id = try(value.subscription_id, var.tfstate_subscription_id) + tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + } + } + } + landingzone_tag = { "landingzone" = var.landingzone.key } - global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings - diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + global_settings = merge( + try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.global_settings, null), + try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings, null) + ) caf = { - tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) global_settings = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].global_settings, {})) + for key, value in try(var.landingzone.tfstates, {}) : key => merge( + try(data.terraform_remote_state.remote[key].outputs.global_settings, {}), + try(data.terraform_remote_state.remote[key].outputs.objects[key].global_settings, {}) + ) } diagnostics = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].diagnostics, {})) + for key, value in try(var.landingzone.tfstates, {}) : key => merge( + try(data.terraform_remote_state.remote[key].outputs.diagnostics, {}), + try(data.terraform_remote_state.remote[key].outputs.objects[key].diagnostics, {}) + ) } managed_identities = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) } + azuread_groups = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) + } + azuread_service_principals = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_service_principals, {})) + } + azuread_applications = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) + } } } diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/backend.azurerm b/caf_solution/add-ons/cross_tenant_hub_connection/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/hub_connection.tf b/caf_solution/add-ons/cross_tenant_hub_connection/hub_connection.tf new file mode 100644 index 000000000..c35ac7b4d --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/hub_connection.tf @@ -0,0 +1,81 @@ + + +data "azurerm_virtual_hub" "vhub-glrtss-phb-sanhub" { + for_each = var.virtual_hub_connections + + name = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].name + resource_group_name = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].object.resource_group_name + + provider = azurerm.virtual_hub +} + +data "azurerm_virtual_network" "vnw-uglife-dev-az1-aks" { + for_each = var.virtual_hub_connections + + name = data.terraform_remote_state.remote[each.value.vnet.lz_key].outputs.objects[each.value.vnet.lz_key].vnets[each.value.vnet.vnet_key].name + resource_group_name = data.terraform_remote_state.remote[each.value.vnet.lz_key].outputs.objects[each.value.vnet.lz_key].vnets[each.value.vnet.vnet_key].resource_group_name + + provider = azurerm.vnet +} + +resource "null_resource" "wait_for_virtual_hub_state" { + for_each = var.virtual_hub_connections + + triggers = { + routing = try(jsonencode(each.value.routing), null) + } + + provisioner "local-exec" { + command = format("%s/scripts/wait.sh", path.module) + + environment = { + VIRTUAL_HUB_ID = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].id + } + } +} + +resource "azurerm_virtual_hub_connection" "conn" { + for_each = var.virtual_hub_connections + depends_on = [null_resource.wait_for_virtual_hub_state] + + name = each.value.name + virtual_hub_id = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].id + remote_virtual_network_id = data.terraform_remote_state.remote[each.value.vnet.lz_key].outputs.objects[each.value.vnet.lz_key].vnets[each.value.vnet.vnet_key].id + internet_security_enabled = try(each.value.internet_security_enabled, false) + + dynamic "routing" { + for_each = try(each.value.routing, null) == null ? [] : [1] + + content { + + associated_route_table_id = data.terraform_remote_state.remote[each.value.routing.virtual_hub_route_table.lz_key].outputs.objects[each.value.routing.virtual_hub_route_table.lz_key].virtual_hub_route_table[each.value.routing.virtual_hub_route_table.key].id + + dynamic "propagated_route_table" { + for_each = try(each.value.routing.propagated_route_table, null) == null ? [] : [1] + + content { + route_table_ids = flatten( + [ + [ + try(each.value.routing.propagated_route_table.ids, []), + ], + [ + for key, value in try(each.value.routing.propagated_route_table.route_tables, []) : + [ + data.terraform_remote_state.remote[value.lz_key].outputs.objects[value.lz_key].virtual_hub_route_table[value.key].id + ] + ] + ] + ) + labels = try(each.value.routing.propagated_route_table.labels, null) + } + } + } + } + + provider = azurerm.virtual_hub +} + +output azurerm_virtual_hub_connection { + value = azurerm_virtual_hub_connection.conn +} diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/main.tf b/caf_solution/add-ons/cross_tenant_hub_connection/main.tf new file mode 100644 index 000000000..0ccb8f1f4 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/main.tf @@ -0,0 +1,23 @@ + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.55.0" + } + null = { + source = "hashicorp/null" + version = "~> 2.1.0" + } + } + required_version = ">= 0.13" +} + + +# provider "azurerm" { +# features {} +# skip_provider_registration = true +# } + +# data "azurerm_client_config" "current" {} + diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/providers.tf b/caf_solution/add-ons/cross_tenant_hub_connection/providers.tf new file mode 100644 index 000000000..b11dee7d9 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/providers.tf @@ -0,0 +1,54 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + container_name = try(each.value.workspace, local.landingzone[try(each.value.level, "current")].container_name) + key = each.value.tfstate + resource_group_name = try(each.value.resource_group_name, local.landingzone[try(each.value.level, "current")].resource_group_name) + sas_token = try(each.value.sas_token, null) != null ? var.sas_token : null + storage_account_name = try(each.value.storage_account_name, local.landingzone[try(each.value.level, "current")].storage_account_name) + subscription_id = try(each.value.subscription_id, data.azurerm_client_config.current.subscription_id) + tenant_id = try(each.value.tenant_id, data.azurerm_client_config.current.tenant_id) + use_azuread_auth = try(each.value.use_azuread_auth, true) + } +} + +data "azurerm_client_config" "current" { + provider = azurerm.vnet +} + +provider "azurerm" { + alias = "virtual_hub" + features {} + skip_provider_registration = true + subscription_id = var.virtual_hub_subscription_id + tenant_id = var.virtual_hub_tenant_id + + # Source tenants for virtual networks. + # Client ID must have permissions on those virtual_networks + auxiliary_tenant_ids = var.landingzone.tfstates[var.virtual_hub_lz_key].auxiliary_tenant_ids +} +provider "azurerm" { + features {} + alias = "vnet" + skip_provider_registration = true + subscription_id = var.virtual_network_subscription_id + tenant_id = var.virtual_network_tenant_id +} + diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/scripts/wait.sh b/caf_solution/add-ons/cross_tenant_hub_connection/scripts/wait.sh new file mode 100755 index 000000000..ae281ff58 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/scripts/wait.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +function canonicalize() { + echo $@ | sed 's|//subscriptions|/subscriptions|' +} + +API_VERSION=2021-02-01 + +RAW_URL=${resourceManager}/${VIRTUAL_HUB_ID}?api-version=${API_VERSION} + +URL=$(canonicalize ${RAW_URL}) + +PROVISIONING_STATE=$(az rest --method GET --uri ${URL} --query properties.provisioningState -o tsv) + +while ${PROVISIONING_STATE} != "Succeeded" +do + sleep 30 + PROVISIONING_STATE=$(az rest --method GET --uri ${URL} --query properties.provisioningState -o tsv) +done + +echo "Virtual Hub is ready" diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/variables.tf b/caf_solution/add-ons/cross_tenant_hub_connection/variables.tf new file mode 100644 index 000000000..be428ef51 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/variables.tf @@ -0,0 +1,58 @@ +# Map of the remote data state +variable "lower_storage_account_name" { + description = "This value is propulated by the rover" +} +variable "lower_container_name" { + description = "This value is propulated by the rover" +} +variable "lower_resource_group_name" { + description = "This value is propulated by the rover" +} + + +variable "tfstate_subscription_id" { + description = "This value is propulated by the rover. subscription id hosting the remote tfstates" + default = null +} +variable "tfstate_storage_account_name" { + description = "This value is propulated by the rover" +} +variable "tfstate_container_name" { + description = "This value is propulated by the rover" +} +variable "tfstate_resource_group_name" { + description = "This value is propulated by the rover" +} +variable "sas_token" { + default = null +} + +variable "landingzone" {} +variable "virtual_hub_connections" { + default = {} +} + +variable "virtual_hub_subscription_id" { + type = string + description = "Subscription ID of the Virtual Hub." +} + +variable "virtual_hub_tenant_id" { + type = string + description = "Tenant ID of the Virtual Hub." +} + +variable "virtual_network_subscription_id" { + type = string + description = "Subscription ID of the Virtual Network." +} + +variable "virtual_network_tenant_id" { + type = string + description = "Tenant ID of the Virtual Network." +} + +variable "virtual_hub_lz_key" { + type = string + description = "Virtual Hub landingzone key in var.landingzone.tfstate." +} diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/README.md b/caf_solution/add-ons/hashicorp_vault_secrets/README.md new file mode 100644 index 000000000..d0b2e8ba7 --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/README.md @@ -0,0 +1,41 @@ +# CAF landing zones for Terraform - Dynamic Hashicorp Vault Secrets Add-on + +Deploys dynamic hashicop vault secrets. + + +## Prerequisites + +Before running the add-on please make sure you are [authenticated](https://learn.hashicorp.com/tutorials/vault/getting-started-authentication) to vault, one of the way is to set the below environment variables. For more information visit : https://www.vaultproject.io/docs/commands#environment-variables + +``` bash +export VAULT_ADDR = "vault address" +export VAULT_TOKEN= "vault token" + +``` + +## Example + +An example of the configurations to deploy this add-on feature can be found [here](./scenario/100-simple-dynamic-vault-secrets/configuration.tfvars) + +Ensure the below is set prior to apply or destroy. + +```bash +# Login the Azure subscription +rover login -t [TENANT_ID/TENANT_NAME] -s [SUBSCRIPTION_GUID] +# Environment is needed to be defined, otherwise the below LZs will land into sandpit which someone else is working on +export environment=[YOUR_ENVIRONMENT] +``` + +## Run vault dynamic serets deployment + +```bash +rover \ + -lz /tf/caf/landingzones/caf_solution/add-ons/hashicorp_vault_secrets \ + -var-folder /tf/caf/landingzones/caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets \ + -tfstate vault.tfstate \ + -env ${environment}} \ + -level level1 \ + -parallelism 50 \ + -a plan + +``` \ No newline at end of file diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/backend.azurerm b/caf_solution/add-ons/hashicorp_vault_secrets/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf b/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf new file mode 100644 index 000000000..bb8847493 --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf @@ -0,0 +1,48 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] +} + +locals { + remote_state = { + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = try(value.workspace, local.landingzone[try(value.level, "current")].container_name) + key = value.tfstate + resource_group_name = try(value.resource_group_name, local.landingzone[try(value.level, "current")].resource_group_name) + storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) + subscription_id = try(value.subscription_id, var.tfstate_subscription_id) + tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + } + } + } + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + remote = { + objects = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key], {})) + } + } +} diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/main.tf b/caf_solution/add-ons/hashicorp_vault_secrets/main.tf new file mode 100644 index 000000000..30138148b --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.43" + } + vault = { + source = "hashicorp/vault" + version = "~> 2.17.0" + } + } + required_version = ">= 0.13" +} \ No newline at end of file diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/providers.tf b/caf_solution/add-ons/hashicorp_vault_secrets/providers.tf new file mode 100644 index 000000000..efbfdc213 --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/providers.tf @@ -0,0 +1,10 @@ +provider "vault" {} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = true + } + } +} +data "azurerm_client_config" "current" {} diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets/configuration.tfvars b/caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets/configuration.tfvars new file mode 100644 index 000000000..3d0624543 --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets/configuration.tfvars @@ -0,0 +1,60 @@ +landingzone = { + backend_type = "azurerm" + global_settings_key = "launchpad" + level = "level1" + key = "vault" + tfstates = { + launchpad = { + level = "lower" + tfstate = "caf_launchpad.tfstate" + } + } +} +hashicorp_vault_secrets = { + secret1 = { + path = "secret/test" + disable_read = true # optional + secrets = { + username = { # key will be used for secret name in vault + value = "vmadmin" + } + password = { + value = "password" + } + subscription_id = { + lz_key = "launchpad" + output_key = "client_config" + attribute_key = "subscription_id" + } + lower_rg = { + lz_key = "launchpad" + output_key = "resource_groups" + resource_key = "level0" + attribute_key = "name" + } + lower_rg_1 = { + lz_key = "launchpad" + output_key = "resource_groups" + resource_key = "level1" + attribute_key = "name" + } + client_secret = { # scenario to push secrets from key vault to hashicorp vault applicable mostly for Service Principal passwords. + secretname = "sp-client-secret" + lz_key = "launchpad" + keyvault_key = "level1" + } + } + } + secret2 = { + path = "secret/password" + disable_read = true # optional + secrets = { + username = { + value = "vmadmin" + } + password = { + value = "Welcome@1" + } + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/secret/variables.tf b/caf_solution/add-ons/hashicorp_vault_secrets/secret/variables.tf new file mode 100644 index 000000000..e607d024b --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/secret/variables.tf @@ -0,0 +1,6 @@ +variable "secrets" {} +variable "path" {} +variable "disable_read" {} +variable "objects" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/secret/vault_secret.tf b/caf_solution/add-ons/hashicorp_vault_secrets/secret/vault_secret.tf new file mode 100644 index 000000000..f6d4c956c --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/secret/vault_secret.tf @@ -0,0 +1,24 @@ +locals { + transposed = { + for key, value in var.secrets : key => coalesce( + try(value.value, null), + try(var.objects[value.lz_key][value.output_key][value.resource_key][value.attribute_key], null), + try(var.objects[value.lz_key][value.output_key][value.attribute_key], null), + try(data.azurerm_key_vault_secret.client_secret[key].value, null) + ) } +} + +data "azurerm_key_vault_secret" "client_secret" { + for_each = { + for key, value in var.secrets : key => value + if try(value.secretname, null) != null && try(value.secretname, null) != "" + } + name = each.value.secretname + key_vault_id = var.objects[each.value.lz_key].keyvaults[each.value.keyvault_key].id +} + +resource "vault_generic_secret" "azuresecrets" { + path = var.path + disable_read = try(var.disable_read, false) + data_json = jsonencode(local.transposed) +} diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/variables.tf b/caf_solution/add-ons/hashicorp_vault_secrets/variables.tf new file mode 100644 index 000000000..e5f003557 --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/variables.tf @@ -0,0 +1,33 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is propulated by the rover. subscription id hosting the remote tfstates" +} + +variable "global_settings" { + default = {} +} +variable "tenant_id" {} +variable "landingzone" { +} +variable "rover_version" { + default = null +} +variable "logged_user_objectId" { + default = null +} +variable "logged_aad_app_objectId" { + default = null +} +variable "tags" { + default = null +} +variable "hashicorp_vault_secrets" {} \ No newline at end of file diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf new file mode 100644 index 000000000..66b03b258 --- /dev/null +++ b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf @@ -0,0 +1,12 @@ +module "hashicorp_vault_secrets" { + source = "./secret" + for_each = { + for key, value in var.hashicorp_vault_secrets : key => value + if try(value.secrets, null) != null && try(value.secrets, null) != "" && try(value.path, null) != null && try(value.path, null) != "" + } + secrets = each.value.secrets + path = each.value.path + disable_read = try(each.value.disable_read, false) + objects = local.remote.objects +} + diff --git a/caf_solution/add-ons/helm-charts/charts.tf b/caf_solution/add-ons/helm-charts/charts.tf index 74b16be73..be33a7906 100644 --- a/caf_solution/add-ons/helm-charts/charts.tf +++ b/caf_solution/add-ons/helm-charts/charts.tf @@ -11,11 +11,12 @@ resource "helm_release" "chart" { depends_on = [kubernetes_namespace.gitlab_runners] for_each = var.helm_charts - chart = each.value.chart - name = each.value.name - namespace = each.value.namespace - repository = try(each.value.repository, null) - timeout = try(each.value.timeout, 4000) - values = [file(each.value.value_file)] - wait = try(each.value.wait, true) + chart = each.value.chart + create_namespace = try(each.value.create_namespace, false) + name = each.value.name + namespace = each.value.namespace + repository = try(each.value.repository, null) + timeout = try(each.value.timeout, 4000) + values = [file(each.value.value_file)] + wait = try(each.value.wait, true) } \ No newline at end of file diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 2b6cbe71c..fdf540a54 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -5,12 +5,7 @@ module "solution" { source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=5.4.0" # source = "../../aztfmod" - # azuread = var.azuread - azuread_api_permissions = var.azuread_api_permissions - azuread_apps = var.azuread_apps - azuread_groups = var.azuread_groups - azuread_roles = var.azuread_roles - azuread_users = var.azuread_users + azuread = local.azuread cloud = local.cloud compute = local.compute current_landingzone_key = var.landingzone.key diff --git a/caf_solution/local.azuread.tf b/caf_solution/local.azuread.tf new file mode 100644 index 000000000..51f5d7bae --- /dev/null +++ b/caf_solution/local.azuread.tf @@ -0,0 +1,17 @@ +locals { + azuread = merge( + var.azuread, + { + azuread_api_permissions = var.azuread_api_permissions + azuread_applications = var.azuread_applications + azuread_apps = var.azuread_apps + azuread_credential_policies = var.azuread_credential_policies + azuread_credentials = var.azuread_credentials + azuread_groups = var.azuread_groups + azuread_roles = var.azuread_roles + azuread_service_principal_passwords = var.azuread_service_principal_passwords + azuread_service_principals = var.azuread_service_principals + azuread_users = var.azuread_users + } + ) +} diff --git a/caf_solution/local.networking.tf b/caf_solution/local.networking.tf index d4a17cc9b..3e2e36263 100644 --- a/caf_solution/local.networking.tf +++ b/caf_solution/local.networking.tf @@ -10,6 +10,7 @@ locals { azurerm_firewall_nat_rule_collection_definition = var.azurerm_firewall_nat_rule_collection_definition azurerm_firewall_network_rule_collection_definition = var.azurerm_firewall_network_rule_collection_definition azurerm_firewall_policies = var.azurerm_firewall_policies + azurerm_firewall_policy_rule_collection_groups = var.azurerm_firewall_policy_rule_collection_groups azurerm_firewalls = var.azurerm_firewalls azurerm_routes = var.azurerm_routes ddos_services = var.ddos_services @@ -34,12 +35,14 @@ locals { virtual_hub_connections = var.virtual_hub_connections virtual_hub_er_gateway_connections = var.virtual_hub_er_gateway_connections virtual_hub_route_tables = var.virtual_hub_route_tables + virtual_hubs = var.virtual_hubs virtual_network_gateway_connections = var.virtual_network_gateway_connections virtual_network_gateways = var.virtual_network_gateways virtual_wans = var.virtual_wans - virtual_hubs = var.virtual_hubs vnet_peerings = var.vnet_peerings vnets = var.vnets + vpn_sites = var.vpn_sites + vpn_gateway_connections = var.vpn_gateway_connections } ) } diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index dae9e3f3e..87b2db943 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -3,6 +3,9 @@ locals { azuread_apps = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_apps, {})) } + azuread_applications = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) + } azuread_groups = merge( tomap({ "launchpad" = try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.launchpad_identities["launchpad"].azuread_groups, {}) }), { @@ -11,6 +14,9 @@ locals { ) } ) + azuread_service_principals = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_service_principals, {})) + } azuread_users = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_users, {})) } @@ -44,12 +50,12 @@ locals { availability_sets = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].availability_sets, {})) } - azuread_applications = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) - } azuread_users = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_users, {})) } + azurerm_firewall_policies = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azurerm_firewall_policies, {})) + } azurerm_firewalls = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azurerm_firewalls, {})) } @@ -166,6 +172,12 @@ locals { vnets = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) } + vpn_sites = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vpn_sites, {})) + } + vpn_gateway_connections = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vpn_gateway_connections, {})) + } wvd_host_pools = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].wvd_host_pools, {})) } @@ -176,4 +188,4 @@ locals { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].wvd_workspaces, {})) } } -} \ No newline at end of file +} diff --git a/caf_solution/locals.remote_tfstates.tf b/caf_solution/locals.remote_tfstates.tf index a36480631..867e1452b 100644 --- a/caf_solution/locals.remote_tfstates.tf +++ b/caf_solution/locals.remote_tfstates.tf @@ -17,28 +17,34 @@ data "terraform_remote_state" "remote" { for_each = try(var.landingzone.tfstates, {}) backend = var.landingzone.backend_type - config = { - storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name - container_name = try(each.value.workspace, local.landingzone[try(each.value.level, "current")].container_name) - resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name - subscription_id = var.tfstate_subscription_id - key = each.value.tfstate - } + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] } locals { - landingzone_tag = { - "landingzone" = var.landingzone.key + + remote_state = { + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = try(value.workspace, local.landingzone[try(value.level, "current")].container_name) + key = value.tfstate + resource_group_name = try(value.resource_group_name, local.landingzone[try(value.level, "current")].resource_group_name) + storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) + subscription_id = try(value.subscription_id, var.tfstate_subscription_id) + tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + sas_token = try(value.sas_token, null) != null ? var.sas_token : null + } + } + } - tags = merge(try(local.global_settings.tags, {}), local.landingzone_tag, { "level" = var.landingzone.level }, try({ "environment" = local.global_settings.environment }, {}), { "rover_version" = var.rover_version }, var.tags) + tags = merge(try(local.global_settings.tags, {}), { "level" = var.landingzone.level }, try({ "environment" = local.global_settings.environment }, {}), { "rover_version" = var.rover_version }, var.tags) + global_settings = merge( try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings, null), try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.global_settings, null), var.global_settings ) - diagnostics = { # Get the diagnostics settings of services to create diagnostic_event_hub_namespaces = var.diagnostic_event_hub_namespaces @@ -69,16 +75,16 @@ locals { ) } # Get the remote existing diagnostics objects - storage_accounts = coalesce( + storage_accounts = merge( try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.diagnostics.storage_accounts, null), try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.storage_accounts, null) ) - log_analytics = coalesce( + log_analytics = merge( try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.diagnostics.log_analytics, null), try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.log_analytics, null) ) - event_hub_namespaces = coalesce( + event_hub_namespaces = merge( try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.diagnostics.event_hub_namespaces, null), try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.event_hub_namespaces, null) ) diff --git a/caf_solution/main.tf b/caf_solution/main.tf index e2aaad5b2..a8ec361aa 100644 --- a/caf_solution/main.tf +++ b/caf_solution/main.tf @@ -48,7 +48,7 @@ locals { } ) , - data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates + try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates, {}) ) diff --git a/caf_solution/variables.azuread.tf b/caf_solution/variables.azuread.tf new file mode 100644 index 000000000..6230c47f8 --- /dev/null +++ b/caf_solution/variables.azuread.tf @@ -0,0 +1,33 @@ +variable "azuread" { + default = {} +} +variable "azuread_api_permissions" { + default = {} +} +variable "azuread_applications" { + default = {} +} +variable "azuread_apps" { + default = {} +} +variable "azuread_credentials" { + default = {} +} +variable "azuread_groups" { + default = {} +} +variable "azuread_credential_policies" { + default = {} +} +variable "azuread_roles" { + default = {} +} +variable "azuread_service_principals" { + default = {} +} +variable "azuread_service_principal_passwords" { + default = {} +} +variable "azuread_users" { + default = {} +} diff --git a/caf_solution/variables.networking.tf b/caf_solution/variables.networking.tf index b8d416089..b72a0e1b4 100644 --- a/caf_solution/variables.networking.tf +++ b/caf_solution/variables.networking.tf @@ -26,6 +26,9 @@ variable "azurerm_firewall_network_rule_collection_definition" { variable "azurerm_firewall_policies" { default = {} } +variable "azurerm_firewall_policy_rule_collection_groups" { + default = {} +} variable "azurerm_routes" { default = {} } @@ -116,4 +119,10 @@ variable "virtual_hub_route_tables" { } variable "virtual_hub_connections" { default = {} -} \ No newline at end of file +} +variable "vpn_sites" { + default = {} +} +variable "vpn_gateway_connections" { + default = {} +} diff --git a/caf_solution/variables.tf b/caf_solution/variables.tf index 2e776cc12..bbbb09bfd 100644 --- a/caf_solution/variables.tf +++ b/caf_solution/variables.tf @@ -1,15 +1,34 @@ # Map of the remote data state for lower level -variable "lower_storage_account_name" {} -variable "lower_container_name" {} -variable "lower_resource_group_name" {} +variable "lower_storage_account_name" { + default = null +} +variable "lower_container_name" { + default = null +} +variable "lower_resource_group_name" { + default = null +} variable "tfstate_subscription_id" { description = "This value is propulated by the rover. subscription id hosting the remote tfstates" + default = null +} +variable "tfstate_storage_account_name" { + default = null +} +variable "tfstate_container_name" { + default = null +} +variable "tfstate_key" { + default = null +} +variable "tfstate_resource_group_name" { + default = null +} +variable "sas_token" { + description = "SAS Token to access the remote state in another Azure AD tenant." + default = null } -variable "tfstate_storage_account_name" {} -variable "tfstate_container_name" {} -variable "tfstate_key" {} -variable "tfstate_resource_group_name" {} variable "landingzone" { default = { @@ -38,7 +57,7 @@ variable "provider_azurerm_features_keyvault" { variable "rover_version" { - default = {} + default = "caf_standalone" } variable "client_config" { @@ -93,7 +112,7 @@ variable "use_msi" { variable "tags" { description = "Tags to be used for this resource deployment." type = map(any) - default = null + default = {} } variable "resource_groups" { @@ -150,30 +169,6 @@ variable "user_type" { default = {} } -## Azure AD -variable "azuread" { - default = {} -} - -variable "azuread_apps" { - default = {} -} - -variable "azuread_groups" { - default = {} -} - -variable "azuread_roles" { - default = {} -} - -variable "azuread_users" { - default = {} -} - -variable "azuread_api_permissions" { - default = {} -} variable "managed_identities" { description = "Managed Identity configuration objects"