From 2d06e40918c845753d085eb3a1d89694d88d2277 Mon Sep 17 00:00:00 2001 From: lolorol Date: Tue, 27 Apr 2021 10:29:39 +0000 Subject: [PATCH 01/65] Initial commit --- .../azure_devops_v1/agent_pools/agent_pool.tf | 36 +++ .../azure_devops_v1/agent_pools/main.tf | 7 + .../azure_devops_v1/agent_pools/variables.tf | 9 + caf_solution/add-ons/azure_devops_v1/azdo.tf | 11 + .../azure_devops_v1/azdo_agent_pools.tf | 39 +++ .../add-ons/azure_devops_v1/azdo_pipelines.tf | 43 ++++ .../azure_devops_v1/azdo_service_endpoint.tf | 34 +++ .../azure_devops_v1/azdo_variable_groups.tf | 39 +++ .../azure_devops_v1/azuredevops_projects.tf | 41 ++++ .../add-ons/azure_devops_v1/backend.azurerm | 4 + .../documentation/images/pat_token.png | Bin 0 -> 78143 bytes .../azure_devops_v1/locals.remote_tfstates.tf | 62 +++++ caf_solution/add-ons/azure_devops_v1/main.tf | 56 +++++ .../add-ons/azure_devops_v1/output.tf | 8 + .../add-ons/azure_devops_v1/readme.md | 60 +++++ .../200-contoso_demo/azure_devops.tfvars | 232 ++++++++++++++++++ .../200-contoso_demo/configurations.tfvars | 123 ++++++++++ .../200-contoso_demo/pipeline/rover.yaml | 70 ++++++ .../add-ons/azure_devops_v1/variables.tf | 117 +++++++++ 19 files changed, 991 insertions(+) create mode 100644 caf_solution/add-ons/azure_devops_v1/agent_pools/agent_pool.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/agent_pools/main.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/agent_pools/variables.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/azdo.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/backend.azurerm create mode 100644 caf_solution/add-ons/azure_devops_v1/documentation/images/pat_token.png create mode 100644 caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/main.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/output.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/readme.md create mode 100644 caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/azure_devops.tfvars create mode 100644 caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/configurations.tfvars create mode 100644 caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/pipeline/rover.yaml create mode 100644 caf_solution/add-ons/azure_devops_v1/variables.tf diff --git a/caf_solution/add-ons/azure_devops_v1/agent_pools/agent_pool.tf b/caf_solution/add-ons/azure_devops_v1/agent_pools/agent_pool.tf new file mode 100644 index 000000000..510929311 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/agent_pools/agent_pool.tf @@ -0,0 +1,36 @@ +# Get Agent pools by name +data "azuredevops_agent_pool" "pool" { + for_each = { + for key, value in var.settings : key => value + if try(value.name, null) != null + } + + name = each.value.name +} + +resource "azuredevops_agent_queue" "agent_queue" { + for_each = var.settings + + project_id = var.project_id + agent_pool_id = try(data.azuredevops_agent_pool.pool[each.key].id, var.azuredevops_agent_pools[each.value.key].id) +} + +# +# Grant acccess to queue to all pipelines in the project +# + +resource "azuredevops_resource_authorization" "queue" { + for_each = var.settings + + project_id = var.project_id + resource_id = azuredevops_agent_queue.agent_queue[each.key].id + type = "queue" + authorized = try(each.value.grant_access, false) + + +} + + +output "azuredevops_resource_authorization" { + value = azuredevops_resource_authorization.queue +} diff --git a/caf_solution/add-ons/azure_devops_v1/agent_pools/main.tf b/caf_solution/add-ons/azure_devops_v1/agent_pools/main.tf new file mode 100644 index 000000000..a533fb035 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/agent_pools/main.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + azuredevops = { + source = "microsoft/azuredevops" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/agent_pools/variables.tf b/caf_solution/add-ons/azure_devops_v1/agent_pools/variables.tf new file mode 100644 index 000000000..d92c82c8b --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/agent_pools/variables.tf @@ -0,0 +1,9 @@ +variable "azuredevops_agent_pools" { + description = "Agent pools created at the organization level." +} + +variable "project_id" { + description = "Azure devops project id." +} + +variable "settings" {} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/azdo.tf b/caf_solution/add-ons/azure_devops_v1/azdo.tf new file mode 100644 index 000000000..77e39c70b --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo.tf @@ -0,0 +1,11 @@ +# The PAT token must be provisioned in a different deployment +provider "azuredevops" { + org_service_url = var.azure_devops.url + personal_access_token = data.azurerm_key_vault_secret.pat.value +} + +data "azurerm_key_vault_secret" "pat" { + name = var.azure_devops.pats["admin"].secret_name + key_vault_id = local.remote.keyvaults[var.azure_devops.pats["admin"].lz_key][var.azure_devops.pats["admin"].keyvault_key].id + +} diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf b/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf new file mode 100644 index 000000000..53a620d16 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf @@ -0,0 +1,39 @@ + +locals { + organization_agent_pools = try(var.organization_agent_pools, {}) + project_agent_pools = try(var.project_agent_pools, {}) +} + + + +## Agent pools +## Those pools are created in the organization, not the project +resource "azuredevops_agent_pool" "pool" { + for_each = var.organization_agent_pools + + name = each.value.name + auto_provision = try(each.value.auto_provision, false) + pool_type = try(each.value.pool_type, null) +} + +# +# add the agent pools into the project +# + + +module "project_agent_pools" { + source = "./agent_pools" + for_each = var.project_agent_pools + + azuredevops_agent_pools = azuredevops_agent_pool.pool + project_id = data.azuredevops_project.project[each.key].id + settings = each.value + + depends_on = [ + azuredevops_agent_pool.pool, + azuredevops_project.project + ] +} + + + diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf new file mode 100644 index 000000000..cb9178b75 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf @@ -0,0 +1,43 @@ +# data "azuredevops_git_repositories" "repos" { +# project_id = data.azuredevops_project.project.id +# } + + +# locals { +# repositories = zipmap(tolist(data.azuredevops_git_repositories.repos.repositories.*.name), tolist(data.azuredevops_git_repositories.repos.repositories)) +# } + +# resource "azuredevops_build_definition" "build_definition" { + +# for_each = try(var.azure_devops.pipelines, {}) +# project_id = data.azuredevops_project.project.id +# name = each.value.name +# path = each.value.folder + +# variable_groups = lookup(each.value, "variable_group_keys", null) == null ? null : [ +# for key in each.value.variable_group_keys : +# azuredevops_variable_group.variable_group[key].id +# ] + +# repository { +# repo_id = local.repositories[each.value.git_repo_name].id +# repo_type = each.value.repo_type +# yml_path = each.value.yaml +# branch_name = lookup(each.value, "branch_name", null) +# # service_connection_id = lookup(each.value, "repo_type", null) == "github" ? null : azuredevops_serviceendpoint_azurerm.github[each.value.service_connection_key].id +# } + +# ci_trigger { +# use_yaml = true +# } + +# dynamic "variable" { +# for_each = try(each.value.variables, {}) + +# content { +# name = variable.key +# value = variable.value +# } +# } + +# } diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf new file mode 100644 index 000000000..b9b469fde --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf @@ -0,0 +1,34 @@ + +# data "azurerm_key_vault_secret" "client_secret" { +# for_each = var.service_endpoints + +# name = format("%s-client-secret", local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].keyvaults[each.value.keyvault_key].secret_name_client_secret) +# key_vault_id = local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].keyvaults[each.value.keyvault_key].id +# } + +# resource "azuredevops_serviceendpoint_azurerm" "azure" { +# for_each = var.service_endpoints + +# project_id = data.azuredevops_project.project[each.value.project_key].id +# service_endpoint_name = each.value.endpoint_name +# credentials { +# serviceprincipalid = local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].azuread_application.application_id +# serviceprincipalkey = data.azurerm_key_vault_secret.client_secret[each.key].value +# } +# azurerm_spn_tenantid = local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].tenant_id +# azurerm_subscription_id = each.value.subscription.id +# azurerm_subscription_name = each.value.subscription.name +# } + +# # +# # Grant acccess to service endpoint to all pipelines in the project +# # + +# resource "azuredevops_resource_authorization" "endpoint" { +# for_each = var.service_endpoints + +# project_id = data.azuredevops_project.project.id +# resource_id = azuredevops_serviceendpoint_azurerm.azure[each.key].id +# type = "endpoint" +# authorized = true +# } \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf new file mode 100644 index 000000000..597948fe1 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf @@ -0,0 +1,39 @@ +# # +# # permissions required: +# # - vso.variablegroups_manage (create) +# # + vso.buid (update) +# # + vso.build_execute (destroy) +# # +# resource "azuredevops_variable_group" "variable_group" { +# for_each = try(var.azure_devops.variable_groups, {}) + +# project_id = data.azuredevops_project.project.id +# name = each.value.name +# description = try(each.value.description, null) +# allow_access = try(each.value.allow_access, false) + +# dynamic "key_vault" { +# for_each = lookup(each.value, "keyvault", null) == null ? [] : [1] + +# content { +# name = try(each.value.keyvault.lz_key, null) == null ? local.combined.keyvaults[var.landingzone.key][each.value.keyvault.keyvault_key].name : local.combined.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.keyvault_key].name +# service_endpoint_id = azuredevops_serviceendpoint_azurerm.azure[each.value.keyvault.serviceendpoint_key].id +# } +# } + +# dynamic "variable" { +# for_each = { +# for key, variable in each.value.variables : key => { +# name = key == "name" ? variable : key +# value = key == "name" ? null : variable +# } +# } + +# content { +# # When used with Keyvault, the name must be the keyvault secret name and value must not be set +# name = variable.value.name +# value = variable.value.value +# } +# } + +# } diff --git a/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf b/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf new file mode 100644 index 000000000..e12926c46 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf @@ -0,0 +1,41 @@ +resource "azuredevops_project" "project" { + for_each = { + for key, value in var.projects : key => value + if try(value.create, false) + } + + name = each.value.name + description = each.value.description + visibility = try(lower(each.value.visibility), null) + version_control = try(each.value.version_control, null) + work_item_template = try(each.value.work_item_template, null) +} + + +data "azuredevops_project" "project" { + depends_on = [azuredevops_project.project] + + for_each = { + for key, value in var.projects : key => value + if try(value.features, null) != null + } + + name = each.value.name +} + +resource "azuredevops_project_features" "project" { + for_each = { + for key, value in var.projects : key => value + if try(value.features, null) != null + } + + project_id = data.azuredevops_project.project[each.key].id + + features = { + "artifacts" = try(lower(each.value.features.artifacts), "disabled") + "boards" = try(lower(each.value.features.boards), "disabled") + "pipelines" = try(lower(each.value.features.pipelines), "disabled") + "repositories" = try(lower(each.value.features.repositories), "disabled") + "testplans" = try(lower(each.value.features.testplans), "disabled") + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/backend.azurerm b/caf_solution/add-ons/azure_devops_v1/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/documentation/images/pat_token.png b/caf_solution/add-ons/azure_devops_v1/documentation/images/pat_token.png new file mode 100644 index 0000000000000000000000000000000000000000..84fc549c5baa26167b2a975c5ae917e5a46b162e GIT binary patch literal 78143 zcmeEuWmH>R*Djn2ZLtCciWdnk#U0vW1&V916fYJi?oiy_-7RQvFYcb;?i$?fa^CMf z@{Mu--#>Sak&&zW`;) zNQI~CpPrCy6eYeOl@1f{JawKMf0p@-gj5lZdZ+v1ssGyQE6@fB38UlB2Xc>PfdLZI zW3Hs=XJtp7{e{(;OFmQ z9}!?ba=mmaJ~=s|dODN8+wVWrA+5-Ncae}hT|(9I{?$iPf1?NdcmF$GfZ_i-AooMu zpBMLci}U*b2A;+s&gBCa*G1)skhGkf+SV@=4Kds+GkpIRkJ;V(Q-8SAyPfi~-hv5{ zXNs`vTg7(QYuofR4~^7Xbt$msHFd%1Bi@cbw{Ly3hx7b{3viHr{~~F%@#=xr-FKcF zALFml_AG70ABcMUmYn;LsYTuVcCj^UtP2k36clWxn4N=t<8mAlSa6xChkxddvVnJb zHWseF5kLz5$^L;#ZhlGaG1V0_+p)mnFLOwwXlDb~m^^%ZEw_k9y;bJw+6^h>c($JF zE^ZJpZd7I-vcu2<^AQr_SCP=7svVWdzCOHj#Km{^;{YDemCg#U9BL@pY8u%fvgNk!HkhhkuIK~f? zlH|tB1v`g=hNeC1f~=~ zlAbS@X8$7iz3|!M*Ixu#l|*@4mAd8U2m6jL4|@+X3o4fEybdh0(Z- z|j;x*qg2CJkn7@#BggPY0kn=s^craJK;UhKNL^0I*Y59aU<%5cV5QrvgK9uD0#%5;zj4_o%Ow+pD^ zKL1Q4##wL?)9^H?)U~d};)HXk>~gj)SuzF>V)RCB@l90`%?zlkuIGu7v^H#eCSgXE z#LAI0R=Dkj|1d3UN@wBc-Q4vf;l5|yqGdQMHfLX6@fW|aKioHk06#(MCfuekZR3@q zVEvqyJ{R(~ahFoBkK`OFsWek2F+t|MmGuoijI3>!;W2Q0=(F*4-fLGUzA(4$?8-&t z*vFJjDo|aMv%_pzyfudLKs{?t#aeOxo{KSvu;D5$-CbxeCEghWpV_T$XEN)Zq&0+4 zxX``Lgst9@Xduxwe?+rnB!bFcgp3U@7`8)zds(B36P zb3Go5J_iQNhsiSTn9t=ZonyL8+Kx79%Wl#gO);B78}3jbnT52lHMqe;=L1-TP(^i# z^SdGeLlVM7Dg;!8=x%3F^r{v)y6axt#m@4{d%oYYNfcG6Dlc23e+x^QZgKv~YTWum zDv@A3f3ZyHrZFL^&TT0ao?nS!Qlr@ED<8I3mAm<6F7?4oF%Vwt+UtJ_VsCoxJV`JI z4ca`&Qrk1&hq{E)Eohy;$p=$!Euzgh*v(`P!jnz`hvF<#k-nvjiEW0orF+hpGOjf( z;kpaCnfphPUN$-V;xoevbj*CGv2bf;YVjgAZ@oNbt%T|M?Ao~!Nz6!dRT)#R$*(bn zM!zw)oc*PrH8fi6fwW~97}ytdlw9#agibP#^t74;INzGK==*_l@9ge%>UpgDn3l8( zZEWHXvWs38*YTW1Dd)$UHC22`{&}}A(rO@y5!O~I*erwgYqFKE6YPNX!tSd+nRie4 zeASW2aqtBsxK9qXB*hrr7NWoJ#UO^Ou#WRZvex}P1uyGX-Eg<^?O%>Yi;qJDK-8zK z9~JyMWp*)<^lO|HGD_Af{>~lz>3%$fFX8*%%=vAUZ2DpIRX5QXFG12Xbh&n9*yi zhmRTEraAa}xCwAuxO~#^q=Fw8uze~6(9vA4UoWDTVh94bi=0$zPGWL8rq+wMPdYZ% zwd}}&;_Y`YD0(=qznj5lRM)?y^*)01&nty){g(RKnHi!)(-;`c!AxSg8;q#+aKn{E zZVcT1MK-BpP4~jfpY@4Sc?%=+RS`p2p?K6`@BE{wYd;3w$Bd{MX#57)VI>j_o7}9( zL=LGfrVui5(@a%Vnx-Xh;U`XlS9q5OcCp15Fv3a86(?>O49qSYL%JHPz@R`9ZejQ3 z$l`;U69rU%7Q-+tkHX8m=l4uM!oZ2H*j;0GlN^y`vVA)5Qqqc@8t)KaJYG|i6-SWZ zzMc7zsihFGW}yQt%I>?%WXP5POLJ&Vcd{8}nN?8YFC{pOfje=Ho4#x6ZEb1qHVZ2$ z%day1eABK}nSaxDdqthFhm9TOJC$1GAmZLoDMmVc?^YNX(p*AOxpD{aX*x?eubt7xD7ZL7C47_k_u z;wGkSNPTLZwoT!_ag|z^NwqE6yot~3K7(eWx%dfhg&D?`ORJx^WaAse@15lbEP8EV zvfiYtykI)1^3pbnWz1)yHs*-Xt+s|NQVGS@ii{Q`F~o%7sM%6lrea%Z?Tu^9t^Di8DOLN61KQnJ565YAnBy4+B^Rois*d7#P3 zu9mYFyRW6OjQOOI?Z}J6Cg4tWTHrWtt}K+p@<0-1*(-UhNh*92MWs?R_i<6g{4%1c z+(@u~W^J;^|GmjMU3QSE<$#o2Kz-A8@b<6B6NO^ZeN`DZkvJ?;!SDLwpGrA?H#*;; zFtH4p{#?;7%Tc2bFOS3Ur;(X2<4}H+h@6aVoGWEFPd{+|_&HdFLRk2RtFE#k84d4} za=m2Bz?#2KZ?w4u9>#>hZ84tisph6k*{PW{gimo1y0o^H&ef3#AZg_~&Rg(PBiwYG zS8O_IHA{1MT;8vDu9r*|w)@fD8os^H12SndwOg5-$s8*gWq zCW}s?=RQ=jg36=B30T%ZAwIJ?l_X*zD*H|Svbkt2H76Ct_>cZU7T>w9zx(>QZ8h!> zs|JS_yfvy0!A}(KY^f}G`$yR}3TuLT_qUivi=H7`%puu+0B`GMaEQ+}gJtP(+-@D; zzEfkM^BbWbv|Qw=Qp7GAz+N;qMVqW*3nL5|Z$u$8JA<^M{Y59q@rGpr`XUbh`WL`8 zpGq0@&eZ-g7Jl4MLBD~0G!VzfU4(t7^FTgg8FeNVOXtg%KB|52LZNxR?Bb}92$CHu zW@~7vo?Hi$rJ>n!@j0)+0g~A5Y<~5)Pe`pw@zjD;q!xh^qCw_HFpUjrS>;98(%J^b z`>J)D7%1aVFR$~qEpRw85PzYW0rE0L&G6;!744xa0maCSc><%_)QL3WI{W5#&wAR& zn)If^_zJ>4nu)kT&^bLO=frPET;gHHAU2~DLA8Q%_5@Ie<0Pa(*IZC;sEAlYC6SKW zn*Et9{+l@|$ysLq;pwmgJ-r`ep>VT}a|bYjLMmYMf%edT`Ynpw!lEh^d(*>7#J%(_ z{uu0TD`+Q)ftkr>ycOj(q1oY@asqE*zCO^n;XiB!bZrC%S{ zs-`o}BSo)`P8@VxFk|#9%+{m5SJBTD&=9$Rt?^RcY`3Vm^=_NubcupDFO**F_3^md z=`~G9ETk36Nc@nnikMof$;_k5HDu&hy+Uo{VkvoYzb76*_$AU-{UNUDOx!Zu z>#o-B$b5&jqM7Y`G~;#eIi0or4A?Py=iT956K;yOf5^#^nIS;Zsj^0!Y2F&B5!_yh zA8N^wRmfKSsijgaFQq@43W6!&#CTo^9AbX_dQjwi%aNKO-0At0P^9j%1ZQ4HQ3XXb zk#h>06Pq>;rD!>LL-NAWvr7%ZWt#VDk?>hJoPMn)X$IE4DQD)no?Y|y$r)qyiGza; zygI|iGGE%jsCU?DbYx$ow<=!nn(r;E8t?yZ5Z5do=tFza@-Z4ywZE>(s@9b&UjRbK zC6$SrxU^{F9nK@;RQX3Ko^6P(1_3*8@@`LsYa5ZnNMmpt3N(pQo@qM_?G~y&MJXvS zq2R(g@@m$+&Q7XIp(jgPJnI1Sp{#0Lb{G?P0%ZffO=%r(tU8E|H_D1}^5@<7Tcf9} z5f$5X+cE$0JVtNUKEa%sZPA$ULq#FPko zn-EpQT;;8?e*Lbpm^07!fX@tux2#Sfxry4QJK{P?V!kl0y!Z;mc^ig(G$X1(PztS- zB}py43pGsb51@g4wV(3y$5 zm*tY9Pdp!uzYjt-j)fw=-=CVB`F+Pg{pPe9WW@HZLo*CL@>;tx1{(D7#^-xp4{vmu z?_Yl53H&j~%BX%dK|^|u~f$UF+!!LQbPV*!<1d*#`73FWmMBVKde8r z)LB;Ln+sv2V+4ihc_9AjFXm&D=oFC_t+wy4Wm$Ij5@AD zF<}NG`&kpZEABb($uvZ)US^Z|$I574gOLH=X~9H`yRkRl=K{kyvAe-$936zNUsJO- z8sv{gzGoa3N62&z9H-Ebi1or%Y%43R3$F_4jJK4K>e7wuVy?cgWm4?7b(~Y^c;f|= zeNIJ96~&(HdUU6pTWS>RnA>5$#!?M(&{h~rVkzdf+o#~> zerL3{pf$0I8%>a#$g*s@Wpaf@G}4fS7xwaE2pzMzNE7{9?Zo1mXa!na^}RlD$jE@7 z@_tv%X3|w?C2U1}pmUJPV)ol)h|?IbU$g4^q!;Nrc)S4?VxcqB^R;`no;NVvNnvW7 zqZ016UIQ>m^f-f&ynYSrCM*0c!z!-p z_VA`7GgEJf1H^y54a`i1e#d-s6l;BnqW?{dnHoW_CjG?Nz`> zTIHmv>yF)9K4|ovsun2$OL#cWk*P@Yiu*gA;UxKb#)ZY5oP^|-)tS#MRkXS-E1T<6 zqn>xjQ7J{}M{iqC7S-MC73B4-A1Gtfq)$FcXQ82S;8m>W1&W~6#}&S2rnAGk;Pp#(1b0&ctP?q zhZdcPm$LPvd1WMMk5hBtqsS62?(}$^Fms@QV;ka1%=mm(J0>JncIl%4$F-eF&ioN& zmoe>-JlCT4-hqjU3JtgA*n?;-FV?fxz**AlrI*DZEI{NBBZc4Sv1#2stVX_aO z7+9qzjy-ZW#TRUJI`!GEotZ6!^xZ_87&VlizIB;E|YhkkwvqVig}nd zA`iUgjU)Xa`HYWeuQO6zjeS1l?nLaCj%O1s|QVO3|j% z6hK!w@Xgyb0t9a4+#qL?P?Cvn2>N*9^S!2r7w2DjF7aKJ>~dNiDw73<#V+$1cI#>0 zgFBtn*4|;RzBb%AMh!MsN~AHvRU20$k&mhMvtkUoZG}VVfuKgoCIKjWn#M|#4d~6x z6IUJVa=}S0pvJNbB5wN<(~=5wK+7vY-C(d5iG`m6E<$R;y3ZD)1}4R*zbIEb&E7n^ z7weoGz))bWtz4dpReFv`XfA!4L<>3>Yqb8kKAs%>wA@X)Ao8p8lR5vIv=)2$bD(^x z=-o_(yqI-4%Pj#NrltQ~Z^F~^0i@%y7DT*LXZ!tpS&~$;GARJ9{v)J8fLTS?NJs+L zdTgQkraYwB41MRriLm9$PZ?r=D=rJ~x_lT!07l%#hNn39E~a9f;TUalXB_tDgrqI* z-oBXSgvQ@K^!3iZHJ_GDR!x;)7|0dmlR3}_aI-Gkrcup=9c0a=N|prrB}ao$8#e+J(XX&$A55hID^_9YeF z5;yl|(Fv0iB}kMRAB9N9Ea{+3$xh-kn=XHhX!{d~Hv)W^mX$ z*I(*=(|n7iz8FQz=aBG9Xt`gg$N?XvxTEio*?X=|UC-)wc639=2Y_?bc~kohs&_Nr z;;?fG9cH8;Uni>%F5Kb(HGoiWk*O%^zTT(O7)+zpEN z;4CYBiAqH*l@?2lIcoo-VX6}hUIfQNRG2BkJ_$-VV25xI#;+0MI*dJ)aX3*%n5~aQ z+jpuutc{Fb63bvl<~)(eSPI#KS9tPs@1H2i6Ny|#Yp6b+9_^_Ki>ckeyaM(Fzuy_D zpM1sJLho`*H-yeCV5jRx<;06V>h|_ytVQ)pJDQfs{r08?|Ax81M$_LrImi#!pQY_5 zt*b(-MXWN3X&q(21%Z$361cj4nysaJ>Ho0C#6Kr=Nbq3&BIKwFX${?nmwd%~>D*3$ zB#Ghm=IU8Q?Yw(5Nm>RsM|k}Y&Dlk_%Y`?VCJ>;ds!Kht#SDVM|Fp8gsY7bjxIK4} zLSrb{NHf>zwtG&;_6(0KTo!foIm%brutnxowPC;SLto$f=FX!3C4c>qpkZh zPDW%*rtpm?BP*kLDqEm4G-$D*9c`5yR=r^^4xbXfx(_#`N4~B44qXGN|hyFBW0mCG$;b z)N$L;*Cjbmq|ds@aEtP>U*Mfdae&dnD;|h!+z1n(+`Qn4SBAhswT8YEHw-$Y5@x3T z(Har+WsWoW+nP&46jV^>4tipeV^58hTZrRVRkCSfrKT)FGo~=&UU0{-7x7SFLC>B- znT@aQ8v|!kr8ZjEatrY184R(YVc#eYj#(pmSwm;(!RAo?-;}FKUL6{mFSjNlWX&^h z2Vx!)wy63r<6xrC0+xixQ3IqCk$c!XONY(ZOx2}hf>Nco9%O1UV?;JE$XW@qsJN=) z85-vKSFYQb#Qr<8tm6M`W-;%fRfT*i;PHW!b7`+g+4(XpjrUPBlB!DlQ}k*!!C0k2 z7VDYFM)wL~nNC3&b5E}NR4-7aB05Bd=LJ-zMzR`0X6@HUi~7GsS~FSf^s%`u%qUIZ z0X7T^5|!M3;MZ4}x-4@pG5972k7HU&YcomO2^4Lkkt)C&Gw;mCA|Wq-k`vHG$HkVS zzMcAYqK+$Op*?`M1#1kQw*qKOX{**TeOx<%{vPLyjm>mqK+jS%DAHx2BRGaU;-TUL z%*GI3WGtRUCXZeca+rxuZg6UnkxzA{bsn2_Fwo;C!uc2X`FFuNUW|wwAEmx7JR;vf z!(vx0ZNEvJ))PEJx-4FWMy1W}Hzg=jzZB z{m*3wd=W(`{A_;EXF_2uuAkD-7Odi?r&OQW-X!3na$61+;E;>&+N$l@4xx)qjeKk4 zG!wM_CA5XQeviq55W_D2BCc}VNm;+QEUq!B zoM2Sz0qkP3;Qfc?Nt+b7+Nh7Ik(V+)C6z3u0PhRXUR-bWVkmMpit(B2;BC8jm z>RckM*T@$z0K72go7;$h2iy&mxE9eapfaJ>?K{x5-iF4QwLsSL=c}Sbs?QQm`8czT znte}3X`&7h10o)Q;p0XS*Vm$Q3#z;B*=N9d?QbW*!uaU9*xj$tnTns}x_e~c9#NPmTKvQ&s^L(hIYoa1LO*=XBLW7SAt zU_8D>3vpV`L|J%mgxe0XHE||2wzARTX*xY+ac;x}x2vi+SxZ)=ClbeQyS>f=h%A-uRlR;1@h)O1(^F159wY%P$(xv09IS!WYSeFrew+{P zXvuF(i9%ayz~%?<48gBuXwqU!h{0RF5DOvH#mjoXV#47tW@v% zyjieN10SbM(b@{}gNC5YH_4|W%9zNP6IjiM`w)`3|ZAAG+hwKHBE0IbRRnZsy1~u~7R`l*#lN2*@ z_gQusT9CsSR(w&DpTeNypQTZ50U18)tc%pWk{PGOYS@fwl>f`hnj0%0|B*i0pY~02 zQ*THWgZz{>Qg*e*C@%FR+<#7}HFKSYXF(Tg^7mCO-_@Hw)r{Fw6VNKP^To7KcgWpY zpLsp;it$tiY5!R1jPFCaS{ClqmP7!ilc5L}=G15a!VM?gT60z`U%(hJ_{i}y4m#!~ zfd3-Tv^y#1%N)}xv7{Hs6$0S z&4_yJdon>QNsaRAmhI(mUe++$kylxg zRPp2nevUlm?^PT`VTy+E(g5~P95*A?@7v^mGyZt+AUBG6<&^W-Hs2>8u{4s zr{*U5z@F2{ky|t4NW3G--;U{Ruls^{B$QBx6$%ra$5cK6nKTU*37-T7qL&V8>|Dz= z9KXBD`#=Ojk_qF5s!o3ZeOwgh5;VFEs26QG=}T)W>af4bP!oV>#%0y_EI0LUsvo|*Z!=HvHL@C3L{EM8I332?IX zPfF>i3w&%+kW=R58&nXB*_;HG1aD>LAM}{of76OeGM4!K93PNSGyeGV%dk%8l#`2unFV^z>BT7a$lGihubK`XIY zd$U_Xn~AFf#pw7w z2jzK=MLd<|VbP`12h_g~M54#8VD!8e?HT4{cCh8B428_R+(vp=Z!+i6Sg-NKF>W~I zqdadxgK)9P)%c;OBp36y%OQEH<7HGS-c3I721h1(CMB7-Y*848JaaC&^NpWKFt-X9 zFHHI5HC7ZT-B@0B&JXsg;^fQ^t|trxw*-*Va$jpQx#qRxM`6q#VzZ3gd@kkUcTpqS z$;5s zT)qpwLUFlig5RLo&3aZ3wJV(Ga~OM);}pznTWzm(r^2b!!+fp}X@(W^bNRq37|mm& zr!B!Hxgmf5pz`}0Lu4sPe?(*{5#LO>UCp?C_zH05e*y?0DWN6s7h~O{BOHTB(J9YA zG(0uw!>&nsl0E8eUH6EI)CW}3SCbe?&d{*bp%?^b$6?Q>e>UzZQ$5-Kd)%M4_2n)@ zM*r1EDu34r{dfPj;@g2g_c8w6q(3wMGjUJvGQOx_{QJYF7ysYD|FbdpP8z7W=vtLp zyIIYnEc5qfH6)}Mj0#4hyom&vX!{vo_rhfV#qgG$^n#XueFcfs;Qu!9am)-{tIKhA z>E8&1^u9WzG6y#Z^P66I7_%%^u~RT>Yh(X2_Xvk43!WQN0#NYgk!)%@sgMa`RmO#q zB3lX0ZBVG7i}iVK(v403Dg(R#(sBNFDfrK^b#hn(e=N7e#uDP!=uh-0rKfJD7f=g+ zQWTkLU2Hzw3w97UDx&L>L=SCB$(7B14y?$0U@&SD(C>MsZvM9!&s(x5Fd~ElQR0VO zHv)%K8!!nu*P_!eMFkxx_jCN+g^UKbB+i^QexCf;Je{wrSc6ZM>kKuS&Or{_lrtf2 zP5i=wo8GbfUzB8?9Q$v(R<(bg-2=|$Wl<^(HrYJ*E4IV1qn%q7AEl)10gY=V;G;qM zQh0i^@bUTUU9;INGB*ce-ljRrKF#}PVz`(>R&w9&KOqZgK=jFn)Z>aN8MY2a4%s>@ z&iD2n@0?vxE-A89_O%2X!>)TBdcrID;quXcp9IpfyHRX>e7>w|c^68yJ8x>n@VvG? z*W_)ty|OXA`f42a?J&Bsu|ak#X4Qm-e>_4CVZtZL)(BF=nmF*YxZHSI0a(bfP>WlA1_`J1#Y@<^~ zMZa&bSd-&_MoCGL6}9o4Cfw*?4ij;IhI6kG1NHYSt{Y87=6{%VXJe#4k;+S1-QI;a zh}`NtdwIFD36!Y}*|Ap4_PQVzaAf>QMM1$xuOj)5;q1r$&cUhUBUnfWVb3LH>Xazp z^-Xm6-Ll@d|mb| zueMfcYw%NFNAs^aIZ$2kP={%njvZ0;&TNYe=ETWI-rZ%yQq3Xyc-&hj^hJ`FXYWTL zAhXMHgoUb>!KdZ4h>YK(I0})0K&dRKv|GdV+4jO>$FeC?!&$$W!$AY*1M*m5*7qs| z=SBzSO`};HYfLn*gX!%e$*(z7F4MzZc;3Cf&@(@bZPPljSlvQe)ku+3Dv1{dm}qbD z#gw&jrsdFo2ExzHCrXr(7gL>G;4HV-Todm_8hGb*FD5NtCUd8V4gWfL;D6@*g%X?d zGyCdMa`pbC<+A7EfnchX%WJd#kqd9+&-oqbCrZo_a#i*Y%Ht=g>UY}=dohv=HZhI{ z3#*n>FH<5=y`v@kr-bi~)NwXazx06?r#r}}(y5MRW8{jXAg=6OJY^5!Jrabp z8y#rv?Ec&U|359`$0c>#Mx(I?Cj=;#Sjxk4>$~Vs!T6JZtNAw5!{Bonn)7~>ILn=$ z_0;UH`A=(X@0R3aM3BDe92Dz9M6C6kS;(EEsm5tqnTko<-Br zpzem|AkxMhW-&PLzO1m`+@42k273qH?`-7b`xF(jU2?n6pGdLWb=L*J55;FCKuraVfas3}ok zch1JS-*t^-is54?E?ian(`+LYv2wZ3r%%*qO2S}1-wH!|Y^X?qoHpIi4}dp4@A=Ex zCWIV4tZ+I)SS%T(^|teH$vkwl5Pjlq_$RWOb`4d7?*_1|O<)mn?XW}&Z|nYyPSZ?{ zr(wSSVeT@)WvqC;!{u?Qc5geS#uEpiVP5v;URub)k2JbtjB$D#c-qqKyqo`m%&kSK z%$#VO%cV!nTc$AaAJ%ynw#q?W#Ig^&EI$-`YWe^4KRgE93N1zz!!{ zV~r?4DWLB;d_bSx-$TyzD?$iK_7~6`e`Yeyqm6LsHwJX7s?$aBx0sI^KJp>34BRq7 zUU!v1A}N77M;~=f$zP0LR3*D);&3XQ2hDDNF?O$az8EAKzdT=kV#UQ@A2I!!kJ`Yg z7}otvd!0}$xo|ozv|ek+njMAF)S))Ht($n;$RsM(qf>=v(gk=?;Y=PD%Q>M6fK5fc zQMP)HBzbo>C_IqDz?Yd5&46{d_|_6y+q;{l(AC zf|tbAwvod>7!l3>_DVFk@s}x#T$)*5F1E$wU?lTH(y5H`DFN{8?ik`Nh2mnIV2Kb7 z)xpH^SUbHLY4*D{YuL;eXckw?^?LGNKaP4f3+3)scznG2y7lT^J{l*T>1VtR;e(+O zxdb|Da%*^inpD_f1+k7dwLRQEA%^s-t)MWOLK=;bPIb3pq^GrBNAne=8gV%(0V-Q+ zVEO*oG9>JNKX#^@TD8V!NP1^`)wIxM!3@4^n$pqCRU%B)z$E6r4YqXUKE!93->6kB zwu1eh%nPB4x3fAnyV3U{P#eB4WBuNuuIT2*`AsG^=VHPQb)2&-{N=WNe^i}CzqGW- zsdOgKVBcx{W1U##dffREVt* z->?z4V`)&hgFeEzCwEd`^%!|Cff~gc{f5l)(M|GhkBje5g=XRkz7aiP$gneCN_PT) z-{i3A9_yhYoWuq7*lRZ&DLDHrBW)*P!zP7rU-Q0qMi!J!rDi_9J++G1qz${f9>B+$ zQ&UXd!Rf=&168_ic-utFoQ6EM5TUlx@({`Kh$LihTgFw?-@)SSt zTi-umA$^61HGu63u#o!}Z14tmttmmx5$H*^B$T zXkyf5R>zC*Mh7=mH}zJG1OC1;_YoNo)ULJ+45<;H95zf235i#tZ5Nq%xH-Q9r(Nq} z+n1(k_jpw9HI7|}wm|EugYWd=JAdqC4m{+O31_};Pq<%ua5EEj-l~7-C#^YzSsW$o zq+cH_ZC%3mWDttp;Q2VZXs&Is+-bXXc4PUra>kyQT4^RVUT@&(qq?wA z-`X`G*(2p%%go_`aJQC}Yp(~ixf>hponDGYPn>N#^BDc*7vmF*+?&?|XdSa(+6@So z-22O1Z@WZZE!a{v<{aLrKFTC5OoYlrX6JR|r$iQKw%IS}L~5BC*rMK7W6PNjx= zLVocn#6&9$FY_#2)q)-GDi*gvDqrJ>xKedJK}{jj+#YezBZpAaOl4qXaP;Tlv|q>{ zP!rX>^okm_JfTfRZXc|F%*DprPWM=Fl1UIK4n%C^X50H$4B*E(KHXpUSNoVM)(ItU zwPyH0E!NcOlSC!y@*mtpW#&*BZ_@r7Hz3}pgk~ESX{1X;Q-+V{#@_v*l)@A~{6$c= z^PxtIrM<%qwiO&w^W;IP0v=xAb&2hp1!wK3R|=YVI}WZC+`PA?Ts8iLvZM>$EpU*7 zW7wN0JJT6WQ!=h=0dbGh$T1myy92mpsxmPe8ogdqPF==<^VnzY^c`)`Q%WeVqiudm zU^3m?h7c2#=5az#6NFatMYAycSfD|Ho1iPm<~eZMS0TKGlCp^!qaUIlK`-we4M?;a ze`mC!kKni5oI`#d1i#|3vbc3!(Pn!~K#fm2!n$<#pz_%~h?tXe<@!#p`W#}hPdgJ}SX>o3U^zqf~PJFU!fS-*l{?HZ`^&50s?)-V3ZA&n8T{jf0 zx}}C{zp&*B+@F9&i ztOn%!w?o%ItlSyQ$?Z#M21#-s!IKF6mrT`>{8Kx=wP%SurEZZ!Ejn)EE``z1I?rfl z?Q#Q>IfIX;A#10o!t^zL*NCr}+FZu5i>3XnpQ1JKEA(ZxZ!e!RvJ1v@0(9iBXtQMw zO+wFHl4+P}*iZBLa{K$~Rmunpky}k(>h^63>FSMbboLpKp6=#!54XzgS}z}wB)KS_(eVO7bEP1M;& zEBIw~i9A&r?dEXDqv?S}CnGwLJYstDLy~bvUGY)TWe6*Fu>AFiBc4a8MbdSMX`U_pEeJNarx_N7miH&~1wqz-3JDa;ahTI0>!K>A(TC`t!RZ>fcz+EgA_G+P0)$&`YuD3IOt$6t60h}(hJi1{)z~BYS50okS ztHGq{^E4Fbdgt|sqe^S-`w<0cd|Z+_a!J{6Oh?&N<@bILw57zaM)z)WJRVEC(l91? z;jpmhIn!qcn2|S5nkk*X4;tZ;p+?y^>)dc6{iz(Fnw{d>cOUS)+(^}(x>P&ObR=AU$YneW!iZ5stN0s~XU?EeM4`UBxYn2>ltZ#uJ3ufGo7hQv0 zs9cYc+w1G>tC^z(nK%nC>r?~6)!YD6sO_yZ_8YH-5x)8o?3xoD`L$jEQ14X;AC#u_0=e!G3)sb~avP9Q2q4BJD?Guzt);0}Rz zW5ktL?d*!on`SX5F=^gtvo;;>K`_LZuOHmpK?LGeRAibDv%j%|{|MgB z0MP5T@&VBIM+al-a;0`H$(a>cg0m>9;=}ICNdX}OhEW#_8nJKR>JQ(uaF+G>o2;#8 z#W!=f10!^y(+)iZzMHz_kcw`2BQO=(x3 zSfnT@n1N41LMN*34dvLG*;2#R_OtBLD>=6Lf{Lm58)Q7OIw!s3#`ImLou?Gy>m76m z`)o8dEFm1zCj4>wLjr(v`+Ei-T_xH4g1YBmV3(f@c%RcG_%jWG{egrGx4_T7p14-Q z9OG~rYPUP3V=UaM((TZEs@yW?`cb@dkp3hjn0~bsP^hHGr$vO9v4WToXF19pY^bFx zwTVwL`>DH|v%0)b^X8yt5pt-tFhMS+LkAM|()L$~KKIi1fl&@nqwg%%KPIQ&5x-ZF zKVeFZmf+^i+UN-v-w~@a5C=QjY=@Bi^z8s*uTfNygN0jX>g835oSLDs+B#9Cd*~xE zcD(doQe#|a6Xf+{_g5MKTE92vAIB@r1ds2%zSb4vKpy5=m6mGpZ_@)B!~F{X(P9u1 zPj|QWWUZpMQ)924dz1EWXk@kZn@%!onVm?XWc^hxzX~er$BU4c=R>=S`6Xk!PLW!YmP#^nN4i9-UVoSpzG2V1*~r z%|@fmffAufyUN#f+&@`L9kp7`yR(2?2)PmM6R$}22s?n)M31G=-h@eKh~1?^Jdv!< z{SxqrU7iFU*zs{nE(orb#ivu=!v`I^2kN*DX0XtvAz##o+3C6jvlR79i6@OPH`Hop@?F~rr~IgN8m z3w^KSr{Kh#e8Q?G7cMK6{Qe#-R#x2J?&>`kB9BD=X=#u0OxMIne*DK;kITnW0oBg0 zf^ogo6Ln>^H!^!V@&#Fw!|L`~qevQf`$Dw5SzorL3eBwv zF|hybQ@`)qpRSsDsXs`n?;Q(kCbgvEnGCO3Qxtn;Y)r_$)>G8#tcs#tgj3VgJ(8v& zx;iO%oN9h-swAngW|EIhl@%&oN=@}u_H3U+u&OWMg_^6dMC@w9kENx2bdCFDPr*JW z2bR&y50eB|D!3&TMjIJU8654O-_|mw&zHp>lQJoWGu|FfSc3V~%o)A1A#Fg09hK~& zlAE#iQcxrJf)tO_`+WmRmefUw{P9=wfY!0?`4VOK_8tkfv60SLt;cX z*gaMIqmRn`4cX;8Cq>YeDd_jZcyo8Hs<7E>#LB}I;^RfI@Z)TQ{j5#Nd6`cA;MoCt zwa>smC5umV|Gkyk{J5bzC(d3s4EHt+E#0fMvRyP{RowKg4A?aq5nK-)(~-b-8Y#}C596nbCZMG@_HJVjzvtxK zt>k4!kBH9~x*cYdI7sE!`)MSKX#?DjokvF02yCetj-q8P!Q)i@8Zt}Ji|ZhSFZG`F z_C>mfdaIM1GBims1F>DkEdx$pa{CJFUKw#OVyB2&t2=x~6IbCTWH&NMs$}PqVIpj{ zHzm?OtG0#ree9_Dx=KW>l(j8l_$c%X6 zc>P_d&J`}DIXpvQ-~`p-apd8qSAxjuXP1ZKqtmV!yV1dMZGWXi;De=5qz7`?1K$tp zTc{loC#gvD)^HRbfyl1q17D?}Z9O+(8H|FwJ(K%Vk2%+wR%s-!Mmt@fJH9 zP^=9~l~8dx#>O#?qAnG4QmIsPiL)xNo}>;k0Vp&cXb$(QnsDSc?iWNJoa|>i zVjmRZDqMjx1iQ*(SrlGRN%~ftPbS}9CG|{mscebSC7+0MA-O=Qs5vuo7uKx5CUkYr zt9{-b5UBCk{m$clB_e1Q?p6+liLX!UD z4m?1=c~##8zOL~~NG_7)gwcsPo0=x_ds-FklnS@YjxJ$2Wb5N8aGck=2%hKh*_BOC z*VmHV4O6tbd0aCkj<#i_r_`Yj{JGA&6$!?;e`C#ao(FDa_v`)q`~YI`$xF5Odtoy<-O3+uhcW*kUuWby`!B>w(c2dvMf5V7nl zEwP8r`KJO!6f7b`@U9#3A0Euec$72WB{$jU{UGHfeu%sp|i7ElIWg&#iMwiro9sAU?){=)6>>Y(vgt9Vo!;gRuz!(ysfWt@5<8szCD8` zZ8X{e(A(oWzvm25@-v?ooP$J@Phx+Jmt>hl06bitlO`cN8&|vjM=b=YRDajI@ z*;d*IuYdh73429=dK|Pdn)b5Y+Q=`aFig-OqhTnNfD_+M^ANovC&?hu{Q{F#sOW=v z&yCVBwB}sI_h2R|6%W~qoQoPw_$HlQIV1o zDQRgH=`!e)?m-9Xk{D_T3t@(mmPWcehY}@+?(QDCVTPG`FQ5DVJ&(`#z29HoU+-G) zf2=iYxUSjfKKDNMaqM%SC}VtWd2w&Z_LMr#MNb4GPu{p&MVq~yXocZS1d5a&?=xUp z>aq*e$sb<)?%xp)zgz^Q*p7 zF)=QTWJlq+B+ftK8~4_8-UDRW$x>1)ZLNdSa|-t0dir$WtixPG#;F#rOD4@^kb0MI z+HM@OKQ5E8D4=d7)?hj|oQ&-lqj<)fY{SUM9a1cx1?ZWx^%yYgWk#lr1T`12@ zkJn~$kr%qw%P0PTl@okYXmOpOzf&?0GEVybs|nMdIoV2?{n}8uJa^@WOn!`bUguQv z>oKF?4eA!`=4M6xbb;V~L(F=k#m@+DcLqRiie`{>VL6#8YrO7@m>{J)u5EcI=nR+3hTjeSD-&bYsgcU8wUENC@OAOn5 z!5_^!hU`<{!E`+_nXK@$vQbx_C`+E|akC%GtrpK42vs3EKMizg(YTOeZ5~b`m-VH759+W(XQoTkkvpoAMbK?iMZkB1$xyccX5JeKX}n9v>`j_- z;Pb(n49O9(0MVg5c6m>LS~nAUX0_fWm7_uJ)9zfVA%SHT4(5@NCX3fo&5;n(BotxN z&Jo&9csj(ek$dvh!|mm}R+y-$$9cVsLMFAVQN_~7vA#zeoeGxYZzkx_JW=VFImGxq zqE>MM|j(}50JOf&=nmI^h2)B&U73Li?(<>+YA=qYL*O~ z4Zn9PN!D5?msJ!S^98w_udQ=(OZ)sv!jzO2 zzYUe|&Ovh>$Auy!@HcfhS0B7@+2MH*Ew?=(6k#Jho0?oc;qNbN;$o}kN8lAJ+NR)# zem%0Uar{wzCh%ln&p8JtxECTKN}l%@6X_miCGkQ|PO1nt_?@t(5c3xKH6=OaaxYGz znAQ$<=CjNC3FOc^^--^3tfqq^N*c7@Yz8&*us3?TyeiE=*x2N`4uRCUl-kKys_SE) z6%=+tqR_$Rm8xB_ml;99I>M$}u*2h_y#Phz%u|P+o+^oGtW2#O=L?>71b${w3UkNn zOl6h3Q-Dp(Fi$`IqhE>$c;sqhZ6M;8_ZF#`n}Bt(5L1NAmiw1Bo9>X5q{3Q?^ve*# z5@I*k?NjuFPrC`%&RVoPY=~ZsOpT``9B6Bj+azpc>Ed^eS~H2qM<{*MjSLJ1G+&LN z^+6B$)gOw1*W-dcsOieyUUl7V{!(xCyrEx1#)0q24Swfo9cKnA3eD*ImB&Y^qc(Z{ z?~Bibul}dD_k3^g_q?6|FOXO_IQ1`(h`LEDNMIffn&k>qa^4l7$zofg=D~{>k@Egb zl9Q!5*SL<4+=2QMz?fjJ@_3zt{Jav@3rjL!k-Yox1O4-pvhq$+5J?t~0SBsmXM?G>uCW4yV~AH)w_> zUL1yb_*!1e6-3OK;hnb(1R(86K>du8*-I1Ge$xNg&UfC{;hv866vg+)sQRxSQ~AT* zfMhjJ{QlwKx2lf$A-LRSkW<(j0z#2YUFlPqV9Jc z@t~WxF>WvANOquY+*dQt?D<_??5@sEnT8S~LnNpuDep(*il}QlLqQv0XIrWLiyVyP zTzhxLm&VSX+MwK^&QBk3^=Gnq-|6J`{JmRIA|NBb7dcP$>}L1stZsOU99ge{bR!)> zB;nFDwD(HK{QbePBMzrwB{#N6pl;xUjFLia4mDn}k78$LyiKmQ{j|^#+AcsHIy2iF zZmCFclUjO@*Tm4wqe`~5ag&uE4qL;`_~ls?bZnG9t-LNRS*B<@kHNk7=!6N&oq6>? zxeyn_n{F1XV5h%1`@`@zU7UIAjV36ik3?Bn2ujCpkvXfc4A;{uFuMb&lCk0H;Rh3G zeJkBkcz;y1oU^yj63sBNc=4n5uAJIu)Gfs)bUbk(_LeD(^7b=?0%!>C!6)_SyaAXr zxi3;3nS_4(?Mts3Z}|C+B2IQQ)KNAoRr( z&)K>%EF-dd%O)qX&{>ax8Wiaj1!9s`EJWsD(lkrd66!spem`h!#=q{qud{v21B<-p zzx!=gU@6iXk{Gn4x#?dP94QlJxMf;=ugo(G*8<2@!!N#7k$-5dKSFp}VzU5bDxJ`$ z|1WA@8Sd#NUJ$y=Pg1Q6w`(V7m(#_M<_nz=CUuP$I`5H9(~Qu{4gCfA`Ex?Z%dab( ziTITJ$rq@p*iS3gH3gXBCyUoX!&70>f) zkF8A8tj`Il^eZ=3nl#d@R#zV# z=~~j++D9MC_{UFA&&sPkTGsiWjBJ6WYJyHV($z!Fq6@$pCRB>}h3wm=!i7|O~+mV)?L}3hJfb2uxJEC4s?dJWS zjpL*VjOwPTs5V~fsOy=mq z-EU~KR}B@Ces=jj0ILe4dq{U7OHXuLoZ9W_LvC)M@5ir$bX2ktC2TvKlc=$&E^DhZ zkn#yiP!pIE1RrttU0j*`Ia}o&~&Se<#yIr`?+O*^iQDMo$E~pQndYR zabY+4UxYPG6Lgfc-(y`{irob@#`9C=W~V%MPCX?ww6B6bk8J#4kU?Vz8G=Lf+|2~( zHO6uMq;xYz_1a4h!>_@#$?m0VdkfL%oAwJ6c%QrNOpT4~P<2b1-`VVhhu*w>>-MD` zrmK6rGRHVdbJqb;?KE8Xa$GM(-6EH|m3Ev@21!f9v#p6Fnl!NC?Ht`b{2}!P`LzAj zUxK^->fo>QNHstI8{AvuRfZj$H<};$89KJ`CCL&f&r{kMWDcCMh~~+lyspw6<>uG14Nm1 zZ|&9}JwAZn#kK-86_T~TI$vD3(rpWYqWrfr0)OyM_%i)h>*#yJFMw1hIa!OY)vIb# zQO#9~%mny5k0PY9axyZe)xP^asxV!d{ipMv|Avu_n*8RHUR`8PMdRPPa5UwCi*u>2 z7Grr8b%i>*g{OK<52eHG<)!5aO7YA)`t#eR_Gd~t93FRFH9|XQ%>+c!5-2f63Aqt8 zm?FieXyml9+SC>csA%spm%r$hG|QN~?Pr6Egny5e0shr{=ngK(qDyK{A5m?JY|83O z3^16k+Br=4Pg1#ZrBUH#_R%+Zh>7TDb^iBpKaGP?~bwl7FnLj^K zb@LB-{6}lxmG}Mx1q}_7kn)dw()UifH?LWs|EH0PUvqczEOWwXG#>s6ue+6Njv&?@#q14oEi3i2o>J zGY9qb&)3*ZhEjBEaqo?wy{$)kQL1te{%a2Ck0+1X|2^LA_xJwa4a6Q!A>X$)4N*E= zVxXi&vS#Jvyqo`RB*5OFW?mus9ptG?Y1Yv_qS(?Me0w+(UaeI8&+g<`B%fj5G%u?s zmpj7o%?`Iv4SJVjq^)$z^M>R}#o2$pcG@GbAsf%MElQ_Nk<_s4wQGn@)?`o~#gte+ zjI=WPsNR>Rp(eK2=#TF_vzY~s$d2tUkLJPO=wiem`7Est#>bF%2*{@v?VMv5{b5G8 zbq|Lrs5Fe<`p+-s%>pMdMTKK#QuK3r)e8*^n?Gg}mKba>MQu&&Hf!I&8DIT_#JF+= zxr@7J-if#n(ZcGpCn8Y|hCx-M{<s_Jq)$io?`8xCjdQ%p#CHIW~bl$Y2-;-TO zl%n+k8{pSd7lZ+Nfw&MN_3nbuP9s~y6YXD->ZUzE9(T8_`)u4HCw+KNQ7em^*ofHE zQFhO=Px;-TjNLr?$)m=;1-$R0frVclBm!R8dhIY%PTvbH9w8-_i|7tlBCAt~UX%`3 zz05ZRS9ZI(v{CNQLDN_^Dj`9KAo$26CU*jQf> z?b?Y5W00uHcjn~q98N!znWHLcz$=zG0tdX`Jr#W(KaQaAxr}BT-59F}PP*(X);RJ< zQZn4LKF|AR3I(&}JujJac@WHYy~NuI=yzE!nHIJluQONJYY&S*9K$>Ha*35@nbOEyI3$eJg-pj}(0s`}=6&G4p&r+h<#@!jV3T~nA zdu%`OM{Ug8Du!^MTE9zYtJ<9ImWwaZ$!J^PBGTtKd*u6*XSuym+IKN-7+R;i8CY|j zn@*tqMq}%3v(18A@iqhcDbyt(kbLlRxyjDKO`c)S62?_LpAznDEC#ue{SB>jt|!5_FI_zUFJQ2bHq0 znW_y#7Du=uW{QnPK1my!+!t|;$aE!NADsmGPAh7G_XiCuK#F4Cyz=Faqc!!8^l<68 zFb1h(nHVVd{%iQkMDa4K?Z+kcp$qK}ckJbCKKoIQdNLaQadvel^3Nccb_x9AW@X|V z>K9HD3S3g>TXNG&Bdu4t##c)tn)O3ODqrhImVZvy>L}h}+e8okvV9j!S#Rf5xiV3w zuK*Qlxwj6IGVb#IWsPyYMMR<=XDc~Z5HBvERCfTCNkyZfM6rdJUgLuc-zYoohfY=H zJGnh&`{T#IuZJ7N-d^00dV?M&&YZc24+uV`Cb1$X8gUveb5$@eu@oASW6KMj@lZjw z{J7a`(lfZQ6=`1IVPvxg*V!x!D=N&|>Imaze$+xpFr0Ox(c4ae9pP%*dv(4v`hIeU zD&Fk^=@ct)dF(%I#&nh>`Cq~Hhm<_8ikvgl1^KyyKFgTAiAew2R+!M+y^V)TnZB8eF29N*xT9s@nxaRP^D?uQw2;K@_x__;Ub%9{SjJ1$Q8s(N`DsgKsAz$TsHSZ_ zKTm7fGE}ZO^^SCw8kf;0$JrU+q2@Wem#$AmO>GD2ot|f^3>ryg(=Kl81xffbQ5oBe zSD8N_P=28rGT}IS-+Q)nB}IlRo6eC{scL3r_YHqr2T#QX(7u2U`ZFWvlC_rWu-Vmo z(B`ZxU{NZ=5$Q3iX@1n-mN?x(Jdh&Prw;e|?Q# zSa8eJE!oWuy>|RNkZ7R~GMYFwl)`2aRhm&2ko!-`)+HA=GbMm2`6pvC`|@TI zgtktQ=d`ZUdhvhy(Q~bl{sxI_1(#`qjv_X$MI1L9v3C} zK_~x-llu%es-g&b?;Sn%$Q1qr+S2NmR})C7&^}zml9NBy&${2(9%7wG z_m}o`FKT73v$g^wlt?mrbVMq1oulghj4nI~u{!4W3Myhzc0c9xO+BveK2y1!)jPM# z0PPQn`bG1plUp#$b%9nD$TX?9s;Rn^^(GQi36X6f%e9}* z2;W?59n?fovQI31{mBR|cP<;T_gSO@E7D=0pk^^w)=@(}o?mz;ck)=~Agm}QQQH57 z1m}q;P?+J;p@|+FH{bs0a-@=A zLtYc)+8z)R#xD6MNGt9aNCc?_BS7g+ewd>#A$IKnj`@Fytx z-yp94-O-;vZ1aE5d|kQnf1(Zk;Y7KZnHM|1f6~;XDX*v)WxG;~pPZWV_7+BsWZ5N) zLq`6{SI@*|LJeX+fBwuQQpgIw{^JvJY|Q<2&ru`lKn=gM7aGaN`rN}HekM=U@hjj*N>>FNAIsn^P{c3zxCEyTHiubm1XaJNd8lh_a-~!w&#FDXzj^P z86Az|3{0)hJ2w(SCqh_~;2sXu@hjHVh1~k*)u_V#06>ms(c4Sq&&M)8J7|bY@`$H7 zI23RmRPq2U=vjKlkO?y~72F({lhPFFEGT$}EZR1;a`J_2v_G1k$R_H?dGV{6{xhNf z+4T%F>MIWgZ0j&VcRRIoYAAj*4>w4Sui?COlew0z!qgGNN7K2%T`rU=LWGGm^>uxl zrrKUaz##q;X=w_NBXwehL>no)%HrjL_TEMHFgH;SUu|oo0DgvkCLNVce!@Lzkf-vG z?Q#{5e^=uB0x?Mu%Z)1ycRQbCn+mZHJ|vA(9vo&Qi61%Nq*hw`I2r93+N4%6QVcdOfcGN4u9)HKCg%%~i1k^XNrTz0E(vO=G4{^i zwt9d2TW~Hpm}*ts};%FpEmOaj8b&KnD$2>3`D(+hJ_tT!0i2n!QVY%317y35hu7z z+^+w6dX$&Q$R86+;-VeJwuvfA${B0lmW~_i%{WA_Z`FxUv1PB{ISM@?t*={eXlOvt z|MTg}J&1dI?CV8yGCNNM(#Me-9@QNC4zB^mKuf8wQxp8k+f(n9X*Mn4 z)6}gGo2?3`ji^CjsbqcROPn%Bb!KYH%gfzu$@)I}e?Q^WYVE(pP@rF}jTaA7YB~b^ z2$L~1H4e3&edHm+Iek8|{idl*g2ica+6Z?2)KpPB_8-pR)$`qO1L}){UpyIRICjUq zfMej-x_XcM6xf|)PK%k*s{%Im>pgf25d+>o`t5G&xO%JA(?Y#2d>h(UYm;k{7t9XU zb;UR2XD>--UMrs&c#w1*|Dprz6r9RuEAgkxzsB>qJotDk>gJp2A z8J}T#pYbDy2fbdgF^GCVKwSzU0>sVNJU0OJa%0R;Upvpg2T0%nn7YyZaED*G~m;Bj7%g=xG%yT_b;L#WAqyQM>HP+fmt1Y z)IU_xM)57PMl;xw%nY9|+nT z&RH(>7?G^Q!+c5J!uf@n(SZhC-`q!AWxWuVnk+JRKub^J@r!}${mB$LAk*UL)@7u^ zk&H>@yTu~o8YAh$pr|NqM??LSw4od2Iwc@eC2|TXhCX0i&Ao}zH@~VKruj_~nf%9l z+M>o|#ip2H4H=mDBi2u)FnCNPGk$P=?QKC%b*@FTjcj9V6$7vc_tu~`-<<3RvZmGS zCIZwPf#xj>KGMf*7M5{kPc$9hro%CNb|vw5O><_hwH>yK{Jmip=G%wN4S~SQ2Q16} z*%wIT?C1#)?kq4b_IndW;(RVHe)da?;m+p4MoK+MBs>&$i&)BwQ^Huu>(y+NFJ|1# zuh=B(aD%2p@_68-Z9d!T%F6bI>sh@vM+xT5PF;nS)4U!lAaHy;Im7cEpdsvXK|we1 zwp}!fi*Hbujjro`$-e?g#6?BjHiu3wXujlR%}&)GC`9Bo5lb4RXJm|WG00OmlV)XQ z@tK@YrsbpHVq)^_M~WC;1XI#{%Ed)`!?^(%a&vjuEzmQ1$i3U? zzdN_Hv^3NRCFS8!!z_MU9m&%^TI(YwCe~fwgZIA4TUw{lKyX#8p1F(YAGkWaEBCb2 z*6Mj-!r|xF9ClyGcDmZZg2+nDd8xH8Sps^xebHUwCv#cqIJnebpd0neF|F2lxjmN8 zIQm&EFyfaB27W=;$KuBg^Z4NTy54qZ5oidB^X#BWwbv@ zjMZI&i%Ur@pbj>bRg^fx+MN~sQ3mVw%kTxolc`#OH?d@=l{X>)g`uHp?ALBY&?f*SRALSqV=jE}HWkfkb@lQVGj zo0nb>j|rz2UlQWQAgIlp)I5->V}RBSu;`dQy%MvtmwI}Z{UGTh8Qe;}kc_pfN5PY0go^~FuM1=El+h)-mxjDI1f%U2?e^#$VCWch{T7gKa#7JR(`2>d`k z8_9H`=(hm}fPdmE>Djd{vFI}y&yCX4CzDl}e$5Pp^|du#tZi4F>-tE5GAoOuq=ZDh zaZN{oUQs>=bL!n9m#r~A_OESy>JHy%C_Ffm0^cbiWrD7rURH0=~%CWQyOOtWXGJbS||2B3=2K ze{gVc-scE{>UEjj!v*xE^?ns_1 z_69NT(pBG<8*kf6u-XcvX!JfoUmo=Xdnu2DDgsz<^vPC*Rj;_;<@r&6b7{+22dL2} zDJf~>j7>>L8}Ll%=4u?~c(Z`xJDFzL-aKAkOKXV27i~>9rK!%z-~IA7iKFJL`@Js= z7n_J|kcEap3K6#`Kr~kPsVY0d(>;A{6;?jAIVL+gmAxFH^eZI>zElV_R5K%TTC*nJ~!V_4VkSX(K!y{6rp3>eJG&cWwiJ$2PK zQ?cf|&^tKhHE-`eLB0a%$wGL74oCV|nYl)&w~^706e+or+Kip}Ld)I++kZxA>^mKn z;L`J}LaWGVXgrSA)P(X3Ja%Wq+>bkF@QFgFtB(PYAhpw&2Sj{}0RcA<2=wB`&+sO? z)_Nq^sZngu1B+OB+xUFfDLb?0o3vU(Yb%hN*8Af5Vogp*BMS?8E&eJ?2_78eiRGhW zqHzj^u4a+0^-8(zzjpkNhroq`PnvP_KDyilXFv(Awr-~fq{3|u&~J*w-OqPt>st%YxkylPsUnTS zeubj&{ryL)vv;|e@arR7kR~=|Yjpr`K%mmr=y${Uy2;^;Vm*(JJRgyDx-^q{0GUR3 zywK3~JcvAtK~8Zb9&?62nX)T!-yXmXy3b|EhmjQ^e2sTiUTy&xZ>gvxmX};4csNI+ z`D84!FIgP6+Li+d0vvZoSKUmSe7rV?O3Ljx~@`B0Fy0kRb;5@L(uFW>) z7dew500kBUs_Hr2U#={^$6?zJ{hDzs!kk0ng%M5(C1FgQ*)eU=8oEg<({)Q$(+KvE zIRhg`eivgWn+&*GL^PYs^T{bxD3-YoqNunJ-BYJxH(QzhYIP&nmANnIu-KC5nlC=q#?V>c^cC?K~<6SxoBHk zF}Ll3GShl@=;?>-wZ~|+(wqAsFQeOntLt0{?NYQydMs&RHUh~ zf6TWl05)ofu7{PsPm`nzs*m(k*)Gk!Y@0<|Ckrs%YZd%bUXf^uAU#aO8&ybd!Di(mOP#heFO zBg_7U$KLyM28UVXoqoM!kfmr|=ApLxHh)n8yFcQzu(%lSiA5u-?CyZoZUdJe;=S9- zsk-CZIy%Plbqmy!#k-W0=*CJg-~*Ofe1!=-G#S5$cgL0qE}!lgRoMl%J$drPeQ)mM zB3(|Yed-4gx%B4~RBE4W{@T_Z}Sq6~lJ1X?-OhvC(>8 zO0w88HZ9E@EY2z@C?cXi(D7|wjOPsL10e*YH3gF+3MD^v6)egoo=kpCb z`gZ|#vtoB9&$y=eJi0at?ja>9*;?tT3n|_YTZ(=cWlLhYyVmk6%dP)42f}^F2)L25 z0~u5Awv*3c!x*JkWO6Y-4w88HV4xx*r?bYm#u4WF*k&jZNf2`xq&2IX*tom9t9%OA z$~XLw|4x&J;&+H-19>i90A3vYf@O>s?rmT&5s{&A=l*7!7Y)tkHySfZe_X>Ui|D;fhgVX5xTxh_Ck&%(bphj->)LNDiw%)8pr}?Irfg~`y z$+@|?mX_y9c|L9rO+d_kEtt{oGNJ^rhZncqHEOM{BMaTm) zdQf5P_Zb;?_x1$zc*1n*sSXYR$ZBmTW(wv3P!SEM>8+>a8nw!Z+*%;+k{-l*KGhv# z)q6-i-Q)|+Gs>%KO<-kZZPU;+-*7*G0QJ-R_gB~npDM6ADN+h*76cRM$7%)zE7B)S zuxnP!1@OjFiuqvCkl6tWg7h9HX*?yEgv?*jm~wt+i$P=MVK;y#_4qc+;vs z$tJ%}H!C}Pcj_eYE;jEGwr=ZqbJ!e2&o_5BXRbn%wM-`cQIJBy#DoFl7P0fId4GJe zXYJjbXC@lw@o$ScO@L;&}Z}h!ObE((Fx0w9 zN{Z2#m(EZKN3W;)N3*x}M{#OM3!ZPTr!d^7qbuNzjgCHE+8F$0=IJ*D-GebGs;TJ? zU3YHAIG&*V)syc+b2C>~RxSa=1lH$NJ>LUZsu|yC@+#mCiS113L%Z12uZ5Xea zvt*m#xMP$r9}vEE)2umw3dIPwL>TW4E0h~;>BrPKSq zeG;tzzf2Ty8UxuaxtTS19*mp$Q*HX=E^>A)?%uoi@Gg)dW0TB^VC!?A2uX_#D=fG4 zPG}7z?Rlj9lcar(4CTpSi;Ihh)i(f01?z&hP(DpZ>-{{9U+BeLWDUPRa90|kleygg zq&SWZDnIh`^OM0}9&sfbX_5^~xNnn5R9p2k&NkEhE2u^Q@sEp3jw^`|K&)hS%zEdT zy=FR{d~hlry(KU^#DmIJOB99)1z8aYaS3R=O^D1*{O~bXrLB)=80j9PXZ1Wobv0x+#rQ`9gWsqV8b{vh=rmFKyn&q>;LZ^RHRVuW zI(F`YIi3v1%q`tcBM%psz~XRC)YI85|`E%&3E5-nuiYsf@?bxr$YCTRriDOSI4G{Q_WHlzV=dUXi7Wr8h5Oj1;XT z!O9x+af&oDZI8J(KLQ9;-sY&~wK1}zyMd407}=1je8R5|7f%&nW(G-m294fe6isV% zZ*^?GJXBfoV|Kiz3{R7k3nIJ$wAh#wQ*bPuifi5I(R?S%@fDtNF4ZSZ^iA?~m z#Xj|0x@Bj#r4k1Q;Bky>Ta}%@sfkJJA!5EF1O^i;#g_n(5GU~lJRHb0R;-W-gnRun zo=ihS!yO5?%YJ{fZa$kotr73!SiR{UCwgUVP5yHrPdDXies;$K0x}z8DChI>QtA;vQvVKFfM>iHV7wUwLlbLP@@3N>_-z zyXreL^e@Osl{R=VlB1WIldxM!>Q#FKMEMG2|y51e@ zyfY=lt{D*y*X^vVL|j%nPj@6ff4*iGgh=CM-K9+Qn_}az?D;lWZlPkK zbWj1=8v7L(E((yrAFvo9qI?!sR=bfxX7^_vE9B126kL2vUb35`&&Wio7^S>}8sT-G z2LL~&41TN(wH6Z-`wMy+bSrhPv5Jl_Wg-hG%<2G+1Zls_q1GPa#U&vD-J5%_Z-yxm z1+7lB5uZ)2r??iHTx{1>Y;JBMf5Okt&lQ-U_Q$~jwoX-G(vD^{3SiQ%5fTP|{`?Te zq5^y6$BN9Bw<1_U34fuQFT&EX5%1HV{iFhwM!R z%NSSLTD*%$5p&JXEi2gl$f1^?Bao*+Z3=#KKn`j5ah$3B9zw_lR4Jr?{qp6I zBh+JOv&m%@$ncdF6|R6H-=@K?DS1{6WNq8gd@!*x(~YjKt{Ne1+DM7nqr+g&t4?Eu z@h!Am+W_1D=FOX>aF()n9bpVHfws?02Gi#}S0WTzjdL_oh0nJZ)_|AtIw;zd)taJm|8)=c4?%*-iOP%ixEhw{G2H(EwI4$Mk!;W`;aVhMP)BK|#TJ zy}as%UJkflt{5)3vAmqo`>bVnSbMucJcE^s%VxgGmqidLvCtFd;V{)#^A9Nw0yX0J zO@SIDr_avaI7y<+=-@1D;<3;PzjAF1cKgdjBh8Sva3u7t^Pt;R+XsRYdv(NR&A1Gb z=UZ=`W(%{i01xNKH2iM(rr;gEOqI*k^He{q>@7*RjkJVJX2-(c)%Kxs(sdyvv%KBz zO&c4)`jk|8r@I_|dv^Hb2OJ~}cPK>wWxN#v4|ePb+X2FEiV(Eg{gshsn-qE@L(r=C zZjC;>oaR&@8A1Bb`Em=_MF&`TGiDBIIin|brcFlG252DtsS+Z-XpCt+WEdm?uIUJ+ z6JWgMzB|F6;vfq@<$59cCu$wK|LzZbvLWevqHhDHs7xf7ucCJL**lJ4^+k z72<_}8r)rJJB0q-5t0X?0&l#lz#OY%NtGV0p72y@X0Q@tg7XnvHTV}4{LvgQ*HM1c zOmdYv+0AM<^^5uFmU3UGmzCWMI+4IQ#-ePF82KWjs%*8?$F8SI*=kIBEJoSQA}6W- zD<$0AL_x4-p58xu-rOk*DbcLJF0$f&4dPU`VU{oc9+^Ht<{F{pb4FH>7p1zCjL zxXu97@q5EHC5QM8uqdE!jfKyZ4&5nyz@eoTSrN=AYZBB<&ndmTVSsQSdtyCQY18O! ziK{93z@uOCIpq!&-QoVePL!LdXeT~WaN~%ncF)?5%~YP^O;>t)dJ4VhGn;&UkU7Xl zlQvg9sh5E8)XKv06t!yp&Ma3C=;EI?7cfP5pTH9`2Z7?YJ@kH_;Qg2a9h;fjh+KS$ znLaD;^3oE(1)?|1!>Njy6Kkybch%MflE#ae9UL5p_TFofN*%v4^*959#$98p?swMU zswLKBad+FP%De=fy8vBGOh+A}!s7YV-k{MNA|N2(CEs z^s3FiH$li8%>N~@`!N~vi9E!59v2bo&FrkMTOe#Vr+UT`@Ir^n7k#`r@col;f-Kp( zk+byE2G8Sw^O4Qr9PH&~2uZNzvuM?Wqtl&fwJa77-YZy zRsl8DU$DS>SXK2 zeAS~@p2?gVDbYNOj8d%ud70K#_JczU#iqYEGO`*sTw*5YRFH(>3Oc0sTzEeSZ9LBa zQDSm8teu@&S*WY~$o_R`Wpb_I3EiA4PIVNd!&cr_q zpaCJ9p%rX=MRX8QBSZ^O3jp&jUAzEd>tyTj+w))=zth8SgNTv?b2Acg_wBH+y8ANs z5AqVZfDW$~87K7S7FG9r>gjGEQ%`+NLViW0Cvve~ey4G>&1di7PB&;g4zj)^WGT-fJ%mXVPQER=J zgMi_a#Itw{d4O9-#l-Re(Pqjiax))gQl%WjeJg?A>ODpWknqlSrq6)lrNpp8UX7cD zJocGR(dbh%|Nh=CT`g5+)DE!Swzs$SJHm&F-N{ST86-N*7S&X`dqyw<&4GbA8MwGe z2YdVID@~P^#>z-ywdU)HSyOAX-+`d;x4$9If`ME!o~NylJQx=dK~7CwZ8NL^Gv`DiK#YJ~d22LL`gA?Fv(UPpJ3 zk)8m5qtlSZciF~cGNLAs3zVc#%g(RI$Hz7E-gc-FE`X&GKz%RRm*Y+mi%}4wI6L}F z4_J=HUxCfXLn^<%5Ci-BUtdP5T|5nbI6aH)X62It&Q zOG}&g!|VV6o?ZHe9UyF|B)K_z-S-Wlgaj$lXrM)$@*CYA-WCu*hCg}N{{3a(h-pvF zn_o9HKLp9Jw|*JP3eN7OZUee9_?{>fdf{7u)xV~~@e^E4s~dbX>ktemf30t#o8LNax-)YjHgg2Hn@wDH3VYrC=KCaO(G zKr(+xGhF}FZAP@gZ--m{s-c#kYw9!^23kwsSq}7kP*3OIvaJ~g%zz3kf=ZC(tfPaYtdT0O#RbF z){yk)d#|cjPfDloN&H8SzqdIXCueZCZ|eVcsf4UrLDk*=DA8f6yUPu&0@Ak(!K)3KtBzi&-#v%O4GRy zz~nHubjX1L{~%76r#!@W2!)2-*k+)oXUgAa)V^JyS&==_f8Ko{&+3HCP6!Sb*S#A- zg)lU#%9jL32L}f`rVFQ5X=!VRGe{Was51~Zm1?XK7Iwx() z$ypzBm<}fl3>p_`M$~G!L$L5 zBY_~&h}YKsS!f`mi(7s=a0{rYsN~zqqVblN@9GY$EzkONlS8F%&GS(5? zc-szm%$ES&95qgRb3ibn6JLo>mhd=CJ}h^3bK3^U2f(VbH*Q?N&h^|J0R*hp7>nrB za@4^0muI`PZTVVST7W)P2{gwnEG#Tx2nN{u>IZp@U>2HIZq_fBLI5;vPgzj|=0Uk| zP-y&=mX=-;h@P35+24P+Lk_4O02NSTTx~Drcj*bJQg@HdD;0yWk1gtfmI@7J<-020 zUS9F+U*#SdWx#y|6ChM5AMjWO9XPywZr8wUHzLbXmz6fsO=5kL`0+IBp6!F+2QJe& z9!?TasMybE!%iJ#<{d9yd<=D$Pq&E0lz-4F(vDVzAg^W1e7<+`%EtRHUw<{SF7@L% zf!=09)_wsH9haPCO^Y{jZc+$2&UlXIkN#2!34+y$0{Vlr#xV$S%(Ii*aa-Hb>NW+F zrimTU{Fs|`Vfk<2rzjH7>Pp&p>^d{Gu=02(Kl$6pInV@mc*w?{-iE>sFP-!tzRj!W zz&$c@goMmfPlbhrebKm%deeI2ygZ;)$~J!uza2H_jaj~)df(T}tBV!$*3!}sKbsG* zcsBFje+4q}8PuUMPI|?-^QeB1_XQP{!uyo}fyYhlgO`DnQ_9KWjzV^$%Bl^H`;Gne z8Pm(*Tb7$qK%oaK31KKn%w4G{2C$g9Ik;h3%4kGi-6?yf0kRbvtkPwsc%7Gik@qY@ z*kKyjayabS!Pa=F2e%%j{`h?+-^&A1nLxCaxcK(kH5)Tt3E#S$x zT7m$FSu`d+fCvy;%9pmBNa$}$dh>4Ns3l6{m%3xKZm~SY4&cOm#o5?u{}*#_85ZTb z_KoARmLevCfPzZ5w6rK8-5@Z4(jXy2w@VfZ0wOUWE!`kPw*d$)tINE=!0&*%iV-U70)(vV{Jk2pjG#-?hJ0H4$Q$P4 zjg60K3eT0jLOOpDyq1_ma+yQCFdJlCMkPi#E47M%5o#_^$&9vm7wd6qC{$u%?1naX zNZxUc;>FpJ@%JF(s!=&JdaYhWR5a0JrQ+1-b3J)_0>Z+=yu6P1u^NWAMgF+J%4A?B z*pmRwg7))I%$-4;b|y4Gn2f3O1ZEiow-d zf7V#yb+7UNZdNQ)i4^;Lq{Ig0AnphBqE8hP_01gB|C5+Zqt|gLqp7)-%iShsEX~kK z8)}L1(W5!&yys4&hzI!ESRp4lPfrQM)O2x|nD@vvA-I>7l-qn#Nm3E*M7Fg z3m**I)lt~^GAoR@XA24&R@`^l&S!{zg2JFcVCD@}y6RkxY+6K2%>GXrpYn=|);PW# zprD3MslT_9X35of7l}e`&8>{m^G|F3T`yUo_lJ}8&*-M7Yen`cjhM1X6q~5e@#C06 zHbr^)z<_`(!#eNQ80ojH=&N$|&Dgq6c@?@kC<=Idn(^^6!IY-vc7)NL@qqOdI+kln z(m!vrWpE>Wll?2J`eQfy^u%-Z`T?Cw^x4~Mp{OYSd8WXJ19M@x48R6L}sj(T??1n{ijWY z>o4*^Q*(1xA2AIkRY`#M;YqMw?gnO;%8W5#5_Q3Ey8~urZ)$QtoX@YqpAlpBmv28* z(O^YRY6fK3`z~8C*EHO^ib}m}Xv+{}spdVY%>#NVkNaXNd9go$$Pz^aok1fiP)Bk2 zG|@S{;@!aK=H@#vFLk1$0oRh0l)NqIF4;nMV^8Dw0Qn-lKz-qO?y$lElgm*U7D&Ez z{FZ(0Nt6Dt)*ek{eg1p#G{AElQc}f}mzS3%w;LPlNBSIpq{xeKy_b=3YUE9xz0Etn zKvFAy`PPHgndWdnFUbSa$J-)Khx{4xv?goE?$iyG__!m5=)VzBPV=NfwsU{6>Dr9m z9$Q%y+6mB|GH;hgVJRBcT{;RseF|I*`Y>E($4cNW(9@CLXVb~mROBP zReug68V7R9akkdFVl_3nW6p}f2K`D>YBGj4rAe-dlBwxwDQ~iF8G%aA|Cs>?76w(VB%HS7-+aILFts`f-%^{JhC=+o%fxTr-!v_L`SQhinL1CWC_TpHj}$K6n=&FYg2Dq} zADFfK5c;-Wu0Ff|Vx23dX+jRSIiC`;h3^;VzHfw`Wyd)6WIM3}!Auz0$l$MRKwun5 z_$|9qJwm7fcZ!aNOB{OBNT9yZ|H!e!aWm7A zFg=q>F3La__@VFR5P20l^{$l7VY@hiX-`MQ{!cLNkm@jA7JTra<8C{O%TzM?TXwxP z@fG5M@vsh=5h;-MZF`Dbk4TTEzYC{;W+A#+9IRFZ8C~y248JRBBgUL-YC6I&Ni|eu++#$>}5&#*8jcUezDum90JE9GB53!t@=Ov^zHvBPz2?JOqp3eHv9?;3nvJ; zv<>l%8-55>KW2}OigE!w!s%tdq_kuYG)kIF{FpGCp<mR%VM3ydFM`6S2Q9qb-gP< z7TRvHjMn}{O?uqF|6Os9vw|X^S`;u7X=&*>4t>^}T;E-sot>?%gC5@2)}H?1ae9Z= zX6F6wzqF;9KsDhN64)5X*mhmlXIwrIj>@)uc3|je!>yVQi9ct^(ISOc8i`L+G?}5sF<8);j79mj6e%`l*Y##tl4X@2N)G z?g$sZV|L!!Xb7yg@tO!%hO(A9V9J2-eRXnbZZ41eO^-r@`5yiL&b!KcJV#T5gTY`ovZA4)oeiukIW01)s^PPakn> z6WD&_xL-qJaxGg(_zF;7o}QjyMS$^A;&MAh&dHEJbYV%uJ>kY#-4U9St<>^YrFP>S zD@jwF7J!Pkz57^Iny2QawGbpZ5@tKXRIcd1ez@7~+rT~%TcB+{h}OCAP~CsY5&S4} z@eCR{TFc@``xzN&UfT;9$zj)GuLexbRXCUf(R$kBAdEPveC!I1SZbBLkpsw=uibE3 z%#=uD7;BR>T?hUSLmRwe8>nzucV4 zlAuze8!ZowjXjw96y?6RNrv^00EvXxZimD&H*+&Hh=O7KjTSF3X0RxAk0&Mtg?sVP zn}Xsg;DQZ?P~<|h$`?;K{nMd{o{sL#pVy6~Ki(@8;Crd5`ED>dPd6=3wTgYYz`0Yc z#3?5;?@z9jfV)O#u1Q|K|9HEcV2Av05W*&rdM|NjX?xSHc200ZuFcK1c1MsxN=nMB z&b1TnR2m{%gZYNKx~X29)1v1?>+;+OQ4Ob+<71AN1)P?8)pfHw*`}t9z@+0r_;#c$ zPL;T@Ibh>fCBC9NSUeG-Hz2kH*gt?vxoi=5XO(x z9&P1Dm*Q+3#HqgWxF?Sz_i*bt?B&ast#C=!zD-Ygj*AsacnF04gAH2k0a}^MKB6zR zl&(}f;OBSSXcc&O=}xw1uK}Xc(Q?E*(V*J3v!kQqTM{}TOZKtncO3Q}uv*>bE%PH4 zfn6=_?RHHQJLfLldgxjICw+3!t8Z^mtlA0*+-Z7cD{^%7^tcQh*5}YZI=6mks1mW? zbV}g+4_ujrD0!5d!yI$}wTjIk1;gRPv&7uNkG=qW)y}A-pr8QYAgfNcMvL!xpuVU? z_oqR}@oS*acmYi_I@(=(%xa=XJlhT>M6jQql;6dhEvEqTnH6 zpp_0i!?+WoACCE<%wJd?Zcc-0 zoo+MVcb8P>M9n#fITD`wtTc|@yR01b_WI4lE~^FH>(yQN()3fbw|nQ+Rom3weYnF@ zqw~0JI~&DH&VlFv*GoI2B0mpL73?Vy%;6!o#n>EB8w_^pO=(y68286Ke8qQHpO{L7 zZvB*)Sy$Gd!O>)|D=)RhH(@kc+4f; zTYG}Q2m{JEV=gz=om*ypu)Q44!{&nG);{Js**3k;GCv`{MMcFrI0*{vQn+Kp)qoprahkZK_*lo7b|wv!kuXOP5cV z++0&u9I;a-;d>6?RrQz)rgONc7hqQEQ8@>kCz{Yz(P6>69xK_$uoR)$JO)BaKwS)N zqVAZsTH#dD)H)-riboCi&7XLmZN}3KzDG8p8{m30p=SnrRkTv>VVI!%X67MlFJI(z zm_8DTT;JS`uNl0+QA8yl!{SXQPTTPZf~mb6^2o`yf`l7{eai6{l?j|5fx~3 z?)@d(8T*{5P5~TxO#liSkD$B1w*Z71^UL({%Eqia3Q_R&sf< zxJc=pbPqS1I58XDQV-P51_OK1*!@UWZ4S<#|MHE$&;8EFAuqvz{&Q!4i?FjCXHc+{ zuhyWt_o?a`S5bCv_SSO0PoKcJnS#l+o2;7qQ#m@L9-GCbrL_(R+rg`=sr8cD0%e7X zo_%Q4zF>K1Xy@y{eD|yOBD)yY+wRN>wIv28PE4#e$-E2EM4L2oC3^0Ed-ucSC&-3T zHoXP-wMHsC7McQaVZpA-t;Ob;A}HRTUnvjA*lUPJ%1aG)oJGUC%l(VvFyoQE2ujX^ErOgmp9y$CJ`_^I4HXn$XPoA z;GTadEzdEo7ODz}fZTI!`Gi5SV)5pCQsm_1!xauuG*O)Rfx_^i`3uc%3^ztZE+P91QOZKN~fg;c`|GP&fk9+B*N@A#MIu#HRIC;W1WQ5Ace5Zdl`paEX5jSL5-G-aOjbz+z7!N}X0G%UtZ!~! zz!sRb#ceq?N|N2fTf=Pc&m*`^e3rmO6`>|aq(9s)aUR_d3SbM-gf5<0J)-W9{;*r%<{z25iDGOhY8)qBuld zRz%!hf;r6th?~7KN8$U)mfh*ZWNe;0H9HY$3AzTyNf?jOjP}?MA2c}(%0FAzfI@to zwy$XJCFY1O!4Gu9hgzV^5OP|K(2z&5^aA$fRuFiTm7(q%bf=~)E*aF+)BrtclZ6V; z!5we5iv+AM@YK@YWYuzXafw1PreonG5CFvNk`Iymyu1;7P3|-K2FF}Y z={yW*k;vR6^Y#R=_aT3B#D3JYEYTf|2BC+D{`T8knLg%&NPA$c0q4TKF zUVyru$z1{xJ@R*4`+?%qzie3XSd6%Jg>+N<#L>ZyY1E8Bz#*L+B410t(s6IPT^y%- zx9(Ad$LT#&KF9e3`C7ug>F^t-B#FUP1y%2v%{@Ak8=kzqq^X4x)^USeC){NoZf>=~ zcB|f;_Bg(mn20QjaB*Z!oRbEXsJlFE-4l~C)aj_<9)!32{LE3cEH<~y-|gSTtBT3Z zWP6>94@1wwo?mw~-tT;qrP|*(vc92#9r1wk(0w;!x^ZwQd;C-@9Rq{(&`98W%7cZ6 zqlq8?nwu1-h&rpBardX`w;QUp^PX!{erJj6a#WJuP_kdGxYDzBeWRAN+pD$LV~v?! zkwmN7wttklAexuTcZIc`5zWgtFi>WM{dKGnM6UE#F!oJU0yjsX?R*qL?nlEtnS#ay zYJEQSY)pm!-(up1#D9ixiUhE6p2rzc-q?74{Kn7Od2{|>w(`~7i`a5DIy7lD8iy&d z?%h{VQR!H)PKYvW(yzUZy!Wu&wx-e)8=?6`=l0Id6)~S$r(tJ0G|{6;^4qsPXXZme zLp7X!MJ42q4)ptqovZJl4p3cV7VSe=mi$6<2T5?G{>2;Ki1ue=3Jh=hnyNjsPyC0K%6&{0)df2AjBv#VLd_n& zsp+YU%MQ%09Myka{R9PRviYS+@oSWnLUwJ|SCrcYU~lL;+oWgp{I%@|3V;YHNEuT@goTo6>LPSxx4HT z=6LzG;jeysF|omg{Py#s7phk)*xtT<%N-mTteH);%B|{M z!Cd!ym`r=-U8D|+cK&^JJA6%Q5s_P*1Hy1B&E}}y2N2B7? z-y)%2d#XPA@9*%{cp-=I+g%_cI(`Yy_qW&QFL(c6Z-WFpmET?$iIM-69jWr{W!NJWGG_n+GTulIiNzdyVYMhm$z$Psnj zZQ0n^dj9QhJ$X~oyYNdQlN6#Z)Bk+NUDp5S%l-es9s9psf`NBzSJut{d^4u{lbI70 z#}Vk~_}dNlqdA!k+}Mb^$G>HjzJ8ql3WrA2#?o>8GP3N*7a?Z2`Z$=@jKljc%nJ8j}}S*mJ8iQ6Zw((Px=EIUiAa0IuS-f5yPDH)lZ zv^ZQsQ5WcuOCorPiHS@8of~sUW{4q9> zE3JGZ?AdhD$7V+3Y=+eIF=Ykn!Q7tJMlIOtdgxDmZt}3^kT*DI=w>ZM{QEG!IOY?M zMRUy`&npUSZCz?1A*r2zK8hZp58M#l3QIQe*)(yPNi*agT4=8kU{xNXrU;J<%Z@@P zm)T%hW*66qRElv4*sz_9KXD0dkAE9quixy@%%EF@UFViWUGfxyYZom>)12*n2OiaQ zkCrs6QsyYRYm%pzJQPRZx1IAE)uz|A|4!QZ*I46>dh^d?q^MaiV^&XrOOB15&!9)g;nTsaqiiCp7%R=uNo+TYF2P`awOEqOck`N#;{sa%<7SxB!a{~=A(MSx z#PlB}$D$@A=!xR(w5^hJ#FwR6mRe+>y+7~iDMle&x|E|oGK#>_`z>^*@9piSrIK#< zM3)Z$8uUTLb#-Fmn=kB?R=+3kO-QN3k{X3)$Xlzjv$g0*b6zXnKk{{7W9+`Jvsq$U z1?(I-D%V9o7H@zy&`36LM32>^7F(*lXQ&Bg;8(%-@$2N9>pM8m`ud$uHu1AuYQCbI z*Ec>+2p%g4xh-hao-hgsfgIwZfdSRKOWt+^?`Rg^_^EU86oGA-3@{eiaEA2s^#996 zQTZmf{xj5VKFbzljK~iohjv-r-Pf2c1K2DJHow=r3t;P%b-%zYN0lcXTk4;Xz$TXB zt7&h18ZXba!0N80o97H_8)GqQFEDLJ=IQ1U_Q=PL3Hy6AQT?GQwNNF0>+M}0;2NK# zA*pg*ORiLyOzE_6uQ}eYJrVbKY`c6ZOh$mfvJC8G$E`3~*4KXm+DgY$}VW(9fG<1((P5b*5%#*5cV8y=`zTKx} zEqWKr^Q9-JT1mN1Qg;wx*xvZGpg3t?4Z~f+-@_xIAaBKm?r_k}Gq;GYO*LTV@iO*0 z(x2OSfrtdLAOxTTb-KCSkIyR`?=vkl0a*yYV^hc5V9GnfvsDL)K0Qpoe{g~?e08oK zi#Tj__Dw&_JEHH{-S^2XP32G5guPvqXU6t)skA=}kNmFoGTgv^Zqu~OVsn#4kAh-h z?nlBqgW6vno0&B=UI1gE6-`v1J<|nwgyKbynw_yapOeT9@J8XgD%77o_3yfylGqRV z{|@@KhcbCW#X$yER*P%X$BEwDrK&&9amz>Q*H~$4Xhd*p-|Ia-+HaP>XxA_^timoepqpHX1fZ8*uvy{GKz4o6E;y1MV~?PcBj?LNHr)Pw>F<_g99%f&&; z9H%?oLhjpFx^@1RP9f{E=t|%)f3X^vCTbPts9a)||dsa$Xy0W4w zid|2O1osufLy$O3)Dw>JN+)bzUnwa3!72q}S`~9~N5ZjJ;%PBxD_djBI#OwrjEts2 zu#q+5j+r4L$~YEHD`8=A*VX68higcef5P$o2gE2_lxkVcjAS>IdR~+%C zpui58hW$f=%j$&0;2y}|pFPW_?K8P~`?iS5st39PdfGupfL=~;!+4!C=e)2~wK7|M zR@R*EjMK)($3Ahf!1tP-pE(3s1*^n0Dk>`E9y%q(xZ(ZH-robqAJwvaw$~{hDqhn- zzX-b*kDGkNV5w%>&$g9)oM)~j6KwLi*NUdu!;J+U*1?*jmbt7k44uP;*;I|@FaZ3) z-wOF(+z%dro;^A#Y4TL!>B7Q7t3XE}kN`l?m~K=|Ez~f*0aWuYl=Xm9)S6;J!0Q8+ zyw+KEJ#i|5m65?B&>qMuC}@yr03dU`_LBxC@L1XS&vwEtcwRK2a|g*a3^&lAWe+jw zZ`OF!*4Cm?(!o?YNo*!|84A&C9_7!!eGfK>HR-N{bOpT6q0aE&UPfZ%`LkzbW7nOX zd$YC0AwZ;6uPh?19*`(sr9^%nfx1^~M$ZQL__8xxMA_JO58br2wO_t``Cse&#{#Hv z8f4HOB=~$!&xbPiE@u0KZYYBNHtMPcup|AGlJ*+{fQfARt9iO3ZV-4*}lgCb>6HomN zVZA1`>PP%D@q0EE{c27se_DQ%){=Lwe_!U8$H}Jxd_WRhpNiYnKSDWPB<)PYpo2L$pY=ya9NegH{5Flg7BRti2~$%$ zZ70Hgu9;S0asKAZm~EANdD5Ix%AA@h=3CxM0c>3?4@B+>^8L8Wah8ee9y@Ku-+q4Y zGGx+8!$D9%BZ>q_EDJ_}Tzxx!9Hz+dl!f~wt@2{NMBpyvRCM%yRm zH>0zKjsApeiCzlO7(Z#=B z&qlGIf4bxfco@`j?xw?7e|UgliJql~l1Y5!p6+ zqY8QA>=1|)78$AbRI%BufLcybHXtN0{4rNi@v;?f%t-{(zyO#iAM!IW)C3s3dbJAD zOubSoO`vw@k@TDI6*81#XIExp7C;B;!p+JsE8+e`6J{+I#hRf-oed!*KhBYy{^NsT zO;{=aHCUzb__@W!SwvOo^=3FKgysJIDGUW*FXR_ELwOP4b~S~rKqE`7p}Cov ziHQl=lWcBLxAngr?wAi!1SCSt8MN8gwW&ok+8Y}B`uJ`B-KqZTMzl+u!l8MK1r(nu zZdIgyU<(T-4K|3=;*?e96!bzw)_b^f{k51YbElk?JSwiMu#uKFUgYfQ)2Cl96{m(1 zlfOaben5-_zrRRAk_S)`hcbi->_(Xs^96Gn-vlh)$@RDv)f87X_Z6CtZ;$a1#-Sc=qfW(~}|- zFBGStfw-4Ail3ar;1aPs72Pagz)_rrwxDN-qtO=Fxf*(Wmrk$+QYJ8AP_t|$#Y1NVLyee}0-M(Rb?#%m^6zudu3xm~JZJ0-7iRDMm3P?FD zOSFSn!W)mHeY4qF5Po02az*=z;GdT+UAk}q#`fFb;Na_AJUlOl^Yu+kX1@8JPjd&I zHu#|VZAa?CEn)e2W3U{MkAmiH`j;wHxNOzwMzk*M9y|DPki4#s&9Q4<$S!i|`|=_s z4eQ-ftcv4v=wqY5+4%MAXNbrsperdUsW&COOcVEh+V-_U zo;*jhz+kLRN$2;fe8P?CGlGmOOL0M?Cc<8i_WKHbPErs&#VksjFlkO%!<^KjTPEJG zKd?P?>R%?c!T2+1SGXr+bu_%!1~B@`4|4&62m4-bbR!?#@a z2474*Kp>#?76o&}V39dw_sq@AL|6`l2{|o&ijE}|9VCMBIx{K5$i$?l(kZt7?=#~m zwcg%@w|;&lHvJsNT?$&T7$dXwh3V+(fBuxoX|;TC8xAHwsDWE|4(A_L?UR?2_5D?E+l?xW*xpF0>lg5Y55AsI6)|&@lds#SV z5fB`_wY`13>U*Wsc2ups!E`|82hU~9eHhsQafg2T_yr#RNrh(JD@WlkmEjJt>%xu^ zI)+|bn*Z8M8Dn>M1`5T!3Ho_+60m}UsOH0+jnO)vIv{Fc8(a*!eT#wuWU!;(zfZlx zM_CWepgw)dZ0qQ*f=htVDX5}*D-#k53++Xt+<`0x8%;@ZafD44Pgl47)Q#)Mdvl2} zbNH=#o~3kV^8B`96D9aU|91_}j}^1gWZ}M-HTgX!TH3rbUGzDV4T4NDtd?Ij#bxQs zScP4$+X5|`_h@A(ez(3|H_A$<0TDcVK3IBVF-ITBwKCWi#yoavx6EjiL-to}^T8Zl z-25p-=KQ$?tKs}nki-~AjZQmdYvwI3E|z)?mx4(E)^`)REnw}bxg9NjNC(5d@@JcP{C&|uYWda4pWTf+t}F=NAA7?oRE!;=ZbDPs;Fq=Q(+1?0zx-jUCbJI0GBJdgQW0Fb)%-`IHv2)+!IXNi?An8qxNO{eZqCp< zUeH0Saq~kE8yovcQV$IcC;^tO!}b5gNL6#z#ppmr4H$9bFJG#@bFH)3vgj-^SuI#a ztx`lj##;}iX8qC&xK58i==WOWn}mY?=x}W=QPiW>%)|teupC!@>FE6ENWydO9iIm_ zs6=0=Y8{8WnyS*CX3&^HF6#j$A~V=7boJ&4&A+3HO;1yL#S|EsA3;wlV^^70&?_Lo zeVCl$)Qw3`yWVbbd9#aQEL^{fhbgIls=cbQMIdvV+UgTT(*=i=ge_^=$-hO>KHyFEQU6B~2vZ0x30(bS($ zO-SvG&!GJ?cO%Z8rzNK@*Bcq=sDCv6%&@M=rinBq$z6B3c*a!k`<*Z~_31Em<1n?i z0_=jx0h+Svr=4VpnAU&8di^9lJr9BBB8(L`fB$au?K>V)zEav;6TpGtty_C|fRDOG zN$I_Z)3$-kC37NI=0o!ew)eoAS93O4i<1{^9G<21{Np#vc6X`=&nKWx zjkrrh#Pk5*0VrG2wUB*D#yeNKc7llQaal#hnpczGXX^-JuB%@`+C^_>pMc7^FY(L^0;t!OPbE`F)S@XN`RuoA5UNl)-?uKN#GBO(};qr39-To;!n3jMQ$xUztx8gy|UMeS73 zk!W7mkpwM+AMrOq^a{Y-JQ2rTP>TvpqkS0!#+fw~Hfbft2jLK&tz`c>@nB>WcW` zXPKQ9_$o@WvY=N<@~mk1!{8xdt(=YMb;j3nwu+_*57=r!h8maN8pEu4=L9*PsJnUf z)nQX8t(BBA9=r=ac0+W2pnr4YxA=UL&0#V)>`%3_aYL}MuwETHcy>wmM~OQ9)c~dmT-K&vOlN5S#t_HrI7k6XZp~y zM6U^mM~}XO(=qM2on4)KYs{d-EQ|KyB&~F=PL(Y#yO(o*=LHmS%sRTys@J4x^xJja zYPR}@IW6vE1Jg`%A}bxl%zh1&EcMxqp-Wuw0#L@7u5d!$!*D?zch{MZGAH=8ff5u^ zr_WuApsI(x?HF=#ltrCs{uvfh2+4YO+(A9;WclEq6CejAQbL>s!5D=tk)XIdA@(oXC^iSw}bm(Iib8=ES$efjbqTzdrhj8{S?vb za>&(H64hE5gM1s1AxmS7eSZA-;Zf74&QY))R$J61&CvlGu|1Pt&njl+5DRDJ#f9$O zld!x?ZeI~mW;x0Nb6CTa#ndtjF;1Vn6k=W};2#o__|X1wW8}rbVIai4c&$Gn^P;jM zuI%n+76I{Ml$vt1Tt-mmNS7z$Y}>ur0U0-r7KHcpX;F75D40WoXne8sP5^yCf>fji z6`%F)0;6%7SNfV1_cjLQkMy`JQK5jfd0kgWpsMk}lDtNi-L)O7z8AlHb4O#3TF7=%|qAzE)D|bN==Zz@+RirGg zxlbg7)MIn$LVFLs*8mDqIYC6w{yXG?ieZ6xdVRkM8V89VZ!r(50X+dMSq^axxwpRX zxm#^@b(3jr-hl*cZ>Y6D*!yvH{MNS~K7XAnY5CbdI5=MPU>I(l3vAw9()W?gV_zoQ zA(zhsk}@vs5fi=s5Uy?m2op0@_ikI+n)dega)$wu<5dFI1MiG~20@uzhH#)y-h7ir zbJPngy_1e*WJszx`_WQoiu4^Z&!-iASPcwVeYJr|SLt~wQw2I`KC8Y6bX0#M#BMRK z)z{aP_}D^3AguhiL`?iqDAaZB`o>02xC!+arcbi9d+Y1}23+UL?FX;0k?d}G;Y1-P ztCpWBoQOXosb2qGUo08?`t`KFp_ir*B;p=mh3M~VTP-AAq2S)d*9nNq7a$@>D;#9- z{T{7};>z1P6b4GR;&i)XRrt1qBb!nIYIKD4&Qa2n6Cq$=6UUv^wLOc0cW?{T^+bFD zjuEiw(qP7BJ(ZISd+a@WXJN(7q#eaWK}NPcjWz|LJ<1`c_>_UztuR}WiG4*;yZa7=jD&!S03w5O z@MP?e3lg>dvozS}h`wy=U_=uxhp5 z4Mb{Y-dnveUxR`b_jhPkAq)fU|Ems~+Lj~STw;Hoe(cut4*cG;?A0V#)M%GYSusDa zq1(}Odtr`x%tu%2N1K{bH~(5$N4jhd=&=o!?QP6rL_==>I>7tfT^cNO=-*wtG4mYO zXL3&6)C0ORu9GKhn`^WsZ;ZBqTN?CjY6f-Q`YI|PueV3m)*dev)YY27Im-r1nn{oS z!sz9}EU)zGfyCarPRByGy^ig(U`2)5w%C3s>!9Qhp*WD8YtyG=@**-c9H-!^y|uYg zMMBB@v5KIYBE!0@obacLZRm{~H*n5^U@EY!HE)j|&enWcaUO-XrH6hXf~s}X4+_Rg z=n{sDEoskISR{zW^Ir^{@^;+h8ZW-o*%%zllxV&nDJ`9DrS9{rQiEe2c`p*X8HTUZ zwmo-vbmTPtfm0;NFeP6uV` zF5u9d`lSk$3pflaoV5UfJ4vpwkIvbQYHe*DUFwI>D^At>%fk1#QRrNWH{9~~-J&3z z%d@}Qs@LiyR}G2<$BnciPJJggW}A}=#nkgs@Cpt!x#kIN-_k%x)2b8hh zZWD9_yZJTF`Gu z!?S%OdI^swfLM<3ly;)wgvFkqH#aleUa7)^F$UARv@^~bN~Pl@O@>iSQs{NBMB=P@ zMh(sATq0p=YhuvPA@M2#*`2O9I?{zMz1i006oHZ)t;)DWeT$5Yp$xA!4@GsDXx>Lu zk3$yY*$SOH@1x4=XNW#Cz3eG3T$ky ztna}C-VWt)OwstA<^TX~5=jyIqq?OQ(e?wnZrF&O`IhLrwK=}zq#$#69c6e^=BLa4 zcKG+uAMeU+ha(TX^sKF|AAMZ|`%O#So+jwkJ?h3-;m!(i22%FEG^rNWSpV$j`^2>9 zlFlSO5D_|EYI&*Q9)_j^sD?nYUKbbar@P`_L%eB z36PPJrZe=Nh-e}Sl(iv=59~K0^G=^9QERji^d9j62P7CHH7G^2fL)RaxxHt(x6T~S zxaBpsEOGig6FJV)+{9!P!@3N@u9<`vjhmJ=21^z0y)2Cb=ZQ$q0tQ?F6sc>|#~@Ce z-hCm>+jqvPL?kal=Hfd<1P-|;ohTN~E$#@-dhD98R06?A_C9)m=68PjyRpXK{ z2`eir$jpUZm5fuD*QkC~z!bRngo@mXHvW4UB|*OI5px4blp4u?$2rJlkc{Y7S@ShspD=;Ho}BqOt{ z?wmW?5pUW>+S_ZIXLMWeLLX*3@~o9J3-Fo#{86--Hxu&xTzp=h%^Dj`69i=1H%)xg zH(ch1;vc**BCZo(q3f5U#1xfZcb8(GuXT?Jg_4t7S61uX{<&y1=k-vn96*3m)tqyu zEUc_p{LJfwE1?Ynxnl3d)vkM>T2r9|OxbgF;v0NXn0z2{?3%Znd20|KrDRJFmGJ|Y zl{hF_p`mY$7PiCo*@%gqcg7Dc{X)ptMEF?SU|Idz#S>@dd_;z$);m;S+ycjw^Acnf z7~{8~F>Fs1w==UfsH>Hs7B`bq4CjBK2e2Q2wBXgjL@8I12@47?@0+`%%YRwd1+{x~ z7=3pXFdl=M;2)pwOv%g5O}%}~6(}RQ0CQG)dei2x2Jr$zPMmg?vwpQ}^N0AV%hW=L zKNG2t`yO3I;Rk1cOaK=;4#2+U&Y}!%X{=q{k&yXz*$Lr$vNE!ar^tADd7+R`pEiNh zas8MsX4Am=J!wl|>>Pb}L-8}NBbitTzc%hlZ4aT@DJw0dhMJY{W&iT%(z3^|@n-e( zO!fpV@D4>hMmR@#jQTH}{neKj3#4)6o9K!Ppjyl#3=TO>1G!I>wx*4f3 zkdIxbre^+{{PM|?l$#PD$bQ;%_7)j6U^*UVp74g$Bt94F#71-J^V1+q#?wgaD&R+xtaPe3MR zcxEh`pDFlK}UunD!3E^EnBm^CQfc3;dWUkRh|8lXu`rWdY{} zA_74852|a9j*jTMO7*L(#X04bl#Vw-#(BI4g@Dflq&@Tv5F}n#t}uh*ro1xkh?y1q ziBQFb1jtsMV7-EZp1U8%>j>%TV(Vq}S=2&~^G5qLfB$09P^ta$!o)q5-je(1st>kgvJ)L^Z9y`w`#Pj6Vvu6Aoy z&rUu>j{|fnb&@V_txZ_0ynCM&z1Nzd&z_D@4^q{(t60aN(X^R`LAs_s)_i^l>tx8W zW<~FdSRp}YLsXHvdjck1(ZJi=lIHGwlNYvKJR{h5YJUD%k(na>E~1&F8O5p%$Gn#T zs+Uz+Rdt0zsApi{8AjRBC8C=>GP=TTd8ke8qss>W-VWCN(s!+A1U8SUi zs`EXRyf(08P{_58jY;e+^*~j_sF+aff~ST>dSwI}E?vEhqX2n0re|_VhetdA*@;6X zGcyx_955J7gIpz22T7}1YORw-0f0}UM`vf<@qQq(xo!92pAR`4N_3aS3BT0Dd{%mw zoX(#=U%|cYof4vfTd#z)G0Tsok(Xw&*+pM27-IQAb2ZvE9hs9|AItwz3*N4P z%2j=VXNhOAB)*6&I0B2;YUb&vaR@gCH+Gj5uwBsXXL)XmA?y9mZp_Ia0N;FE?74Ry ze+9+YlnaGgNl}qHd6V{-M~xGs=XTk6?ciK6^=MV|DNOEATiXprZv#*wq~UIi-oZHr z(uZmfmAN=NMtlzu=Qx-NFFes3X*Ta|Zfw9KSq6+MB8Xx_N)ihC&ldJbr2A4{S+CqV zsOsi>vWiA%85kMCK%TGHuZ`ORu=VnteMmyFq&>z}Y31qpY<9jHG%_;!o~_9`eupbD#BTQZkDVwYqC;_GW|aPig6Dm^1mb5sXOY%fz#kblz+lx zS^^g?m;W7xh(W1iR<%|>;^YvO3zSr`LAM0auqw(?br$RJ;DqTwo|*bi*mOM=Kv__J zmDr6-yy-fdAnc+vr*!9_y7K^Nw2$9!z^P<3aIniXI;~4D$_g4v zgQcxXZuve?dEJUWz;cSq&(*;BahAm9t9Kd9CcmGpF~s+#JLPN_z#+%p2v^tgU&9@6 zRGE@b?H@#+AAB^}z}Fq6BhWmxkGm5uIxfI~6+gU7t_52%Y}dW1C%l@5scoq;Wy}Nb z$6x_X^%>oXFB3m9`2}G7^@z(`{*$y3q~RHNV^6BB@1Xj@y_LxC0T-?oIQ{-QeU63X zpCP1&G+$pZPi)qIzdl%1Ojz~ZA!pxLvNZ!h1zd{vW%5efv;l1Dj$eg(`t14h@+$oM zV-w%xBR%c#6Q6l+z~V!Pu4xFff?d2(`O|05r}vA*s72+p=;s{3l|0*$;04pY*zYg$ z8~^Op4PU=blo-@RRLv;B`&X}4No|RW%p^Rs<-qm94}rrjPpe`*sZEr}7KS0NBGF^p zQ*q~B3@KP#HTe8d|KdZz!6xVH|gGHu(%b(|T;Kt@bZ!JC&YWRP+(^j#Hlk^T4=2A{FA~kv(!)W8m_H2kx=_t#b9Yc8d{l zkE7H{J_IuFEaYsIcc5|kLudBT*Nf5Rd@}pDyzo*8cRK7Pq_{rw6?0}3`ck1_e&O}B z&}sr{ih`C^Eycw%u!X0W`50vUu&QG~9n0z+Ih1HkKa{N2rauTqnkPgd4E_6GwuDm= zDKNb9xBMa-p;W?l2iQO}SiK)FR+7JL6i?J4T!BypL)}IP2V3J!w77Qu_$yy^h`SbE zFTD8d=BomjL%?-%uSR)dHX+7Eb`N#5)YQm?{WWX+Uw`|ViwicKdQ7&%)^gbbCQd+h zb+-k<+>(+jaS_~-S1;k)5fNl{|{WsAn2 z7)Z!Y^2Z-n5|?T|eYzM6W>8{oDB7v}(ma27ucJVl@bNj@w`6j+wqA5EG~yMpQVycX z$zk}-;vex<2GQiBA@GDZkeX{%?0dhFNu|R{v){^GOaeV3-vDR z;h$pL`%qv6gjdRh=N;flj8eGFp~dB=+b?Dtc2s&kEQeP|WD7RLXKz3FR+4e7B`xYO zHcA{^&hB%|Gv_ol<1)9pnkaFUPG7#*`1Hy#*;XHJOF5QPhlBP|-WIZXsPa>YK$p0a zGccxf*}&9lbtGf_bxcgd76?s(>8M^VK{M*t(YwHGe%>9j3vK2%4S zrsNnszuSoXTmzuecZrs~4szq=lvVkC%j&?m#Y7mF_r>mPHGD6LrRC>)C@-Dd<+ZPe zo5TNL8X_AFr>gAF&F8 zL?B0=T_t0^tc+AyCMx^y9|WB8+p=4va*G-pd#2Zz-t27n&E*yYYE8(@mKH}h0EyT^ z=YN0eHj>@D8wY#J>}Cg`Vp4niHPfUA{8+MB1SyEr1_TZY=tuw=ZTj>6hk+vdk}Fsp z-vd*wFp>tzhF|BiexYK+?fu3$f%?VMYb!~XmH})kT{a>dPHuniBU`TQKQPPT`*+M<56k){iBPg5Cte21t`&%b{%YQYfIn=|AK93 zeYJ;FYhTaSZInwrch)^BeHxU)f9uKE!uojQu37;9fuw{OX9*_4JF_SEbp!EXV#Rk2 zbp&M2F004>?kMc(wCb+3JL!s~ zi2@Br`)-E_Dbm9gdW>5pL$#Mh^3FsEJPKI{UTH=G1p~8%f(3J}INYN}MVeW|P5c()H*eipuT514GJ);( z+;T+IK5Pn%LwJ~a>n@xV@}Y{CD!(x;LMv?huP6LAU8~wbK^?V&IU5NPz7k z!C|5EL-G+6JI$)b^FwV>E5OmBhx@YRR4NpMLK>7XofzV(F#ac)0Kv?`_qFF#b$?CJ zsg(q$X(sr5pfD$Xp|!*OL6n|KMzmienXbZ4cgdfr%qYQqJixuX*xH26|#N{Gzn0w$r}NyFTiNvt^(pjCJJqgI(Y>Zv~j^)zY~e0L{@gD89Wag zl*I8eN_;Vj0p>~O9^Y8qBjrhD68oMI0&z#vm zAva((wn-l{+CzKy)`5eC7{SGlUZLdECoVd=K~x?RN5YVZ-h?s5kPzVAQ#CT14H~*3 ziCmg{)17V9F+MazPXdzhxc82I!hzF})c9xQ->M1LxuF+S_)uIC@kVW*q5EmPk_&2G zWIJi_X5CEFk?$LtEniWQ2%P|4&jAvaGP1^^r*Oey3i`j@W>ji!hEPmd{>F4o zRaLT4o+X4y0QJhx=3c&b?RnZn9nq(fAb*jhFKDxiAX+s1{Np3diK1O6C#0#7s7{13S4>xRY}z)wqx~ zwzN!qPVXs`i}Gifx4KbKdDE>47U>}IK znh0U4gO<`RNBOMJ0A}-7)+qO5gSk#kU7b-e{zdcF{3||d9!W9g#K%5__}F`rSIlbt z@%QSUBx7tf0jnc#dc=l$0cYktsCyT&9m~TcL?}@sVPs_F;aR{~_=vTYrYp8hz58JNc~UlgWr`8TdyoqFU^RX zzj&b|LD5tCvqsy0?4rHe!9G19{2*Fm_%IUfY-}5ep;nxeb93&6RGEbZ|lulyyt(%9(osQHn zTzWE_ld+$k4e65mU6d;?^ii>;bF3x3|KWH;v9uH)2g|O>FXTBni81Eo>s$5z8XOJq zUq|^>PsX6&QQf|TSS$|zX-=vvpD-wuwxR!F9MV?SwrZ4cxSOip0b3;CeV#kfYgvm# z)98*JDIKrNm!%>EfP~>QsNcNsFj*m~Nvu0>U}S_b#ET?>IsD2!H8p+PaA~EM;?&B< z&z~h zV#NX#6ciM>aDiyM?+t6TOjK`se?DwY5z(i8y}jmD3$1H|gI{+Y@}OEU##}hYx9S)Ni%POhy8HlDB<*;;W~2(7wa; zpNqO>%e3nI{)}iy=5%7qIH~QIMRhEgm&7km$K%ccpltjc1wAq%bXV6_f%ns2-L~x! z9985L6zfZ+6U0NRz4yDutRVG{{&eU)S&s0T_D(`jl_7~7&2qc0%h{^*Sa48{XdA(m z@}3sf5tz=$`&Es+#-U0z!nX%PiRwKozbc~I= z-VaI#3UYzF5fd}&J2PVV#@m- z7tx%o5e6y5N1&Yv3URljB53;7*R~I`8&hCjY1A!g-e)nAj!#`(6&_~zG3A++?txc@ z`rkekuN2gKbq%r6B(HHqK9h=gm7Gc)fg^imaTJ78xz0N>fP*E{!Iy5JU_s3-`M#>k zus6OA;amb=dm84Nlr=}%qKK2uXU98Z%xEE4nSD}^gRlJ4r#Uzd?YnKnDcqK<&EZGo zW97#I56k9TD1uYN_(Ql1?id&()s$TwSCyOAT3?;<1k?D&s71!(V%XHI1i9R^@aYN#W5^?aVFYd*N6FiJD0dt+n>q2SjZVm;URlthwN(Bz}w0&%opQnj+U2 zp&*tH7l@IMi7Su$*|*_gx>)LXhDEadHe&+0m`1r7L0wXa*M8aj_U+p+sf~?}HeRuW z<_m5UB8f~?#qGu20-P1jZ>#|ZCKb_eAZA%uFzxD&h;yMaG|{hNKw3c<7NGiW)LHo2 zWIPb&-0J(-)YJse8!vAT4tjtR`kcx+#uv*yy&sM`LgwoCwZX)~LN)KsTA6p0HYvP> z$DIwewK-utv>0&=NHO4~fc{=qw)Oy{^8Dyh@ar8rcjkhInBeJxYNik9lY;8c?E(MM zd(L;FC#?5n^P9LHE*e`3_i)2meEA7w!=*kMNsdz!E@MqxCzUZ9=lL0jFV2XNqL{rC z>*o9L5E&Ue$}0cSF@WeHp_IeF9`}TBT8HA#KVK-$gdyTs%A)qSJcTK5HO`Tw_xd~D z&3yEv{QIrxg4aXP$@tYNg}?eTT^V*_FKK^)Z_lX*UUHEB2r#dDbIl+Wh;svD1In-K|~_Xg&vrPPn1mOmxGu0H*`Yi1VPZUw=8#sakl?1{wFL_Vm$T{aXAX)gGCUsDV_n^|_4NXNg`= zlP$9Dy1Tbt+$wDQ_qTs>n*p8=0b?jVA=>6eky92UnUIjM4-%gy<-CKv+nnH>6!SQ| z_po&Zyesfqq1=(Pnjos$nMg@%FZ(xm2#je2grikA!12N)B+4@4BL&GUn(4D396>@Zqj_@zT&`Cz&B)(b!# zE6GwHx5@X%KU$Ia6TQQ{VQf?pYe>k|EL<)1^V|Q!giEK50`D;N^-4R_X~%MxmzHw! za#++}ol6GR{WNg25oce3 zh-tE=WKw(euCtn}L%ERFgLozALW!u#i$8E_8($|VKyI7Rtf{@cybMVkokFbC%)x3& z;wHR#`=6A+jfhzQxApXRjk^SXd^wwQL=5ucQ*=Do6<|kAm^-AFd8_)3QTG(IQRIjq zK0$n^bKhP_YERA2Tbh{2McHuMRA7MAfLj_Q@I=8hP|(JI93s86)ZQ2A1umW>wxRb= zdi;#HIagRmosoPk{Ur)iiOrH{n)rwBDrvxF-S@1e)haElPff`SF^F;^aVA-rEc;fB z<)Xh!P!JcIoiVNjAZe4Otu3}}Mgy-o&`Dhm1`xoaSQ;K>!dg;FQ=zVg_x{ofoTs1& zEw8n5Sy*-vjZC3uBP9o9K=eW!j00|LM$DL?niRlXVn5#o1{skW0JGX$R0N3oZ8SuzR{!m*hdrLfz90D-K=cD<7~TnxKb-;68JnsJx~`48zB88K((LXQLOWDEgY)o5EcTom6> z(?d9XxgDN<*~7x)&J#tlQkNvN%PuK7%$9j~r7Z*tJ8Hnfh+ZMa9<3LgbP(zr%=A~= ztX45b@xL*46f*f>wX3Ls=ZsetIXO9+2D7jCA0V!|{B1UjFGXzwByC=PF$DKuE)kKm zsvz{YRcXC{o{mqe^Mk$C$_VBxc4l0Me@k0d z$a-T9{8@r~yoUdi*gJ;E%im_;1XZ(_1}n-u6?i5?8Y2q2Ua zNSBRoZ9O~W5)vFd+EuUuoi?P^jR$4xag)+H!kLG9YgUJoAU6*awvw4ICF&q@8e(Pq z58N8{s<1_Z#8gd(v2p8U99Fzxj3RxKKcmt;c1tNRaqYy$hVBFL3ClM$tneeen#@}h z5*lgQfPpz!c>N1e$qV^16h6679oEO9#bjsf8FAcL(bPL~1b%cV{x50iMK$R=t{}MG zVZ9BM$Aa+sDth(kpqAJoUZqkeUxWBGddbN&e+k`9E0k1i+YC@ zk1)6(=;dB6jv8+aOiZL9VdC-Q$BMgJ<*!VITh)BC51>6t{U?lkx;JW&EAL>o}lL_o2I1MB$K76c=VM<$Hn^}H;^J-~g z{dz>FonKG;U>1h3(W7m_R`?b>V}1YS+UFPAL!Lih&#IH8*nl_%hHCtI8z_6^SQOcg z1t*K{5w{X8^;jBTAE2@*5AAgI2w!Rngd)*&UB_968k=(<(j#lS)6KP93*>^vDA2154{ebs%~6P}0nebyZLH+9s@NN&oK$9~E06>X1b!Ic zu6^L9zT_I^8A@u%4|b$Qym_-g!^*18Aa(uvS5W;}>DA$a;+A6BCs&G$I_4q1VO7ml z)FaXg`*n!ahX#i4%p#TC1qLwrrrVsbAGwEc!aSd2(&I!{b^3=Q<`Ape;$-m+#8pb1 zVPY~w*k^)+kRArl`}@5}PJwv>fvE`)Dv`+{JaIki#-n82*-|R2vG!`{GF(PRM&1}U z4TJUaB^0f$W=i>6YhbMh&}+@qO@2EV5FjC{)Dfjy3tkwN$V6wR_>P^r1AG&+V@U=y zP}{cTNFE}J^yN?1erRko2MUoD76t^|@99xxr0<7@hNeDt>MFEX-62#*-axPMQgmbM zT6M#Kgcgt0LW|vk2O4ffp7)l?<<6DuEodIGpbzDDWTB_Oyrcg#H%&=H=KB}tCxV|p zr}IY3MPk3PjUEWV@t(LK;!V2$1=-dWP^hL4F)f9cu|uG_u-kK6K;X0h!D3#dUT)3s z%OZr8j8})$_tuv#C!_3H`riy-I9Nw=nlZfdk%eEZU> zQ1iR|?;R9GmkzNuqPsJq=bC^9}b z0)bqZu%pb7F{taU%_$)v4VT_DfRz4k zGIx=ZwvhO&it0+0HZ){P1!X{f85ScpMtvx@?Q9sH{N?qguV5_dQHxJm*7CnfY?hJrZ<+3g|d61u{FlxwAScOD~ z)*wx3V2G~=(M+YfjyEq{KqY8#nd2Gs`pxl#nc*x0a2$S#hK;oonlN$-L}4M8Mp^Sg`-(uY^5|=eU(d`ZCoM5 zZ4+9(Nf3*XAow#Y^e@kikejh%o*Fis97utv0+~bDZR$Z#FV(%eF`QrO^1}!g8&CI`MotrG-6+b3IeMbj>pn1T6Ul}_}B~*jtnjvot{aK>#E1@{0xVp8%|jaU}T;@`vdwxtVviGjEW7!Y$@ z?VlALI+>gGW;{{jIVlfQD0~g0%~=L84oPE9($X|lajH6_k(NS~So;+!LMQSOYDU}S z7tw}c-(h_3#vY^4ThsXZTCD8n{kacrn1_y>C8cCP-H?P{k&i#DuuzPr=C0VMdMw+> zKvu`a_uIqxEQtpk_Btw|d#&i@Vbu&^T5CXLoN=dCdZmA0u+N1=JZ$5yENM^@IX|AOeWvpAz-tvFeThVs^5a56E2Z7o9lHvTLG^h1 z`U?#k%z0j0U!l^rZR0;nXJ!VjqX3G};6pshLNmGOTn=egc}00~8Fpq@FSHi%TdhG8 zAN=6a@AP9-m1SjvO{wY}<1YEyCUBs|peA&zU7uGKFHIySgq04wlk3(Znj*f~%oy(%yjWbp;`C45` ztQ$qeBR~nsq+9$!Zc>f53r??d<=8jOGxZ~UNv`Y6t zdgv122<%@kz(GZ{dDf4XPm6e??;mny38Q9Uz>pUu=p^h;kP$YkR}@D6-rjPTB`eW+ zYTMp4R-&OP$v6UN)deAaX6*@6yV zy6qyYn$xZ>t*sctlN1t?cPH96|B{SM7c8!^etr;_L%Ch%(X=pmrC#ag&H4=8Q9OWB zrQR{f-H21lfH^f)rD7Z5YSp9XIED!-Dv6Qik&%IY)=d(*Kfk<8iFp7}D2Zq<+Ki|9 z3;G9aM%{F&O_blfj$6Cakv(_bQq#OHJu~SFAQ9wN?4y;9k91~nemvR{sAWY26X^Ih zm%o6y=7aVeEpY%VWN!Ipy6;z)A_J+&_YIc0=uH6ek{{G9?WRv zUfrgorN@xK+MfrFj8`q?2?A;VU2)LZo#SUKPUW2=T+BiG(+taKw^ zJ~t=MzJ=iw9wR_*mu^f)Uq?`X66IJq~&P9hVeDYrrMm)hDiyEo(YVk#1LypLst*QE5kdoAeF7+VaM@1FK2~z`-$;Z$Y0YHAYb&Y7q4lY+z0f{+@TtKCN--Bn+ zwoN!*Ye((~x{8Je55&Zr8e(e?a}KH_<16b)co#TgZsT>;oh~jr5O+_!{%q-NNI1Hh zcyD8kbb^G^P)k8YMWetp1_RJgy&JT?imX3=%3Z7h)fW&!O6bP?{jcH~W;(*GaAwOE zwo{iy&nwEuN(DjCnWFxttEoxaXyNpi&Q5sxh*Lu0YUH+Ck)ZMc#J_q|PJ?mf!_K3c9Venv>Hi{g{)o%U+<4Q;3yGxti1rNU85G5#0tp3n<|bE8)v) zLMD>>GhFJqxn~Ag2isFZ#Dr(uKkf@-!%E!pZdW42&0wsR8J}nn*Z1!vM^dllu;WV0 zL~G{j&r#@U;;BGZmF)K4L9(JG?JMo8#B&zSEkV^a2USVYs{SqPb^a-38VHpqD!Bla z5ls)Mt{UjJ9flC<_5Aj#zh){E508Id|5aXg@%%uBb%e_ZFRNOS@vHn~Cyk@8XK$Um{g4k{p?_W% zRUhF2cOdoCS2Ro%{UTI!QsFFDCqIk7 zd;jF-FJ9Hs=8K8Tl5dhb`M*U%6SIey^$#jYddh#7OHbJBUVeW3iwgMkp>KU&1W+FE zqD0=>fBtGo00d3n&6I4+FAa|Ml0+Hiey^_rRS=vX#|)aPW4}@4x;^*K}`yD6s^W5@Q>8lPl)Mt3rz8fN;{KrK^(X>iRzy zc&bQP+4Qjf!+?E_QMxMM60=SDd=0hkdPM$s; z4a~2;A7D(>M5jU&FTeTVm2I+~ZXNz>dM(-2=tO=Cu>3~{xaFN={MQnlz1sr2)cSn6 z1J3bEZc{iVea~mUq~Num_j)yx(r$V%VF&&U{@OtnUDiT+B z7gTi9m|R4pOUSMFV9;sSw_0Jl*-?ZKsdM?Ig`2A#1QS#%w5AQNh*-Vc<|HE^IuhV4 z&5xj&7-(v_OI!RB|K%GM+KprN^Nc^U(ZL$fpI}_`(4%$v)nwzek|m#Pn7QFFyS7B&MQZ! zN5(pGxV9O0RM~XgvSl@(FQwHJPjkN#&bc2qiF@kVLeSfTT&|ZN3GQPsJgmcL^zx>H zUCl1dAfebwGqTO}Lgf_OIhnA$&FnM-WhBP&k+SA>ZmgOijs)cagGT;a3foZRJ^(gs zPEd`rUzvv$0IIW^z8pn=hBp3|)B?wibz)^1$ppJ%0rwqF9u9++Yybdd7;x1xQh>;Y zdy8dze_-xEzQsGq#QUZMu{8HW~u6` zB!V7FNifxF;#sLY$Sf!z5EdE=iQwIy5)uo`gLuTMgL!HN`!JMm7RfR0M|iNn@3cPKyKR9)8=Lz z5i2tNMSYSX2Sh3)ZeXL6IPHju+y8?G4M6;NIkF<_Zyx|(*UdGt;MsSl;6?ySai?c2 zeQ%63%3nh(z}+MQg3;Cd#zk3rzRGH9{rAGk5JiF_FM`Kb@`xTU^kPLkMipR zDhU{iD5_s<=9X@!|JR9eKTzX5a@V^lsT=48QzV0Rig=yt#iH5+6)Wnc=(X!R3*Or6 ziLh=wk8f4v@^ehZ=tv|q0&Y@Hz9yIGPQjB6bh0aN0i3MFBR4WKvV+Xa;}|FO<<&RY zgl0?2RK@c3ZmrV!(Ke#SYk`}Mjg6ydS${TO4rA+*GzxNkeHGvm2H_+earOkF67^1D zBVL)c(JKG`DdRHA0wCi?6hC2l@qWB4evtY2(56$}5}(dWkSh3F6$oyj z6{Lx^Q>kX}9t(^)gAyiY9uneA1mNG+5ZL`q|G{W50u-Gw;f#&mzQczH`}*dP7XyzzQEgX$ZYTW49=^T} zFoOY8IDN9g%gal`f7GDKv_2&c$YywWc!sZyo|iB2^uTh56j02Af|)<|R~?d!VkB<( zW?=74*IEHkp+m|s9*t)xt_Cy}(WjuOBmj3GX zJ?t@$!@|^8_8fzduXhk?OQKD%Yt1txL>0z09J1|jbi55YeKD(rD)`4lG^sLU}I9seI9L6Vrvj0_)+gv!Ld@~LtK zPW#72yKF|aJk7r#ALVX5U@pPVb%ACyUx;y4udlB!qf2X3goO7XC1n?;nD3I5#Ak0w zwlU+4;1vv%I1%>}LBREf9nD5{3*!lLf`SBz_1_AoCG7fgBzgaWZRy$AmJ%pq{#8D| z+ZBo7Hq-GJZG9>p9+Q*c^0n!Rx?oIeWyYaAvLtGw;kG}R(V8MRm0xW((_X zIN+TRAsk_<{vr^66`8EYD+g`#e7(H>q@;vlh?umF!;f5(Wq$wcw@S9LbIYvrdD8g3G1v@#ALtYy42 z5Bf8rWGvDg)-6a4sK+B~UgsW&aDE>}6^6BgU zUQp%xU#bOct!6^a+;2~F=ksP&qtDri-LqjMS4%He_)VHSMFnle%uQ_L&W@Z+S~4zi z;%fj0E^R~6>?U&i56NA-%x1ppQ*XD?)4F~8+CNxvefXczb+uI%VZQf=SdI#hQ$h1Q z;NC^P@t)J5T^kc9>W`0@C9o5NB|2l>(_JpBl*k(f?g=Ch0o zew8QJDbx$mF`;~`iFh)vaaic?khG1BOkus5hP9&M@E0q?qY?vi=g-(g3$Ti%Xw&xm zN%5>v(MW&zR021lFUCE~Vh98K)Cz2+vsuBxNpQG}DFAMvhMRk6qG)A->Mn39D4-pS zv@Zi){BorCG#B+;x+nEBPglw68)y7SZnWs=-+o>@acMlfVLduJwzqGGG%!HuOJHtS z8fhzER!fuusL_;aln3z;CYP3$l#I8gM?s1VC7RcQoRRkVm)#Hn5H;NAo! z=(5AYG8_H;7T=ydb&7Dl{rOK35^j3F-%dG}N{NU;hA5H3b^ht|_Y%#1>!E3qR&8XW zVYS-s`%!z0)NZ_7O)o0Dh7=1M$~j*Jng^~)Nr>kzis*)}JqDtx&(V4uA|l1Y8;fhi zvMfYuy0xUKT74&~0DBPgWK~gs&bf{2P7(>D6Dv2ADLc+xQ&qLIw#Ht2r)$+pVoz~s zZx_wK6r58*Fk9UUeD0Hi3xR$OIYv2_b1L`l+$2x>0r-QvY8S zqW<@u=D(hroy6ceZ^)H)9jfxII&&7Y4n6>bgwi)f5fkV<*%{ovBe4VF?yu3eyn5v| zj1^M}JuWCpQ*&aBss=F~P=JeTt}#IY5;i|>#9b-2Cld-(cnn>XlKi=0`}Z~7SlJK|Pmfjr17slkzP$YmY$KjenA2j)UN6zxxq=y6t!uNZBt%>bM4oiM0urm*o({ z?A|2T##D$}f~35Ie!V`!W~QG@zwwH0uS8cH&bzh_{{NWN1@$bQDM4vjRCa|?ey3Si zb{j67$WWEbFVxkuakH*lDHSO1e_&_kxactbF;Hmi)x9XZ{hDv zM0zS7{G0mviDG6J!@UU~A3p|9|BrnPyHjADV??Ftl4X7AP!7E?2|a(J$Ytv1%!llJ z3-&Qy3m&TN-rHv{c1)r-Axy}gkCXE?<^ft%8sd!EO3JTT`meQwlTIB9TprO9zQU z(FP@8DM+iKcyYfKN0{m7CJ&eN-=}wKgA%9u*_PB4CsX1<4lRBr|qSDNet$QK7eOrz%)dN6JEE{@DqitrGN7Qut zgxR}N3^w#E^;qkj>ndpa^I7)4fI=UkU%bZUbgUSZ6n5_XZ>6(6CIp~0%1hB}7cIy6 zxg~nj@ovp8Z?R2qm~4&Qq5p^V?-+2Y73Tq=_?N{NCyWJHBE8t1w=lyh+ydUtB7|H2 z-)B}}qho00cY7LUix(Qy`wrR7mGj%w71^|3*{T63mtoQk8W*11Uta*=LF`TH#=Yp5 zQreF5qn$_0muJ785P>%0FF*K?Zu{|Ce0Y8}2VV;1`|q{>z1q|NfeD4V9uNrC_erL~8TQbqF0% zezp}(3z`rs-E-eI|LVmuG%c~PrcL)=&RS<@*jc)HjaeTf3}(;Kv!CnhO>v_CxbV~W ziQ>;^e}cs0F`9<{Lwg8QG;y1XmI7CK36RWPK)_6NyI-LCzDumb;M0z*25&v96 z{~yeL+SjM-<^J%K2xG?Q*em|SIFJ86-aSZ0Py%CJU}r_W`8jNefB01gMfBZwr33KE zYU;H#cUEV~6+0C>*<1Hcegn}7$|)SkO^|(9+60_Gq?S^h*3L(OFMvWZaAdo5zN0;)OyRWvKd zclE!fMW(w6_Vl2d=S->mC$l<5umf$3=RK(ip-x)+?n(|hSlL?@OjD8rM}DBZ(eCwZGY-XujJ0N`^!urU}*cEK~_PNp=S=un{yWu+$k&U(v-Xd z!Abc!OhKOKC;L0dDLvo7ROfKHYpq$VMT-}PdMnHk(}m(*VPKPF4SA1awCUuQufIjE zPOZaXBf-^;g&>21gGPeu&iI9x7iU)9z7h_$1vms@@$OxDca-1DczkO7c>2tly8|`H zp{J6K5<8f>7_^kt6($>-olo>nb(b1qF|cmZtFu zI-vJ++2JA&Nz+TT(J1sJx|73|lI9?!MvyU5s^LJm2^|v%ydwt=Sj-OA32%Hku9}8a z%#68G39$S{j@{Kqc!0uPJL4G6bVtXC+5o<3f33FFg?`iQZ1NUN)fR@S0?03V)%w&6 zzrpcP87M+}bTY?c7dqXApxhYYcS}(OH2MwwX`7p1L&^mI#7ERjf z2M6sAXur?U$x2Arx^*k4UK1SQo`u-iw9(T0R(p6Tbp~hG(#m|eFa+uY(ap0Q>5|iaPm1|4j6kCVXvX8>;(R<$D(9XO-vjA!S9`lHMlzG{ zV`Yc!qXc%do}TaP3uA`j;^Ho450j9l^Y-ooXXk;(F8dg-CPx1%8^*_FQPKM&9_vXr zb0UKHmUqsc7P`48E`ONI)*Pp@o4b2VXmD_@V|QU%;W9ft{ZL1n85rY+4k0m!nUUOM z?@5{U#wuQ;V0mSt$oPidW-hd}h6a?eFb!6`9eVdpVQc9&U}}k+`%KUJ3T=SQpBOP- zZ^<@NrKW@R;1wdCh#4Y1;dI)L4jM%oGJ4f2SqRNLeXDaYhFCYC5eCMfD8gHowwK^{r?Zz{0i))|mPB!I5T`?Z{dxrk`>B023_*-o6q&bb zzT{fCGV$M8wT>|kFVz_HwJeEg5JdZp^xLCC7uLof%(}t>J+oq5L_#UhJ7(1-$FBVS z`*(-MZ!26$%5X2f+<@Gj!EKuSp|^M4;~o1#-rSB%i{QR*+Z)E;?XXe=xyGfXbY~)o zmH!2EpRvU^AP60!+UZ=FE=1K~?grPL`qOyDy*Iq6rG@-q^QjL{NQIBaLR*DZDjkv^ zy}T5CSwh9hD+?3l7kp)fxj1-~Pd)=)fwXb2Ik)x2o>cWbda8gO_V#N5DJu5ML%QrE zff0@Y31qddWo2dTI)*|NLTZ7LCzB9BsEdeKv_uQoa({gSg#)s!nz;6%@E$=F?Eacx zf1~%-1?0;I&l2=(72*yTJdzOnhkFHt22WwiuR3XfSA*x=a`N(IS663}bw}T5I{JRV z*Vji?b}t|!vrD8iS3%rD# z(EQ*;g`SP5WOmB%o$x#uEMyeCQmng`G0l)ljWO(FZHape$@`mk?g$~qjA+!>^v7H)bl z6EkgkSvM2rg%uPUOm!M4L(Rw9uTuLUr*dr|LLHSW)!ExuP!%=BV~X9rt&kOs@tc?N z`AHhZwKpXSvfmQhG(*dwc?w`wVbCKd&%wkRn9Yzm*e8;dKY~CngcU%Y`hTmDC5FnL ztROc4%m;5qDEap7bGwZAvO@LCw`$~@vq5)EGBiXW5YU;?_2l@QLU0;=mv&mFB!GA| z@82WJM38>_+poW3EUnLk3zN;jw{JJ#%TuZ+z?%uhMj24WZrLL(+_VI{EI!#5@)rFp z&<5ZMpe6@3BL^*LhFEnlM%(3CjKnJHL^(KY5Wy*1TZaEmB71l_TaR#@FX3AosgZm< zB!0-jGy$)QSX%ePY^zz}g4oz1Ybw7Z1hGCD^Xu?W_*8MMWOM^n#L zh5euEg;lG5Or`en1Ry*#HWntyS7|1p9fbJxh>aid?Q*NhO*1o`rapF;TWn-;T>pKpk;{ncj=E&ZH2DNXN{DMuM zVfG%}*8Np>bZ=nLA0X6LwaO`JFw0%TI6cNN@QDkc>g zL^Nz^lsGfYJ(J&vc?KXrhDS@i?GfgABR0%tEKB1Pl|yU*SRgQr9>>}j5cZpiX7G*?<|fuQw0#`TN!zumyoESa$Lo2cY!u4NiHgu!d@|Zn*#y~ z8vPUz8+`BH{I=RhOoMJM_1&KMISc3T%oh*HAR zGv;P;l0!bLyNp>R!M358LhT15F3}|g8lmq7k4g$RHCM>5t_61U#Mv5fI=?pRU>VUe zsj9Blv7P$xTTE~r29Ah`uqOGNf2yuV1TKq4{{ES-50DB%LqnVwh+g(}I(?MW^l?y3 zBO-Yhu_yr{fe`bHv!W7)WVL$(=cj}5|)=D-3N^+EyiAull^bQ^_|4Uhf=b0&B<)9Duqa2_;9tpV-+Zni!^A$!jah z%HpGA`hZHZ3#Rih?%Su;=ANXS`tcLM1m|d0N5C7KQBTj#5chh|8@=;o_y2SW$+UTH zegb4B1fse|>bb-bN_3qmsyz1g_DvNc2#Qt9xAxX*N~GFzBw%dd%AA@SYE|f{8Bd

<$jY7FEMgD{fXh?<6m`OwGIeDxCZqXd1lO#0NW*qRe#1iY?(5_n3FNCtSgC?pP1*b+RH zQ=;u&awv7xq^isfo8FR%b_`J)VGavm>Y~bvMU}i?$z5Eh;TG)%O@nl-8G$D@HPv2-i<8Ha}{H?s0H0}C}r@s}S;}z(2en)9t zNMIjeFe)rMQM~*AgmEOvfM~~)Pr6s(5TUK9pVnM2Te@)%iw*BtuC=l`5 z&L&@TvfV?o?4*=Hl0VIJH&4B$N>=e?4f~<*pQ8Fhz4NaP4h`j4Po=?)p23*LDG#x7hn=f#3ez|-}YBa;rKQMxQ zwBry}QDIS0QS{rhtgJ1?BF8e$eypyRwwYE?ePd|P#izn~r@L@%-EN`_CQ5B>Ni#Dr zV1;Ysn)gh;jRXfrMNsbAvBSHTtVkYlqF(lAB=4R3KFoaW)w#Z5h7#9D&}DJz+)7nj zQsA8qg5^${CdzuvXorhV3^g+U@>Mq8y$kEGvbD1daDEcx7JKpid(&4jQN`EKpf4g& z_A)8AfGdR2gX9SJcxMP@REy)}gZB*$3VTx1(i}&u_(AzTdi1CuzaXkvl$x5h(z!%V zUA=%uP(9xQB|&J8S&Ap6%bV5$J6WT#mVw#`RywAgq*S>xQqLO?_)>|=$$j&smF*p% zwya>B-xoIS>nU8fr<-E|$E12f!b@*0?vARr#E_rfXr9=k$*WD5 za{1mTTb?F|2%4Fi`qRmvW3=Rlv}JOQR6nQ|UryrmIppewEGN;ign*6UKZDZhw4b@V z`xc{GTet4Ld-39v+m)kNk6slKd35PI_O;ACM_&oE;$yr^IKzq~wavlLFaEWYAPbY6 ziU%#zXiWa zePxgZ4D|ICdHeN_@N2r|ceF0LDr@jw)u7zF_r!L$Sos84Qk5N+XC8n2i0VuogH!}Q z)z%ufq;^Lz2n$YK2&h9pGP7$Zkn~S%uF4pL_$~ZRcTFwp;k6V^d6+2q_Z*+oM99_4 zPR47^Nf5o@TaFfo@|uM!oT!{RGnK5gOb`{-BeX4+FOu%HnR4-saG3)8$79^=s<<%K zlZrrqm)~y|tSeW%_%@{8ZMQ+XJ!4Pmwe$;xu4fa0(9zNwzbWb?iZ}0GsKSsLj@k_e zjerOJ0o#*a`qi9~PHj8oG0_qc=&kx*to9~*RC&(TFQ>A&YpKBxS7hjn!2E6dA%>MSf{Mn=v;RqLy>Lx(i(u_47hMxLJZ zoyQ+^dzB<~pZ~$P*H+@EV)`BL<_Dp}2dQouR?-TnR-U+P+RfzW5c>-LTgkQd!-=yt`q!9k*V#qSwbhoL{H3DM z{AaSvJ96Ulb=@?7w;mq;m>IvO=Q_ekzuuoB;e7M-T{^4w-VVSd-o zK|T@}cb)OTG8+F-kCi0)>S=by0{vd4lW+Zq?Zf8lzJff-o;?iGTi)%yk$&xCc5JMe zWn7p{+&{;j&Fb}5Cctt3uNXwE4qwY8+V zO-e~z|6j{ggqg~a70HnBtZ+`0*uw3bRvD{?zGRpo1Wb(j&bMY1%YqsvU|=DA4Y=C> zPq@vaFLvdxfAY!A*tD>;_1wRi3hp=fS?83{1OO@uq>}&u literal 0 HcmV?d00001 diff --git a/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf new file mode 100644 index 000000000..8b878f87f --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf @@ -0,0 +1,62 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name + container_name = local.landingzone[try(each.value.level, "current")].container_name + resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id + key = each.value.tfstate + } +} + +locals { + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + + remote = { + aad_apps = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aad_apps, {})) + } + + azuread_groups = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) + } + + keyvaults = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) + } + + managed_identities = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) + } + + vnets = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) + } + } + + +} diff --git a/caf_solution/add-ons/azure_devops_v1/main.tf b/caf_solution/add-ons/azure_devops_v1/main.tf new file mode 100644 index 000000000..5ee46dde9 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/main.tf @@ -0,0 +1,56 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.56.0" + } + azuread = { + source = "hashicorp/azuread" + version = "~> 1.4.0" + } + azuredevops = { + source = "microsoft/azuredevops" + version = "~> 0.1.3" + } + } + required_version = ">= 0.13" +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = true + } + } +} + +data "azurerm_client_config" "current" {} + + +locals { + + # Update the tfstates map + tfstates = merge( + tomap( + { + (var.landingzone.key) = local.backend[var.landingzone.backend_type] + } + ) + , + data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates + ) + + + backend = { + azurerm = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + key = var.tfstate_key + level = var.landingzone.level + tenant_id = var.tenant_id + subscription_id = data.azurerm_client_config.current.subscription_id + } + } + +} diff --git a/caf_solution/add-ons/azure_devops_v1/output.tf b/caf_solution/add-ons/azure_devops_v1/output.tf new file mode 100644 index 000000000..91eeb4691 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/output.tf @@ -0,0 +1,8 @@ +# output "keyvaults" { +# value = tomap( +# { +# (var.landingzone.key) = module.caf.keyvaults +# } +# ) +# sensitive = true +# } \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/readme.md b/caf_solution/add-ons/azure_devops_v1/readme.md new file mode 100644 index 000000000..1ebbb675f --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/readme.md @@ -0,0 +1,60 @@ +# Cloud Adoption Framework for Azure - Landing zones on Terraform - Azure Devops add-on + +The Azure Devops add-ons allow you to setup you Azure Devops environment as a platform to automate all your subsequent landing zone deployment from level 0 until level 4 through Azure pipelines with self hosted agents. + +* Azure Devops: + - Agent Pools (Organization and Project Level) + - Service Endpoint + - Variables and Variable Groups + - Pipelines + +* Azure (Connection with Azure Devops): + - Azure AD Application + - Custom Role + - Keyvault and access policies for Azure AD App + +Azure Devops add-on landing zone operates at **level 0** + +For a review of the hierarchy approach of Cloud Adoption Framework for Azure landing zones on Terraform, you can refer to [the following documentation](../../documentation/code_architecture/hierarchy.md). + +## Dependencies + +Landing zone: +* CAF Launchpad (Scenario 200 or above) + +Azure Devops (example): +* Organization: https://dev.azure.com/azure-terraform +* Project : contoso_demo (https://dev.azure.com/azure-terraform/contoso_demo) +* Repo : caf-configuration (https://dev.azure.com/azure-terraform/contoso_demo/_git/caf-configuration) + - In order for pipeline to work properly, YAML file should be in this repo and referred accordingly under pipeline section in azure_devops.tfvars + - sample yaml attached [here](./scenario/200-contoso_demo/pipeline/rover.yaml). + +Azure: +* PAT Token : PAT Token should be updated in keyvault secret that deployed by launchpad LZ as below + +![](./documentation/images/pat_token.png) + +## Deployment + +```bash +rover -lz /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops \ + -tfstate azure_devops-contoso_demo.tfstate \ + -var-folder /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops/scenario/200-contoso_demo \ + -parallelism 30 \ + -level level0 \ + -env sandpit \ + -a apply + + +# If the tfstates are stored in a different subscription you need to execute the following command +rover -lz /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops \ + -tfstate_subscription_id \ + -tfstate azure_devops-contoso_demo.tfstate \ + -var-folder /tf/caf/landingzones/caf_launchpad/add-ons/azure_devops/scenario/200-contoso_demo \ + -parallelism 30 \ + -level level0 \ + -env sandpit \ + -a apply +``` + +We are planning to release more examples on how to deploy the Azure Devops Agents. diff --git a/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/azure_devops.tfvars b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/azure_devops.tfvars new file mode 100644 index 000000000..bf0bd6c1f --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/azure_devops.tfvars @@ -0,0 +1,232 @@ + + +azure_devops = { + + url = "https://dev.azure.com/azure-terraform/" + project = "contoso_demo" + + # PAT Token should be updated manually to the keyvault after running launchpad + pats = { + admin = { + secret_name = "azdo-pat-admin" + lz_key = "launchpad" + keyvault_key = "secrets" + } + } + + organization_agent_pools = { + level0 = { + name = "caf-sandpit-level0" + auto_provision = false # When set to false the agent pool is not populated automatically into the devops projects (recommended) + } + level1 = { + name = "caf-sandpit-level1" + auto_provision = false + } + level2 = { + name = "caf-sandpit-level2" + auto_provision = false + } + level3 = { + name = "caf-sandpit-level3" + auto_provision = false + } + level4 = { + name = "caf-sandpit-level4" + auto_provision = false + } + } + + project_agent_pools = { + level0 = { + name = "caf-sandpit-level0" + } + level1 = { + name = "caf-sandpit-level1" + } + level2 = { + name = "caf-sandpit-level2" + } + level3 = { + name = "caf-sandpit-level3" + } + level4 = { + name = "caf-sandpit-level4" + } + } + + service_endpoints = { + contoso_demo = { + endpoint_name = "terraformdev (terraformdev.onmicrosoft.com) - contoso_demo" + subscription_name = "ase-landingzone" + subscription_id = "1d53e782-9f46-4720-b6b3-cff29106e9f6" + aad_app_key = "contoso_demo" + secret_keyvault_key = "devops" + } + } + + variable_groups = { + global = { + name = "release-global" # changing that name requires to change it in the devops agents yaml variables group + allow_access = true + variables = { + HOME_FOLDER_USER = "vsts_azpcontainer" + ROVER_IMAGE = "aztfmod/rover:2010.2808" + TF_CLI_ARGS = "'-no-color'" + TF_CLI_ARGS_init = "" + TF_CLI_ARGS_plan = "'-input=false'" + TF_VAR_ARGS_destroy = "'-auto-approve -refresh=false'" + ENVIRONMENT = "sandpit" + LANDINGZONE_BRANCH = "2010.0.0" + } + } + + level0 = { + name = "release-level0" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level0" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level0" + } + } + + level0_kv = { + name = "release-level0-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level0" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + + level1 = { + name = "release-level1" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level1" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level1" + } + } + + level1_kv = { + name = "release-level1-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level1" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + + level2 = { + name = "release-level2" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level2" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level2" + } + } + + level2_kv = { + name = "release-level2-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level2" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + + level3 = { + name = "release-level3" + allow_access = true + variables = { + TF_VAR_pipeline_level = "level3" + ARM_USE_MSI = "true" + AGENT_POOL = "caf-sandpit-level3" + } + } + + level3_kv = { + name = "release-level3-msi" + allow_access = true + keyvault = { + lz_key = "launchpad" + keyvault_key = "level3" + serviceendpoint_key = "contoso_demo" + } + variables = { + name = "msi-resource-id" + } + } + } + + pipelines = { + + # + # Agent pools + # + + devops_agent_level1_plan = { + name = "devops_agent_level1_plan" + folder = "\\configuration\\level1" + yaml = "configuration/pipeline/rover.yaml" + repo_type = "TfsGit" + git_repo_name = "caf-configuration" + variables = { + landingZoneName = "azdo-agent-level1", + terraformAction = "plan", + tfstateName = "azdo-agent-level1.tfstate" + configPath = "/configuration/level1/azuredevops/agent" + landingZonePath = "/public/landingzones/caf_launchpad/add-ons/azure_devops_agent" + level = "level1" + } + variable_group_keys = ["global", "level0", "level0_kv"] + } + devops_agent_level1_apply = { + name = "devops_agent_level1_apply" + folder = "\\configuration\\level1" + yaml = "configuration/pipeline/rover.yaml" + repo_type = "TfsGit" + git_repo_name = "caf-configuration" + variables = { + landingZoneName = "azdo-agent-level1", + terraformAction = "apply", + tfstateName = "azdo-agent-level1.tfstate" + configPath = "/configuration/level1/azuredevops/agent" + landingZonePath = "/public/landingzones/caf_launchpad/add-ons/azure_devops_agent" + level = "level1" + } + variable_group_keys = ["global", "level0", "level0_kv"] + } + devops_agent_level1_destroy = { + name = "devops_agent_level1_destroy" + folder = "\\configuration\\level1" + yaml = "configuration/pipeline/rover.yaml" + repo_type = "TfsGit" + git_repo_name = "caf-configuration" + variables = { + landingZoneName = "azdo-agent-level1", + terraformAction = "destroy", + tfstateName = "azdo-agent-level1.tfstate" + configPath = "/configuration/level1/azuredevops/agent" + landingZonePath = "/public/landingzones/caf_launchpad/add-ons/azure_devops_agent" + level = "level1" + } + variable_group_keys = ["global", "level0", "level0_kv"] + } + } +} diff --git a/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/configurations.tfvars b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/configurations.tfvars new file mode 100644 index 000000000..e4d8208f3 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/configurations.tfvars @@ -0,0 +1,123 @@ +landingzone = { + backend_type = "azurerm" + global_settings_key = "launchpad" + level = "level0" + key = "azdo-contoso_demo" + tfstates = { + launchpad = { + level = "current" + tfstate = "caf_launchpad.tfstate" + } + } +} + +resource_groups = { + rg1 = { + name = "devops-agents-security" + } +} + + +keyvaults = { + devops = { + name = "devops" + resource_group_key = "rg1" + sku_name = "standard" + + creation_policies = { + keyvault_level0_rw = { + # Reference a key to an azure ad group + lz_key = "launchpad" + azuread_group_key = "keyvault_level0_rw" + secret_permissions = ["Set", "Get", "List", "Delete", "Purge", "Recover"] + } + } + } +} + +keyvault_access_policies_azuread_apps = { + level0 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level1 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level2 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level3 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } + level4 = { + contoso_demo = { + lz_key = "launchpad" + azuread_app_key = "contoso_demo" + secret_permissions = ["Get", "List"] + } + } +} + + +azuread_apps = { + + contoso_demo = { + useprefix = true + application_name = "caf-level4-contoso_demo" + password_expire_in_days = 60 + tenant_name = "terraformdev.onmicrosoft.com" + reply_urls = ["https://localhost"] + keyvaults = { + devops = { + secret_prefix = "aadapp-caf-level4-azdo-contoso_demo" + } + } + } + +} + +custom_role_definitions = { + + caf-azdo-to-azure-subscription = { + name = "caf-azure-devops-azure-app-service-improvement-program-TO-azure-subscription" + useprefix = true + description = "CAF Custom role for service principal in Azure Devops to access resources" + permissions = { + actions = [ + "Microsoft.Resources/subscriptions/read", + "Microsoft.KeyVault/vaults/read" + ] + } + } + +} + + +role_mapping = { + custom_role_mapping = { + subscriptions = { + logged_in_subscription = { + "caf-azdo-to-azure-subscription" = { + azuread_apps = { + keys = ["contoso_demo"] + } + } + } + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/pipeline/rover.yaml b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/pipeline/rover.yaml new file mode 100644 index 000000000..156fb9983 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/scenario/200-contoso_demo/pipeline/rover.yaml @@ -0,0 +1,70 @@ +parameters: + - name: timeoutInMinutes + displayName: 'Timeout in minutes' + type: number + default: 60 + + variables: + - group: release-global + + resources: + containers: + - container: rover + image: $(ROVER_IMAGE) + options: --user 0:0 -e TF_PLUGIN_CACHE_DIR="/home/$(HOME_FOLDER_USER)/plugin-cache" -e TF_DATA_DIR="/home/$(HOME_FOLDER_USER)" + + trigger: none + + jobs: + - job: CAF_Rover + + displayName: Azure Landing Zone + + pool: $(AGENT_POOL) + + continueOnError: false + + workspace: + clean: all + + container: rover + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + steps: + - checkout: self + + - bash: | + git clone --branch $(LANDINGZONE_BRANCH) https://github.com/Azure/caf-terraform-landingzones.git ${BUILD_REPOSITORY_LOCALPATH}/public 2>/dev/null + + az login --identity -u $(msi-resource-id) + + /tf/rover/rover.sh -lz ${BUILD_REPOSITORY_LOCALPATH}$(landingZonePath) \ + -tfstate $(tfstateName) \ + -var-folder ${BUILD_REPOSITORY_LOCALPATH}$(configPath) \ + -parallelism=30 \ + -level $(level) \ + -a $(terraformAction) \ + -env $(ENVIRONMENT) + + condition: ne(variables['terraformAction'], 'destroy') + failOnStderr: true + displayName: 'Terraform $(terraformAction)' + + - bash: | + git clone --branch $(LANDINGZONE_BRANCH) https://github.com/Azure/caf-terraform-landingzones.git ${BUILD_REPOSITORY_LOCALPATH}/public 2>/dev/null + + az login --identity -u $(msi-resource-id) + + /tf/rover/rover.sh -lz ${BUILD_REPOSITORY_LOCALPATH}$(landingZonePath) \ + -tfstate $(tfstateName) \ + -var-folder ${BUILD_REPOSITORY_LOCALPATH}$(configPath) \ + -parallelism=30 \ + -level $(level) \ + -a $(terraformAction) \ + -auto-approve \ + -env $(ENVIRONMENT) + + condition: eq(variables['terraformAction'], 'destroy') + failOnStderr: true + displayName: 'Terraform destroy' diff --git a/caf_solution/add-ons/azure_devops_v1/variables.tf b/caf_solution/add-ons/azure_devops_v1/variables.tf new file mode 100644 index 000000000..b1edc771b --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/variables.tf @@ -0,0 +1,117 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is propulated by the rover. subscription id hosting the remote tfstates" +} + +variable "global_settings" { + default = {} +} +variable "tenant_id" {} +variable "landingzone" { +} +variable "rover_version" { + default = null +} + +variable "logged_user_objectId" { + default = null +} +variable "logged_aad_app_objectId" { + default = null +} +variable "tags" { + default = null +} +variable "app_service_environments" { + default = {} +} +variable "app_service_plans" { + default = {} +} +variable "app_services" { + default = {} +} +variable "diagnostics_definition" { + default = {} +} +variable "resource_groups" { + default = {} +} +variable "network_security_group_definition" { + default = {} +} +variable "vnets" { + default = {} +} +variable "azurerm_redis_caches" { + default = {} +} +variable "mssql_servers" { + default = {} +} +variable "storage_accounts" { + default = {} +} +variable "storage_account_blobs" { + default = {} +} +variable "azuread_groups" { + default = {} +} +variable "keyvaults" { + default = {} +} +variable "keyvault_access_policies" { + default = {} +} +variable "keyvault_access_policies_azuread_apps" { + default = {} +} +variable "virtual_machines" { + default = {} +} +variable "diagnostic_storage_accounts" { + default = {} +} +variable "virtual_machine_extension_scripts" { + default = {} +} +variable "azure_devops" { + default = {} +} +variable "role_mapping" { + default = {} +} +variable "custom_role_definitions" { + default = {} +} +variable "azuread_apps" { + default = {} +} +variable "dynamic_keyvault_secrets" { + default = {} +} + +### new +variable "organization_agent_pools" { + default = {} +} +variable "organization_url" {} +variable "projects" { + default = {} +} +variable "project_agent_pools" { + default = {} +} +variable "service_endpoints" { + default = {} +} \ No newline at end of file From 2336f3431490b62123c6f0f8832192da06732882 Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Mon, 3 May 2021 06:27:41 +0000 Subject: [PATCH 02/65] update devops config based on devops v1 LZ --- .../add-ons/azure_devops_v1/azdo_pipelines.tf | 87 ++++++++++--------- .../azure_devops_v1/azdo_service_endpoint.tf | 54 ++++++------ .../azure_devops_v1/azdo_variable_groups.tf | 54 ++++++------ .../add-ons/azure_devops_v1/variables.tf | 6 ++ 4 files changed, 104 insertions(+), 97 deletions(-) diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf index cb9178b75..38c5a83e2 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf @@ -1,43 +1,44 @@ -# data "azuredevops_git_repositories" "repos" { -# project_id = data.azuredevops_project.project.id -# } - - -# locals { -# repositories = zipmap(tolist(data.azuredevops_git_repositories.repos.repositories.*.name), tolist(data.azuredevops_git_repositories.repos.repositories)) -# } - -# resource "azuredevops_build_definition" "build_definition" { - -# for_each = try(var.azure_devops.pipelines, {}) -# project_id = data.azuredevops_project.project.id -# name = each.value.name -# path = each.value.folder - -# variable_groups = lookup(each.value, "variable_group_keys", null) == null ? null : [ -# for key in each.value.variable_group_keys : -# azuredevops_variable_group.variable_group[key].id -# ] - -# repository { -# repo_id = local.repositories[each.value.git_repo_name].id -# repo_type = each.value.repo_type -# yml_path = each.value.yaml -# branch_name = lookup(each.value, "branch_name", null) -# # service_connection_id = lookup(each.value, "repo_type", null) == "github" ? null : azuredevops_serviceendpoint_azurerm.github[each.value.service_connection_key].id -# } - -# ci_trigger { -# use_yaml = true -# } - -# dynamic "variable" { -# for_each = try(each.value.variables, {}) - -# content { -# name = variable.key -# value = variable.value -# } -# } - -# } +data "azuredevops_git_repositories" "repos" { + for_each = var.projects + + project_id = data.azuredevops_project.project[each.key].id +} + +output "repos"{ + value = data.azuredevops_git_repositories.repos +} + +resource "azuredevops_build_definition" "build_definition" { + + for_each = var.pipelines + project_id = data.azuredevops_project.project[each.value.project_key].id + name = each.value.name + path = each.value.folder + + variable_groups = lookup(each.value, "variable_group_keys", null) == null ? null : [ + for key in each.value.variable_group_keys : + azuredevops_variable_group.variable_group[key].id + ] + + repository { + repo_id = data.azuredevops_git_repositories.repos[each.value.project_key].repositories[0].id + repo_type = each.value.repo_type + yml_path = each.value.yaml + branch_name = lookup(each.value, "branch_name", null) + # service_connection_id = lookup(each.value, "repo_type", null) == "github" ? null : azuredevops_serviceendpoint_azurerm.github[each.value.service_connection_key].id + } + + ci_trigger { + use_yaml = true + } + + dynamic "variable" { + for_each = try(each.value.variables, {}) + + content { + name = variable.key + value = variable.value + } + } + +} diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf index b9b469fde..e5fca29ff 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf @@ -1,34 +1,34 @@ -# data "azurerm_key_vault_secret" "client_secret" { -# for_each = var.service_endpoints +data "azurerm_key_vault_secret" "client_secret" { + for_each = var.service_endpoints -# name = format("%s-client-secret", local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].keyvaults[each.value.keyvault_key].secret_name_client_secret) -# key_vault_id = local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].keyvaults[each.value.keyvault_key].id -# } + name = format("%s-client-secret", local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].keyvaults[each.value.keyvault_key].secret_name_client_secret) + key_vault_id = local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].keyvaults[each.value.keyvault_key].id +} -# resource "azuredevops_serviceendpoint_azurerm" "azure" { -# for_each = var.service_endpoints +resource "azuredevops_serviceendpoint_azurerm" "azure" { + for_each = var.service_endpoints -# project_id = data.azuredevops_project.project[each.value.project_key].id -# service_endpoint_name = each.value.endpoint_name -# credentials { -# serviceprincipalid = local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].azuread_application.application_id -# serviceprincipalkey = data.azurerm_key_vault_secret.client_secret[each.key].value -# } -# azurerm_spn_tenantid = local.remote.aad_apps[each.value.lz_key][each.value.azuread_apps_key].tenant_id -# azurerm_subscription_id = each.value.subscription.id -# azurerm_subscription_name = each.value.subscription.name -# } + project_id = data.azuredevops_project.project[each.value.project_key].id + service_endpoint_name = each.value.endpoint_name + credentials { + serviceprincipalid = local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].azuread_application.application_id + serviceprincipalkey = data.azurerm_key_vault_secret.client_secret[each.key].value + } + azurerm_spn_tenantid = local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].tenant_id + azurerm_subscription_id = each.value.subscription.id + azurerm_subscription_name = each.value.subscription.name +} -# # -# # Grant acccess to service endpoint to all pipelines in the project -# # +# +# Grant acccess to service endpoint to all pipelines in the project +# -# resource "azuredevops_resource_authorization" "endpoint" { -# for_each = var.service_endpoints +resource "azuredevops_resource_authorization" "endpoint" { + for_each = var.service_endpoints -# project_id = data.azuredevops_project.project.id -# resource_id = azuredevops_serviceendpoint_azurerm.azure[each.key].id -# type = "endpoint" -# authorized = true -# } \ No newline at end of file + project_id = data.azuredevops_project.project[each.value.project_key].id + resource_id = azuredevops_serviceendpoint_azurerm.azure[each.key].id + type = "endpoint" + authorized = true +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf index 597948fe1..73edb1f5a 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf @@ -4,36 +4,36 @@ # # + vso.buid (update) # # + vso.build_execute (destroy) # # -# resource "azuredevops_variable_group" "variable_group" { -# for_each = try(var.azure_devops.variable_groups, {}) +resource "azuredevops_variable_group" "variable_group" { + for_each = var.variable_groups -# project_id = data.azuredevops_project.project.id -# name = each.value.name -# description = try(each.value.description, null) -# allow_access = try(each.value.allow_access, false) + project_id = data.azuredevops_project.project[each.value.project_key].id + name = each.value.name + description = try(each.value.description, null) + allow_access = try(each.value.allow_access, false) -# dynamic "key_vault" { -# for_each = lookup(each.value, "keyvault", null) == null ? [] : [1] + dynamic "key_vault" { + for_each = lookup(each.value, "keyvault", null) == null ? [] : [1] -# content { -# name = try(each.value.keyvault.lz_key, null) == null ? local.combined.keyvaults[var.landingzone.key][each.value.keyvault.keyvault_key].name : local.combined.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.keyvault_key].name -# service_endpoint_id = azuredevops_serviceendpoint_azurerm.azure[each.value.keyvault.serviceendpoint_key].id -# } -# } + content { + name = local.remote.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.keyvault_key].name + service_endpoint_id = azuredevops_serviceendpoint_azurerm.azure[each.value.keyvault.serviceendpoint_key].id + } + } -# dynamic "variable" { -# for_each = { -# for key, variable in each.value.variables : key => { -# name = key == "name" ? variable : key -# value = key == "name" ? null : variable -# } -# } + dynamic "variable" { + for_each = { + for key, variable in each.value.variables : key => { + name = key == "name" ? variable : key + value = key == "name" ? null : variable + } + } -# content { -# # When used with Keyvault, the name must be the keyvault secret name and value must not be set -# name = variable.value.name -# value = variable.value.value -# } -# } + content { + # When used with Keyvault, the name must be the keyvault secret name and value must not be set + name = variable.value.name + value = variable.value.value + } + } -# } +} diff --git a/caf_solution/add-ons/azure_devops_v1/variables.tf b/caf_solution/add-ons/azure_devops_v1/variables.tf index b1edc771b..7444f09db 100644 --- a/caf_solution/add-ons/azure_devops_v1/variables.tf +++ b/caf_solution/add-ons/azure_devops_v1/variables.tf @@ -114,4 +114,10 @@ variable "project_agent_pools" { } variable "service_endpoints" { default = {} +} +variable "variable_groups" { + default = {} +} +variable "pipelines" { + default = {} } \ No newline at end of file From 4fcd7a74cd34c6fd4bffffa689364cd0b38ba65f Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Mon, 3 May 2021 07:00:51 +0000 Subject: [PATCH 03/65] update pipeline to fix repo issue --- .../add-ons/azure_devops_v1/azdo_pipelines.tf | 25 +++++++++++-------- .../azure_devops_v1/azuredevops_projects.tf | 5 +--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf index 38c5a83e2..1652cb4ae 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf @@ -4,13 +4,9 @@ data "azuredevops_git_repositories" "repos" { project_id = data.azuredevops_project.project[each.key].id } -output "repos"{ - value = data.azuredevops_git_repositories.repos -} - resource "azuredevops_build_definition" "build_definition" { - for_each = var.pipelines + project_id = data.azuredevops_project.project[each.value.project_key].id name = each.value.name path = each.value.folder @@ -20,12 +16,19 @@ resource "azuredevops_build_definition" "build_definition" { azuredevops_variable_group.variable_group[key].id ] - repository { - repo_id = data.azuredevops_git_repositories.repos[each.value.project_key].repositories[0].id - repo_type = each.value.repo_type - yml_path = each.value.yaml - branch_name = lookup(each.value, "branch_name", null) - # service_connection_id = lookup(each.value, "repo_type", null) == "github" ? null : azuredevops_serviceendpoint_azurerm.github[each.value.service_connection_key].id + dynamic "repository" { + for_each = { + for key, value in try(data.azuredevops_git_repositories.repos[each.value.project_key].repositories, {}) : key => value + if value.name == each.value.git_repo_name + } + + content { + repo_id = repository.value.id + repo_type = each.value.repo_type + yml_path = each.value.yaml + branch_name = lookup(each.value, "branch_name", null) + # service_connection_id = lookup(each.value, "repo_type", null) == "github" ? null : azuredevops_serviceendpoint_azurerm.github[each.value.service_connection_key].id + } } ci_trigger { diff --git a/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf b/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf index e12926c46..ebe0afc6c 100644 --- a/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf +++ b/caf_solution/add-ons/azure_devops_v1/azuredevops_projects.tf @@ -15,10 +15,7 @@ resource "azuredevops_project" "project" { data "azuredevops_project" "project" { depends_on = [azuredevops_project.project] - for_each = { - for key, value in var.projects : key => value - if try(value.features, null) != null - } + for_each = var.projects name = each.value.name } From 59d6f5adae26f46c02b24fe7dbf02a92e0764016 Mon Sep 17 00:00:00 2001 From: Jor Seng Date: Wed, 5 May 2021 13:05:47 +0000 Subject: [PATCH 04/65] Add output agent_pools --- caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf b/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf index 53a620d16..2d5b99a13 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_agent_pools.tf @@ -16,6 +16,11 @@ resource "azuredevops_agent_pool" "pool" { pool_type = try(each.value.pool_type, null) } +output agent_pools { + value = azuredevops_agent_pool.pool +} + + # # add the agent pools into the project # From 6b7931f52675fc7f8c47f22c32c9f7657dab42c5 Mon Sep 17 00:00:00 2001 From: Jor Seng Date: Wed, 5 May 2021 13:06:43 +0000 Subject: [PATCH 05/65] Update azuread_applications output attribute --- caf_solution/local.remote.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index b369b0f5d..b052b6fc3 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -28,7 +28,7 @@ locals { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].availability_sets, {})) } azuread_applications = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aad_apps, {})) } azuread_groups = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) From 82095df7a642f79c83ea4efaa9407c254ab24114 Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Thu, 6 May 2021 09:45:47 +0000 Subject: [PATCH 06/65] added support for remote repo for pipeline --- caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf index 1652cb4ae..0716577fd 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf @@ -18,7 +18,7 @@ resource "azuredevops_build_definition" "build_definition" { dynamic "repository" { for_each = { - for key, value in try(data.azuredevops_git_repositories.repos[each.value.project_key].repositories, {}) : key => value + for key, value in try(data.azuredevops_git_repositories.repos[each.value.repo_project_key].repositories, {}) : key => value if value.name == each.value.git_repo_name } From 44b747fb706148716555635a4410d54873e0c39c Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Fri, 7 May 2021 07:26:38 +0000 Subject: [PATCH 07/65] default to project key for repo in same poject --- caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf index 0716577fd..9d7e2ed70 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_pipelines.tf @@ -18,7 +18,7 @@ resource "azuredevops_build_definition" "build_definition" { dynamic "repository" { for_each = { - for key, value in try(data.azuredevops_git_repositories.repos[each.value.repo_project_key].repositories, {}) : key => value + for key, value in try(data.azuredevops_git_repositories.repos[try(each.value.repo_project_key, each.value.project_key)].repositories, {}) : key => value if value.name == each.value.git_repo_name } From ea3ddb4658a1109c7afd7c2b7d93801d7a6ae6b2 Mon Sep 17 00:00:00 2001 From: lolorol Date: Wed, 19 May 2021 04:46:05 +0000 Subject: [PATCH 08/65] use integration branch --- caf_launchpad/landingzone.tf | 6 +++--- caf_solution/landingzone.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index 915158c21..e7ae25669 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -1,8 +1,8 @@ module "launchpad" { - source = "aztfmod/caf/azurerm" - version = "~>5.3.2" + # source = "aztfmod/caf/azurerm" + # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" azuread_api_permissions = var.azuread_api_permissions diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 65c64c65e..5c69d7fa2 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -1,8 +1,8 @@ module "solution" { - source = "aztfmod/caf/azurerm" - version = "~>5.3.2" + # source = "aztfmod/caf/azurerm" + # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" azuread_api_permissions = var.azuread_api_permissions From cb2ab22cffceedbd9f21d3c166f8a641b7be8be2 Mon Sep 17 00:00:00 2001 From: lolorol Date: Wed, 19 May 2021 14:18:36 +0000 Subject: [PATCH 09/65] Update azuread in landingzone --- caf_solution/landingzone.tf | 1 + caf_solution/local.azuread.tf | 10 ++++++++++ caf_solution/variables.azuread.tf | 12 ++++++++++++ caf_solution/vm_extensions.tf | 8 ++++++-- 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 caf_solution/local.azuread.tf create mode 100644 caf_solution/variables.azuread.tf diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 5c69d7fa2..9cfc3326c 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -5,6 +5,7 @@ module "solution" { source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" + azuread = local.azuread azuread_api_permissions = var.azuread_api_permissions azuread_apps = var.azuread_apps azuread_groups = var.azuread_groups diff --git a/caf_solution/local.azuread.tf b/caf_solution/local.azuread.tf new file mode 100644 index 000000000..dca335922 --- /dev/null +++ b/caf_solution/local.azuread.tf @@ -0,0 +1,10 @@ +locals { + azuread = merge( + var.azuread, + { + azuread_applications = var.azuread_applications + azuread_service_principals = var.azuread_service_principals + azuread_service_principal_passwords = var.azuread_service_principal_passwords + } + ) +} diff --git a/caf_solution/variables.azuread.tf b/caf_solution/variables.azuread.tf new file mode 100644 index 000000000..3ddca35c4 --- /dev/null +++ b/caf_solution/variables.azuread.tf @@ -0,0 +1,12 @@ +variable "azuread" { + default = {} +} +variable "azuread_applications" { + default = {} +} +variable "azuread_service_principals" { + default = {} +} +variable "azuread_service_principal_passwords" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/vm_extensions.tf b/caf_solution/vm_extensions.tf index e54d3290a..04266e2ff 100644 --- a/caf_solution/vm_extensions.tf +++ b/caf_solution/vm_extensions.tf @@ -3,7 +3,9 @@ # module "vm_extension_monitoring_agent" { - source = "../../modules/compute/virtual_machine_extensions" + source = "aztfmod/caf/azurerm//modules/compute/virtual_machine_extensions" + version = "~>5.3.2" + # if you are not running CAF modules locally, change the source to "github.com/aztfmod/terraform-azurerm-caf/modules/compute/virtual_machine_extensions" depends_on = [module.solution] #refer landingzone.tf for the correct module name. @@ -22,7 +24,9 @@ module "vm_extension_monitoring_agent" { } module "vm_extension_diagnostics" { - source = "../../modules/compute/virtual_machine_extensions" + source = "aztfmod/caf/azurerm//modules/compute/virtual_machine_extensions" + version = "~>5.3.2" + # if you are not running CAF modules locally, change the source to "github.com/aztfmod/terraform-azurerm-caf/modules/compute/virtual_machine_extensions" depends_on = [module.solution] #refer landingzone.tf for the correct module name. From 78680639038e6b4a67b5e1130c72cdeed3959894 Mon Sep 17 00:00:00 2001 From: Hieu Nguyen Nhu <5441003+hieumoscow@users.noreply.github.com> Date: Wed, 19 May 2021 17:07:48 +0000 Subject: [PATCH 10/65] Refactor AKS addons --- .../add-ons/aad-pod-identity/variables.tf | 2 +- .../aks-pod-identity-assignment.tf | 10 + .../local.remote_tfstates.tf | 3 + .../add-ons/aks-secure-baseline/providers.tf | 9 + .../add-ons/aks-secure-baseline/variables.tf | 5 +- .../add-ons/aks_applications/app/main.tf | 20 +- .../add-ons/aks_applications/app/module.tf | 5 +- .../add-ons/aks_applications/app/variables.tf | 12 +- .../add-ons/aks_applications/applications.tf | 24 +- .../add-ons/aks_applications/kustomization.tf | 31 +++ .../kustomize/kustomization_build.tf | 16 ++ .../aks_applications/kustomize/main.tf | 7 + .../aks_applications/kustomize/variables.tf | 2 + .../locals.remote_tfstates.tf | 32 +-- caf_solution/add-ons/aks_applications/main.tf | 68 +---- .../add-ons/aks_applications/providers.tf | 54 ++++ .../add-ons/aks_applications/variables.tf | 36 ++- .../aks_azure_devops_agents/app/main.tf | 13 + .../aks_azure_devops_agents/app/module.tf | 47 ++++ .../aks_azure_devops_agents/app/output.tf | 0 .../aks_azure_devops_agents/app/variables.tf | 13 + .../aks_azure_devops_agents/applications.tf | 7 + .../aks_azure_devops_agents/backend.azurerm | 4 + .../aks_azure_devops_agents/kustomization.tf | 245 ++++++++++++++++++ .../kustomize/kustomization_build.tf | 16 ++ .../aks_azure_devops_agents/kustomize/main.tf | 7 + .../kustomize/variables.tf | 2 + .../locals.remote_tfstates.tf | 63 +++++ .../add-ons/aks_azure_devops_agents/main.tf | 21 ++ .../add-ons/aks_azure_devops_agents/output.tf | 0 .../aks_azure_devops_agents/providers.tf | 54 ++++ .../aks_azure_devops_agents/variables.tf | 52 ++++ .../yamls/akvs-secret.yaml | 14 + .../yamls/azdopat-secret.yaml | 6 + .../yamls/placeholderagent.yaml | 53 ++++ .../yamls/placeholderjob.yaml | 38 +++ .../yamls/roverjob.yaml | 48 ++++ .../yamls/secretstorecsi.yaml | 22 ++ .../add-ons/azure_devops_v1/output.tf | 6 +- 39 files changed, 939 insertions(+), 128 deletions(-) create mode 100644 caf_solution/add-ons/aks_applications/kustomization.tf create mode 100644 caf_solution/add-ons/aks_applications/kustomize/kustomization_build.tf create mode 100644 caf_solution/add-ons/aks_applications/kustomize/main.tf create mode 100644 caf_solution/add-ons/aks_applications/kustomize/variables.tf create mode 100644 caf_solution/add-ons/aks_applications/providers.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/app/main.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/app/module.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/app/output.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/applications.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/backend.azurerm create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/locals.remote_tfstates.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/main.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/output.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/providers.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/variables.tf create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/yamls/akvs-secret.yaml create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/yamls/azdopat-secret.yaml create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderagent.yaml create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderjob.yaml create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/yamls/roverjob.yaml create mode 100644 caf_solution/add-ons/aks_azure_devops_agents/yamls/secretstorecsi.yaml diff --git a/caf_solution/add-ons/aad-pod-identity/variables.tf b/caf_solution/add-ons/aad-pod-identity/variables.tf index 971033737..3e960c0c7 100644 --- a/caf_solution/add-ons/aad-pod-identity/variables.tf +++ b/caf_solution/add-ons/aad-pod-identity/variables.tf @@ -4,7 +4,7 @@ variable "lower_container_name" {} variable "lower_resource_group_name" {} variable "tfstate_subscription_id" { - description = "This value is propulated by the rover. subscription id hosting the remote tfstates" + description = "This value is populated by the rover. subscription id hosting the remote tfstates" } variable "tfstate_storage_account_name" {} variable "tfstate_container_name" {} diff --git a/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf index 96c672df0..6635d9494 100644 --- a/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf +++ b/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf @@ -56,3 +56,13 @@ locals { ) : format("%s-%s", msi.key, msi.msi_key) => msi } } + +resource "azurerm_key_vault_access_policy" "keyvault_policy" { + provider = azurerm.launchpad + for_each = var.keyvaults + + key_vault_id = local.remote.keyvaults[each.value.lz_key][each.value.key].id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_clusters[var.aks_cluster_key].key].kubelet_identity[0].object_id + secret_permissions = each.value.secret_permissions +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf b/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf index d458c18a1..c30b190de 100644 --- a/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf +++ b/caf_solution/add-ons/aks-secure-baseline/local.remote_tfstates.tf @@ -43,6 +43,9 @@ locals { aks_clusters = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) } + keyvaults = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) + } managed_identities = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) } diff --git a/caf_solution/add-ons/aks-secure-baseline/providers.tf b/caf_solution/add-ons/aks-secure-baseline/providers.tf index 06e22a736..8d20ad216 100644 --- a/caf_solution/add-ons/aks-secure-baseline/providers.tf +++ b/caf_solution/add-ons/aks-secure-baseline/providers.tf @@ -4,6 +4,13 @@ provider "azurerm" { } } +provider "azurerm" { + alias = "launchpad" + subscription_id = var.tfstate_subscription_id + features { + } +} + provider "kubernetes" { host = local.k8sconfigs[var.aks_cluster_key].host username = local.k8sconfigs[var.aks_cluster_key].username @@ -31,6 +38,8 @@ locals { } } +data "azurerm_client_config" "current" {} + # Get kubeconfig from AKS clusters data "azurerm_kubernetes_cluster" "kubeconfig" { for_each = var.aks_clusters diff --git a/caf_solution/add-ons/aks-secure-baseline/variables.tf b/caf_solution/add-ons/aks-secure-baseline/variables.tf index 971033737..010f75139 100644 --- a/caf_solution/add-ons/aks-secure-baseline/variables.tf +++ b/caf_solution/add-ons/aks-secure-baseline/variables.tf @@ -33,4 +33,7 @@ variable "managed_identities" { description = "Map of the user managed identities." } -variable "aad_pod_identity" {} \ No newline at end of file +variable "aad_pod_identity" {} +variable "keyvaults" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/app/main.tf b/caf_solution/add-ons/aks_applications/app/main.tf index 1191eedce..c45957ba5 100644 --- a/caf_solution/add-ons/aks_applications/app/main.tf +++ b/caf_solution/add-ons/aks_applications/app/main.tf @@ -1,7 +1,13 @@ -provider "kubernetes" { - alias = "k8s" -} - -provider "helm" { - alias = "helm" -} +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + } + helm = { + source = "hashicorp/helm" + } + kustomization = { + source = "kbst/kustomization" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/app/module.tf b/caf_solution/add-ons/aks_applications/app/module.tf index 2d4288893..138110a32 100644 --- a/caf_solution/add-ons/aks_applications/app/module.tf +++ b/caf_solution/add-ons/aks_applications/app/module.tf @@ -6,7 +6,6 @@ resource "kubernetes_namespace" "namespaces" { name = each.value.name } - provider = kubernetes.k8s } # https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release @@ -22,6 +21,7 @@ resource "helm_release" "charts" { timeout = try(each.value.timeout, 900) skip_crds = try(each.value.skip_crds, false) create_namespace = try(each.value.create_namespace, false) + values = try(each.value.values, null) dynamic "set" { for_each = try(each.value.sets, {}) @@ -39,9 +39,8 @@ resource "helm_release" "charts" { } } - provider = helm.helm - depends_on = [kubernetes_namespace.namespaces] + # depends_on = [kubernetes_namespace.namespaces] # values = [ # "${file("values.yaml")}" # ] diff --git a/caf_solution/add-ons/aks_applications/app/variables.tf b/caf_solution/add-ons/aks_applications/app/variables.tf index 4a0225192..6c2baef70 100644 --- a/caf_solution/add-ons/aks_applications/app/variables.tf +++ b/caf_solution/add-ons/aks_applications/app/variables.tf @@ -1,5 +1,13 @@ variable "cluster" {} -variable "namespaces" {} +variable "namespaces" { + default = {} +} -variable "helm_charts" {} \ No newline at end of file +variable "helm_charts" { + default = {} +} + +variable "kuztomization_settings" { + default = {} +} diff --git a/caf_solution/add-ons/aks_applications/applications.tf b/caf_solution/add-ons/aks_applications/applications.tf index b1f674491..d36d39b0d 100644 --- a/caf_solution/add-ons/aks_applications/applications.tf +++ b/caf_solution/add-ons/aks_applications/applications.tf @@ -1,27 +1,7 @@ -module "app1" { +module "app" { source = "./app" - for_each = try(local.clusters[var.cluster_re1_key], null) != null ? { (var.cluster_re1_key) = local.clusters[var.cluster_re1_key] } : {} - cluster = each.value + cluster = local.aks_clusters[var.aks_cluster_key] namespaces = var.namespaces helm_charts = var.helm_charts - - providers = { - kubernetes.k8s = kubernetes.k8s1 - helm.helm = helm.helm1 - } } - -module "app2" { - source = "./app" - for_each = try(local.clusters[var.cluster_re2_key], null) != null ? { (var.cluster_re2_key) = local.clusters[var.cluster_re2_key] } : {} - - cluster = each.value - namespaces = var.namespaces - helm_charts = var.helm_charts - - providers = { - kubernetes.k8s = kubernetes.k8s2 - helm.helm = helm.helm2 - } -} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/kustomization.tf b/caf_solution/add-ons/aks_applications/kustomization.tf new file mode 100644 index 000000000..478352120 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomization.tf @@ -0,0 +1,31 @@ +module "kustomization" { + source = "./kustomize" + for_each = try(data.kustomization_overlay.manifest, {}) + + settings = each.value + +} + +data "kustomization_overlay" "manifest" { + for_each = var.kustomization_overlays + + resources = each.value.resources + + namespace = each.value.namespace + + dynamic "patches"{ + for_each = try(each.value.patches, {}) + content { + patch = patches.value.patch + target = patches.value.target + } + } + kustomize_options = { + load_restrictor = "none" + } +} + +output "manifests" { + value = data.kustomization_overlay.manifest +} + diff --git a/caf_solution/add-ons/aks_applications/kustomize/kustomization_build.tf b/caf_solution/add-ons/aks_applications/kustomize/kustomization_build.tf new file mode 100644 index 000000000..985bd96f5 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomize/kustomization_build.tf @@ -0,0 +1,16 @@ +resource "kustomization_resource" "p0" { + for_each = var.settings.ids_prio[0] + manifest = var.settings.manifests[each.value] +} + +resource "kustomization_resource" "p1" { + depends_on = [kustomization_resource.p0] + for_each = var.settings.ids_prio[1] + manifest = var.settings.manifests[each.value] +} + +resource "kustomization_resource" "p2" { + depends_on = [kustomization_resource.p1] + for_each = var.settings.ids_prio[2] + manifest = var.settings.manifests[each.value] +} diff --git a/caf_solution/add-ons/aks_applications/kustomize/main.tf b/caf_solution/add-ons/aks_applications/kustomize/main.tf new file mode 100644 index 000000000..e65c6fa22 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomize/main.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + kustomization = { + source = "kbst/kustomization" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/kustomize/variables.tf b/caf_solution/add-ons/aks_applications/kustomize/variables.tf new file mode 100644 index 000000000..f5c321890 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/kustomize/variables.tf @@ -0,0 +1,2 @@ +variable "settings" { +} diff --git a/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf b/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf index cf16e86c6..d4c3a5da0 100644 --- a/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf +++ b/caf_solution/add-ons/aks_applications/locals.remote_tfstates.tf @@ -19,8 +19,9 @@ data "terraform_remote_state" "remote" { backend = var.landingzone.backend_type config = { storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name - container_name = local.landingzone[try(each.value.level, "current")].container_name + container_name = try(each.value.container, local.landingzone[try(each.value.level, "current")].container_name) resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id key = each.value.tfstate } } @@ -30,25 +31,24 @@ locals { "landingzone" = var.landingzone.key } - tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) - - global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.global_settings + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics remote = { + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + global_settings = local.global_settings + diagnostics = local.diagnostics + + aks_clusters = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.aks_clusters[key], {})) + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) } - } - - clusters = local.remote.aks_clusters[var.landingzone.global_settings_key] - k8sconfigs = { for key, value in values(local.clusters) : key => { - host = value.enable_rbac ? value.kube_admin_config.0.host : value.kube_config.0.host - username = value.enable_rbac ? value.kube_admin_config.0.username : value.kube_config.0.username - password = value.enable_rbac ? value.kube_admin_config.0.password : value.kube_config.0.password - client_certificate = value.enable_rbac ? base64decode(value.kube_admin_config.0.client_certificate) : base64decode(value.kube_config.0.client_certificate) - client_key = value.enable_rbac ? base64decode(value.kube_admin_config.0.client_key) : base64decode(value.kube_config.0.client_key) - cluster_ca_certificate = value.enable_rbac ? base64decode(value.kube_admin_config.0.cluster_ca_certificate) : base64decode(value.kube_config.0.cluster_ca_certificate) + managed_identities = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) + } + vnets = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) } } -} +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/main.tf b/caf_solution/add-ons/aks_applications/main.tf index 2222bc4d2..6b7201fb4 100644 --- a/caf_solution/add-ons/aks_applications/main.tf +++ b/caf_solution/add-ons/aks_applications/main.tf @@ -2,74 +2,20 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 2.43" + version = "~> 2.55.0" } kubernetes = { source = "hashicorp/kubernetes" - version = "~> 1.13.2" + version = "~> 2.0.2" } helm = { source = "hashicorp/helm" - version = "~> 1.3.0" + version = "~> 2.1.2" } - } - required_version = ">= 0.13" -} - - -provider "azurerm" { - features { - key_vault { - purge_soft_delete_on_destroy = true + kustomization = { + source = "kbst/kustomization" + version = "~> 0.5.0" } } -} - - -provider "kubernetes" { - load_config_file = false - host = try(local.k8sconfigs[0].host, null) - username = try(local.k8sconfigs[0].username, null) - password = try(local.k8sconfigs[0].password, null) - client_certificate = try(local.k8sconfigs[0].client_certificate, null) - client_key = try(local.k8sconfigs[0].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[0].cluster_ca_certificate, null) - alias = "k8s1" -} - -provider "helm" { - kubernetes { - load_config_file = false - host = try(local.k8sconfigs[0].host, null) - username = try(local.k8sconfigs[0].username, null) - password = try(local.k8sconfigs[0].password, null) - client_certificate = try(local.k8sconfigs[0].client_certificate, null) - client_key = try(local.k8sconfigs[0].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[0].cluster_ca_certificate, null) - } - alias = "helm1" -} - -provider "kubernetes" { - load_config_file = false - host = try(local.k8sconfigs[1].host, null) - username = try(local.k8sconfigs[1].username, null) - password = try(local.k8sconfigs[1].password, null) - client_certificate = try(local.k8sconfigs[1].client_certificate, null) - client_key = try(local.k8sconfigs[1].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[1].cluster_ca_certificate, null) - alias = "k8s2" -} - -provider "helm" { - kubernetes { - load_config_file = false - host = try(local.k8sconfigs[1].host, null) - username = try(local.k8sconfigs[1].username, null) - password = try(local.k8sconfigs[1].password, null) - client_certificate = try(local.k8sconfigs[1].client_certificate, null) - client_key = try(local.k8sconfigs[1].client_key, null) - cluster_ca_certificate = try(local.k8sconfigs[1].cluster_ca_certificate, null) - } - alias = "helm2" + required_version = ">= 0.13" } \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/providers.tf b/caf_solution/add-ons/aks_applications/providers.tf new file mode 100644 index 000000000..e8b9dcfc5 --- /dev/null +++ b/caf_solution/add-ons/aks_applications/providers.tf @@ -0,0 +1,54 @@ +provider "azurerm" { + features { + } +} + +provider "kubernetes" { + host = try(local.k8sconfigs[var.aks_cluster_key].host, null) + username = try(local.k8sconfigs[var.aks_cluster_key].username, null) + password = try(local.k8sconfigs[var.aks_cluster_key].password, null) + client_certificate = try(local.k8sconfigs[var.aks_cluster_key].client_certificate, null) + client_key = try(local.k8sconfigs[var.aks_cluster_key].client_key, null) + cluster_ca_certificate = try(local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate, null) +} + +provider "helm" { + kubernetes { + host = local.k8sconfigs[var.aks_cluster_key].host + username = local.k8sconfigs[var.aks_cluster_key].username + password = local.k8sconfigs[var.aks_cluster_key].password + client_certificate = local.k8sconfigs[var.aks_cluster_key].client_certificate + client_key = local.k8sconfigs[var.aks_cluster_key].client_key + cluster_ca_certificate = local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate + } +} + +provider "kustomization" { + kubeconfig_raw = local.k8sconfigs[var.aks_cluster_key].kube_admin_config_raw +} + +locals { + aks_clusters = { + for key, value in var.aks_clusters : key => + local.remote.aks_clusters[value.lz_key][value.key] + } + k8sconfigs = { + for key, value in var.aks_clusters : key => { + kube_admin_config_raw = data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config_raw + host = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.host : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.host + username = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.username : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.username + password = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.password : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.password + client_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_certificate) + client_key = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_key) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_key) + cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.cluster_ca_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.cluster_ca_certificate) + } + } +} + +# Get kubeconfig from AKS clusters +data "azurerm_kubernetes_cluster" "kubeconfig" { + for_each = var.aks_clusters + + name = local.remote.aks_clusters[each.value.lz_key][each.value.key].cluster_name + resource_group_name = local.remote.aks_clusters[each.value.lz_key][each.value.key].resource_group_name +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/variables.tf b/caf_solution/add-ons/aks_applications/variables.tf index 7ad77ea93..a24be3ab1 100644 --- a/caf_solution/add-ons/aks_applications/variables.tf +++ b/caf_solution/add-ons/aks_applications/variables.tf @@ -3,34 +3,42 @@ variable "lower_storage_account_name" {} variable "lower_container_name" {} variable "lower_resource_group_name" {} +variable "tfstate_subscription_id" { + description = "This value is populated by the rover. subscription id hosting the remote tfstates" +} variable "tfstate_storage_account_name" {} variable "tfstate_container_name" {} +variable "tfstate_key" {} variable "tfstate_resource_group_name" {} -# variable tfstate_key {} variable "global_settings" { default = {} } -# variable tenant_id {} -variable "landingzone" {} - -variable "namespaces" {} -variable "tags" { +variable "landingzone" {} +variable "rover_version" { default = null - type = map(any) +} +variable "tags" { + default = {} +} +variable "namespaces" { + default = {} } -variable "helm_charts" {} +variable "helm_charts" { + default = {} +} +variable "aks_clusters" {} -variable "rover_version" { - default = null +variable "aks_cluster_key" { } -variable "cluster_re1_key" { - default = null +variable "kustomization_overlays" { + default = {} } -variable "cluster_re2_key" { - default = null + +variable "kustomization_builds" { + default = {} } \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/main.tf b/caf_solution/add-ons/aks_azure_devops_agents/app/main.tf new file mode 100644 index 000000000..c45957ba5 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/app/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + } + helm = { + source = "hashicorp/helm" + } + kustomization = { + source = "kbst/kustomization" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/module.tf b/caf_solution/add-ons/aks_azure_devops_agents/app/module.tf new file mode 100644 index 000000000..138110a32 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/app/module.tf @@ -0,0 +1,47 @@ +resource "kubernetes_namespace" "namespaces" { + for_each = var.namespaces + metadata { + annotations = try(each.value.annotations, null) + labels = try(each.value.labels, null) + name = each.value.name + } + +} + +# https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release +resource "helm_release" "charts" { + for_each = var.helm_charts + + name = each.value.name + repository = each.value.repository + chart = each.value.chart + + namespace = each.value.namespace + wait = try(each.value.wait, true) + timeout = try(each.value.timeout, 900) + skip_crds = try(each.value.skip_crds, false) + create_namespace = try(each.value.create_namespace, false) + values = try(each.value.values, null) + + dynamic "set" { + for_each = try(each.value.sets, {}) + content { + name = set.key + value = set.value + } + } + + dynamic "set_sensitive" { + for_each = try(each.value.sets_sensitive, {}) + content { + name = set_sensitive.key + value = set_sensitive.value + } + } + + + # depends_on = [kubernetes_namespace.namespaces] + # values = [ + # "${file("values.yaml")}" + # ] +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/output.tf b/caf_solution/add-ons/aks_azure_devops_agents/app/output.tf new file mode 100644 index 000000000..e69de29bb diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf b/caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf new file mode 100644 index 000000000..6c2baef70 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf @@ -0,0 +1,13 @@ +variable "cluster" {} + +variable "namespaces" { + default = {} +} + +variable "helm_charts" { + default = {} +} + +variable "kuztomization_settings" { + default = {} +} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/applications.tf b/caf_solution/add-ons/aks_azure_devops_agents/applications.tf new file mode 100644 index 000000000..d36d39b0d --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/applications.tf @@ -0,0 +1,7 @@ +module "app" { + source = "./app" + + cluster = local.aks_clusters[var.aks_cluster_key] + namespaces = var.namespaces + helm_charts = var.helm_charts +} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/backend.azurerm b/caf_solution/add-ons/aks_azure_devops_agents/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf new file mode 100644 index 000000000..5a6a941af --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf @@ -0,0 +1,245 @@ +data "kustomization_overlay" "manifest" { + for_each = var.kustomization_overlays + + resources = each.value.resources + + namespace = each.value.namespace + + dynamic "patches"{ + for_each = try(each.value.patches, {}) + content { + patch = patches.value.patch + target = patches.value.target + } + } + # kustomize_options = { + # load_restrictor = "none" + # } +} + +output "manifests" { + value = data.kustomization_overlay.manifest +} + + +# module "kustomization_azdopat-secret" { +# source = "./kustomize" + +# settings = data.kustomization_overlay.azdopat-secret + +# } + +# data "kustomization_overlay" "azdopat-secret" { +# resources = [ +# "yamls/azdopat-secret.yaml", +# ] + +# namespace = var.agent_pools.namespace + +# patches { +# patch = <<-EOF +# - op: replace +# path: /data/personalAccessToken +# value: "dDVjYmljc2R0Y3Juc2RlZmh1cnU2bHBueHdzZ2hxbjdhc2JnMjVkZ2E0bW16dGdldHgzYQ==" +# EOF +# target = { +# kind = "Secret" +# name = "azdopat-secret" +# } +# } +# } + +module "kustomization_azdopat-secret" { + source = "./kustomize" + + settings = data.kustomization_overlay.azdopat-secret + +} + +data "kustomization_overlay" "azdopat-secret" { + resources = [ + "yamls/akvs-secret.yaml", + ] + + namespace = var.agent_pools.namespace + + patches { + patch = <<-EOF + - op: replace + path: /spec/vault/name + value: "${local.remote.keyvaults[var.keyvault.lz_key][var.keyvault.key].name}" + EOF + target = { + kind = "AzureKeyVaultSecret" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/vault/object/name + value: "${var.keyvault.secret_name}" + EOF + target = { + kind = "AzureKeyVaultSecret" + } + } + +} + +module "kustomization" { + source = "./kustomize" + for_each = try(data.kustomization_overlay.roverjob, {}) + + settings = each.value + +} +data "kustomization_overlay" "roverjob" { + for_each = local.remote.agent_pools[var.agent_pools.lz_key] + + resources = [ + "yamls/roverjob.yaml", + ] + + namespace = var.agent_pools.namespace + + patches { + patch = <<-EOF + - op: replace + path: /metadata/name + value: "azdevops-${replace(each.key,"_","-")}" + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/metadata/labels/aadpodidbinding + value: ${each.value.name} + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/spec/containers/0/env/0/value + value: "https://dev.azure.com/afopssre" + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/spec/containers/0/env/2/value + value: ${each.value.name} + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/jobTargetRef/template/spec/containers/0/image + value: "${var.agent_pools.image}" + EOF + target = { + kind = "ScaledJob" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/triggers/0/metadata/poolID + value: "${each.value.id}" + EOF + target = { + kind = "ScaledJob" + } + } +} + + +module "kustomization_placeholderagent" { + source = "./kustomize" + for_each = try(data.kustomization_overlay.placeholderjob, {}) + + settings = each.value + +} + +data "kustomization_overlay" "placeholderjob" { + for_each = local.remote.agent_pools[var.agent_pools.lz_key] + + resources = [ + "yamls/placeholderjob.yaml", + ] + + namespace = var.agent_pools.namespace + + patches { + patch = <<-EOF + - op: replace + path: /metadata/name + value: "placeholder-job-${replace(each.key,"_","-")}" + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/spec/containers/0/env/0/value + value: "https://dev.azure.com/afopssre" + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/metadata/labels/aadpodidbinding + value: ${each.value.name} + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/spec/containers/0/env/2/value + value: ${each.value.name} + EOF + target = { + kind = "Job" + } + } + + patches { + patch = <<-EOF + - op: replace + path: /spec/template/spec/containers/0/image + value: "${var.agent_pools.image}" + EOF + target = { + kind = "Job" + } + } +} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf new file mode 100644 index 000000000..985bd96f5 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf @@ -0,0 +1,16 @@ +resource "kustomization_resource" "p0" { + for_each = var.settings.ids_prio[0] + manifest = var.settings.manifests[each.value] +} + +resource "kustomization_resource" "p1" { + depends_on = [kustomization_resource.p0] + for_each = var.settings.ids_prio[1] + manifest = var.settings.manifests[each.value] +} + +resource "kustomization_resource" "p2" { + depends_on = [kustomization_resource.p1] + for_each = var.settings.ids_prio[2] + manifest = var.settings.manifests[each.value] +} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf new file mode 100644 index 000000000..e65c6fa22 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + kustomization = { + source = "kbst/kustomization" + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf new file mode 100644 index 000000000..f5c321890 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf @@ -0,0 +1,2 @@ +variable "settings" { +} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/locals.remote_tfstates.tf b/caf_solution/add-ons/aks_azure_devops_agents/locals.remote_tfstates.tf new file mode 100644 index 000000000..46b41e90a --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/locals.remote_tfstates.tf @@ -0,0 +1,63 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name + container_name = try(each.value.container, local.landingzone[try(each.value.level, "current")].container_name) + resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id + key = each.value.tfstate + } +} + +locals { + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + + remote = { + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + global_settings = local.global_settings + diagnostics = local.diagnostics + + + aks_clusters = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) + } + keyvaults = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) + } + azure_devops = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.azure_devops, {})) + } + agent_pools = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs[var.agent_pools.key], {})) + } + } + +} +output "keyvaults" { + value = local.remote.keyvaults +} +output "azure_devops" { + value = local.remote.azure_devops +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/main.tf b/caf_solution/add-ons/aks_azure_devops_agents/main.tf new file mode 100644 index 000000000..6b7201fb4 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/main.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.55.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0.2" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.1.2" + } + kustomization = { + source = "kbst/kustomization" + version = "~> 0.5.0" + } + } + required_version = ">= 0.13" +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/output.tf b/caf_solution/add-ons/aks_azure_devops_agents/output.tf new file mode 100644 index 000000000..e69de29bb diff --git a/caf_solution/add-ons/aks_azure_devops_agents/providers.tf b/caf_solution/add-ons/aks_azure_devops_agents/providers.tf new file mode 100644 index 000000000..e8b9dcfc5 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/providers.tf @@ -0,0 +1,54 @@ +provider "azurerm" { + features { + } +} + +provider "kubernetes" { + host = try(local.k8sconfigs[var.aks_cluster_key].host, null) + username = try(local.k8sconfigs[var.aks_cluster_key].username, null) + password = try(local.k8sconfigs[var.aks_cluster_key].password, null) + client_certificate = try(local.k8sconfigs[var.aks_cluster_key].client_certificate, null) + client_key = try(local.k8sconfigs[var.aks_cluster_key].client_key, null) + cluster_ca_certificate = try(local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate, null) +} + +provider "helm" { + kubernetes { + host = local.k8sconfigs[var.aks_cluster_key].host + username = local.k8sconfigs[var.aks_cluster_key].username + password = local.k8sconfigs[var.aks_cluster_key].password + client_certificate = local.k8sconfigs[var.aks_cluster_key].client_certificate + client_key = local.k8sconfigs[var.aks_cluster_key].client_key + cluster_ca_certificate = local.k8sconfigs[var.aks_cluster_key].cluster_ca_certificate + } +} + +provider "kustomization" { + kubeconfig_raw = local.k8sconfigs[var.aks_cluster_key].kube_admin_config_raw +} + +locals { + aks_clusters = { + for key, value in var.aks_clusters : key => + local.remote.aks_clusters[value.lz_key][value.key] + } + k8sconfigs = { + for key, value in var.aks_clusters : key => { + kube_admin_config_raw = data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config_raw + host = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.host : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.host + username = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.username : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.username + password = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.password : data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.password + client_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_certificate) + client_key = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.client_key) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.client_key) + cluster_ca_certificate = local.remote.aks_clusters[value.lz_key][value.key].enable_rbac ? base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_admin_config.0.cluster_ca_certificate) : base64decode(data.azurerm_kubernetes_cluster.kubeconfig[key].kube_config.0.cluster_ca_certificate) + } + } +} + +# Get kubeconfig from AKS clusters +data "azurerm_kubernetes_cluster" "kubeconfig" { + for_each = var.aks_clusters + + name = local.remote.aks_clusters[each.value.lz_key][each.value.key].cluster_name + resource_group_name = local.remote.aks_clusters[each.value.lz_key][each.value.key].resource_group_name +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/variables.tf b/caf_solution/add-ons/aks_azure_devops_agents/variables.tf new file mode 100644 index 000000000..2789ae8f4 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/variables.tf @@ -0,0 +1,52 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is populated by the rover. subscription id hosting the remote tfstates" +} +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "global_settings" { + default = {} +} + + +variable "landingzone" {} +variable "rover_version" { + default = null +} +variable "tags" { + default = {} +} +variable "namespaces" { + default = {} +} + +variable "helm_charts" { + default = {} +} +variable "aks_clusters" {} + +variable "aks_cluster_key" { +} + +variable "kustomization_overlays" { + default = {} +} + +variable "kustomization_builds" { + default = {} +} + +variable "agent_pools" { + +} + +variable "keyvault" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/akvs-secret.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/akvs-secret.yaml new file mode 100644 index 000000000..83c9cefa5 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/akvs-secret.yaml @@ -0,0 +1,14 @@ +apiVersion: spv.no/v2beta1 +kind: AzureKeyVaultSecret +metadata: + name: pat-secret-sync +spec: + vault: + name: afops-kv-afopssre-djo # name of key vault + object: + name: azdo-pat-admin # name of the akv object + type: secret # akv object type + output: + secret: + name: pat-secret-sync # kubernetes secret name + dataKey: personalAccessToken # key to store object value in kubernetes secret \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/azdopat-secret.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/azdopat-secret.yaml new file mode 100644 index 000000000..c3bd660ef --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/azdopat-secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: azdopat-secret +data: + personalAccessToken: ${pat} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderagent.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderagent.yaml new file mode 100644 index 000000000..b9c5d8ae4 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderagent.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rover-deployment + labels: + aadpodidbinding: podmi-caf-rover-platform-level0 + app: rover-agent +spec: + replicas: 1 + selector: + matchLabels: + app: rover-agent + template: + metadata: + labels: + aadpodidbinding: podmi-caf-rover-platform-level0 + app: rover-agent + spec: + containers: + - name: rover-agent + image: aztfmod/rover-agent:0.15.1-2105.041640-preview-azdo + env: + - name: VSTS_AGENT_INPUT_URL + value: "https://dev.azure.com/afopssre" + - name: VSTS_AGENT_INPUT_TOKEN + valueFrom: + secretKeyRef: + name: azdopat-secret + key: personalAccessToken + - name: VSTS_AGENT_INPUT_POOL + value: "aks-agents" + - name: VSTS_AGENT_INPUT_AUTH + value: "pat" + - name: VSTS_AGENT_INPUT_RUN_ONCE + value: "--once" + volumeMounts: + - name: secrets-store01-inline + mountPath: "/mnt/secrets-store" + readOnly: true + lifecycle: + preStop: + exec: + # SIGTERM triggers a quick exit; gracefully terminate instead + command: ["/home/vscode/agent/config.sh","remove","--unattended"] + volumes: + - name: secrets-store01-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "azure-sync" + + # terminationGracePeriodSeconds: 60 \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderjob.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderjob.yaml new file mode 100644 index 000000000..2c9df8b05 --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/placeholderjob.yaml @@ -0,0 +1,38 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: placeholder-agent + labels: + app: placeholder-agent +spec: + activeDeadlineSeconds: 180 + template: + metadata: + labels: + aadpodidbinding: ${podmi} + app: placeholder-agent + spec: + containers: + - name: azdevops-agent-job + image: ${image} + imagePullPolicy: Always + env: + - name: VSTS_AGENT_INPUT_URL + value: ${VSTS_AGENT_INPUT_URL} + - name: VSTS_AGENT_INPUT_TOKEN + valueFrom: + secretKeyRef: + name: pat-secret-sync + key: personalAccessToken + - name: VSTS_AGENT_INPUT_POOL + value: ${VSTS_AGENT_INPUT_POOL} + - name: VSTS_AGENT_INPUT_AUTH + value: "pat" + - name: VSTS_AGENT_INPUT_RUN_ARGS + value: "--once" + # lifecycle: + # preStop: + # exec: + # # SIGTERM triggers a quick exit; gracefully terminate instead + # command: ["/home/vscode/agent/config.sh","remove","--unattended"] + restartPolicy: Never \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/roverjob.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/roverjob.yaml new file mode 100644 index 000000000..83de01aef --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/roverjob.yaml @@ -0,0 +1,48 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledJob +metadata: + name: ${azdevops-scaledjob} +spec: + jobTargetRef: + ttlSecondsAfterFinished: 100 + template: + metadata: + labels: + aadpodidbinding: podmi-caf-rover-platform-level0 + app: rover-agent + spec: + containers: + - name: azdevops-agent-job + image: ${image} + imagePullPolicy: Always + env: + - name: VSTS_AGENT_INPUT_URL + value: ${VSTS_AGENT_INPUT_URL} + - name: VSTS_AGENT_INPUT_TOKEN + valueFrom: + secretKeyRef: + name: pat-secret-sync + key: personalAccessToken + - name: VSTS_AGENT_INPUT_POOL + value: ${VSTS_AGENT_INPUT_POOL} + - name: VSTS_AGENT_INPUT_AUTH + value: "pat" + - name: VSTS_AGENT_INPUT_RUN_ARGS + value: "--once" + # lifecycle: + # preStop: + # exec: + # # SIGTERM triggers a quick exit; gracefully terminate instead + # command: ["/home/vscode/agent/config.sh","remove","--unattended"] + pollingInterval: 30 + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 + maxReplicaCount: 10 + scalingStrategy: + strategy: "default" + triggers: + - type: azure-pipelines + metadata: + poolID: ${poolID} + organizationURLFromEnv: "VSTS_AGENT_INPUT_URL" + personalAccessTokenFromEnv: "VSTS_AGENT_INPUT_TOKEN" diff --git a/caf_solution/add-ons/aks_azure_devops_agents/yamls/secretstorecsi.yaml b/caf_solution/add-ons/aks_azure_devops_agents/yamls/secretstorecsi.yaml new file mode 100644 index 000000000..cb084bc0a --- /dev/null +++ b/caf_solution/add-ons/aks_azure_devops_agents/yamls/secretstorecsi.yaml @@ -0,0 +1,22 @@ +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: azdo-pat-sync +spec: + provider: azure + secretObjects: # secretObjects defines the desired state of synced K8s secret objects + - secretName: azdo-pat-secret + type: Opaque + data: + - objectName: azdo-pat-secret + key: personalAccessToken + parameters: + usePodIdentity: "true" + keyvaultName: afops-kv-afopssre-djo + objects: | + array: + - | + objectName: azdo-pat-admin + objectAlias: azdo-pat-secret + objectType: secret + tenantId: gitopsprod.onmicrosoft.com \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/output.tf b/caf_solution/add-ons/azure_devops_v1/output.tf index 91eeb4691..51c08bdf0 100644 --- a/caf_solution/add-ons/azure_devops_v1/output.tf +++ b/caf_solution/add-ons/azure_devops_v1/output.tf @@ -5,4 +5,8 @@ # } # ) # sensitive = true -# } \ No newline at end of file +# } + +output "azure_devops" { + value = var.azure_devops +} \ No newline at end of file From fb3e3ef154957df904d6499626b5a2e5f5944f1c Mon Sep 17 00:00:00 2001 From: lolorol Date: Thu, 20 May 2021 16:00:11 +0000 Subject: [PATCH 11/65] Add support for mtms bootstrap scenario --- caf_solution/locals.remote_tfstates.tf | 51 +++++++++++++++++--------- caf_solution/main.tf | 2 +- caf_solution/variables.tf | 31 ++++++++++++---- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/caf_solution/locals.remote_tfstates.tf b/caf_solution/locals.remote_tfstates.tf index ca11e499e..c66dd03a2 100644 --- a/caf_solution/locals.remote_tfstates.tf +++ b/caf_solution/locals.remote_tfstates.tf @@ -17,22 +17,37 @@ data "terraform_remote_state" "remote" { for_each = try(var.landingzone.tfstates, {}) backend = var.landingzone.backend_type - config = { - storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name - container_name = try(each.value.workspace, local.landingzone[try(each.value.level, "current")].container_name) - resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name - subscription_id = var.tfstate_subscription_id - key = each.value.tfstate - } + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] } locals { - landingzone_tag = { - "landingzone" = var.landingzone.key + + remote_state = { + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = try(value.workspace, local.landingzone[try(value.level, "current")].container_name) + key = value.tfstate + resource_group_name = try(value.resource_group_name, local.landingzone[try(value.level, "current")].resource_group_name) + storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) + subscription_id = try(value.subscription_id, var.tfstate_subscription_id) + tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + } + } + } - tags = merge(try(local.global_settings.tags, {}), local.landingzone_tag, { "level" = var.landingzone.level }, try({ "environment" = local.global_settings.environment }, {}), { "rover_version" = var.rover_version }, var.tags) - global_settings = merge(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings, var.global_settings) + global_settings_key = try(var.landingzone.global_settings_key, keys(var.landingzone.tfstates)[0]) + + tags = merge( + # tomap(try(local.global_settings.tags, {})), + # tomap({"landingzone" = var.landingzone.key}), + # tomap({ "level" = var.landingzone.level }), + # tomap(try({ "environment" = local.global_settings.environment }, {})), + # tomap({ "rover_version" = var.rover_version }), + var.tags + ) + + global_settings = merge(data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].global_settings, var.global_settings) diagnostics = { # Get the diagnostics settings of services to create @@ -41,25 +56,25 @@ locals { diagnostic_storage_accounts = var.diagnostic_storage_accounts # Combine the diagnostics definitions - diagnostics_definition = merge(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.diagnostics_definition, var.diagnostics_definition) + diagnostics_definition = merge(data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].diagnostics.diagnostics_definition, var.diagnostics_definition) diagnostics_destinations = { event_hub_namespaces = merge( - try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.diagnostics_destinations.event_hub_namespaces, {}), + try(data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].diagnostics.diagnostics_destinations.event_hub_namespaces, {}), try(var.diagnostics_destinations.event_hub_namespaces, {}) ) log_analytics = merge( - try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.diagnostics_destinations.log_analytics, {}), + try(data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].diagnostics.diagnostics_destinations.log_analytics, {}), try(var.diagnostics_destinations.log_analytics, {}) ) storage = merge( - try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.diagnostics_destinations.storage, {}), + try(data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].diagnostics.diagnostics_destinations.storage, {}), try(var.diagnostics_destinations.storage, {}) ) } # Get the remote existing diagnostics objects - storage_accounts = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.storage_accounts - log_analytics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.log_analytics - event_hub_namespaces = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.event_hub_namespaces + storage_accounts = data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].diagnostics.storage_accounts + log_analytics = data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].diagnostics.log_analytics + event_hub_namespaces = data.terraform_remote_state.remote[local.global_settings_key].outputs.objects[local.global_settings_key].diagnostics.event_hub_namespaces } } diff --git a/caf_solution/main.tf b/caf_solution/main.tf index 216cc29da..628657d92 100644 --- a/caf_solution/main.tf +++ b/caf_solution/main.tf @@ -51,7 +51,7 @@ locals { } ) , - data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates + data.terraform_remote_state.remote[local.global_settings_key].outputs.tfstates ) diff --git a/caf_solution/variables.tf b/caf_solution/variables.tf index 58b0dfd36..bc129f343 100644 --- a/caf_solution/variables.tf +++ b/caf_solution/variables.tf @@ -1,15 +1,30 @@ # Map of the remote data state for lower level -variable "lower_storage_account_name" {} -variable "lower_container_name" {} -variable "lower_resource_group_name" {} +variable "lower_storage_account_name" { + default = null +} +variable "lower_container_name" { + default = null +} +variable "lower_resource_group_name" { + default = null +} variable "tfstate_subscription_id" { description = "This value is propulated by the rover. subscription id hosting the remote tfstates" + default = null +} +variable "tfstate_storage_account_name" { + default = null +} +variable "tfstate_container_name" { + default = null +} +variable "tfstate_key" { + default = null +} +variable "tfstate_resource_group_name" { + default = null } -variable "tfstate_storage_account_name" {} -variable "tfstate_container_name" {} -variable "tfstate_key" {} -variable "tfstate_resource_group_name" {} variable "landingzone" { default = { @@ -93,7 +108,7 @@ variable "use_msi" { variable "tags" { description = "Tags to be used for this resource deployment." type = map(any) - default = null + default = {} } variable "resource_groups" { From 285ab82e9ddc9db839c1b3626aa09ca55c4dfa61 Mon Sep 17 00:00:00 2001 From: lolorol Date: Fri, 21 May 2021 06:29:31 +0000 Subject: [PATCH 12/65] Update for mtms scenarios --- caf_solution/local.remote.tf | 3 +++ caf_solution/local.remote_objects.tf | 36 -------------------------- caf_solution/locals.remote_tfstates.tf | 10 +++---- caf_solution/variables.tf | 2 +- 4 files changed, 9 insertions(+), 42 deletions(-) delete mode 100644 caf_solution/local.remote_objects.tf diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index e977aa844..a6db31099 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -3,6 +3,9 @@ locals { azuread_apps = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_apps, {})) } + azuread_applications = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) + } azuread_groups = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) } diff --git a/caf_solution/local.remote_objects.tf b/caf_solution/local.remote_objects.tf deleted file mode 100644 index 602dc1229..000000000 --- a/caf_solution/local.remote_objects.tf +++ /dev/null @@ -1,36 +0,0 @@ -# locals { -# remote_objects = { -# aks_clusters = local.remote.aks_clusters -# app_service_environments = local.remote.app_service_environments -# app_service_plans = local.remote.app_service_plans -# app_services = local.remote.app_services -# application_gateway_applications = local.remote.application_gateway_applications -# application_gateways = local.remote.application_gateways -# availability_sets = local.remote.availability_sets -# azuread_applications = local.remote.azuread_applications -# azuread_groups = local.remote.azuread_groups -# azuread_users = local.remote.azuread_users -# azurerm_firewalls = local.remote.azurerm_firewalls -# container_registry = local.remote.container_registry -# event_hub_namespaces = local.remote.event_hub_namespaces -# front_door_waf_policies = local.remote.front_door_waf_policies -# keyvaults = local.remote.keyvaults -# managed_identities = local.remote.managed_identities -# mssql_databases = local.remote.mssql_databases -# mssql_elastic_pools = local.remote.mssql_elastic_pools -# mssql_managed_databases = local.remote.mssql_managed_databases -# mssql_managed_instances = local.remote.mssql_managed_instances -# mssql_servers = local.remote.mssql_servers -# mysql_servers = local.remote.mysql_servers -# network_watchers = local.remote.network_watchers -# postgresql_servers = local.remote.postgresql_servers -# private_dns = local.remote.private_dns -# proximity_placement_groups = local.remote.proximity_placement_groups -# public_ip_addresses = local.remote.public_ip_addresses -# recovery_vaults = local.remote.recovery_vaults -# resource_groups = local.remote.resource_groups -# storage_accounts = local.remote.storage_accounts -# synapse_workspaces = local.remote.synapse_workspaces -# vnets = local.remote.vnets -# } -# } \ No newline at end of file diff --git a/caf_solution/locals.remote_tfstates.tf b/caf_solution/locals.remote_tfstates.tf index c66dd03a2..efe90da88 100644 --- a/caf_solution/locals.remote_tfstates.tf +++ b/caf_solution/locals.remote_tfstates.tf @@ -39,11 +39,11 @@ locals { global_settings_key = try(var.landingzone.global_settings_key, keys(var.landingzone.tfstates)[0]) tags = merge( - # tomap(try(local.global_settings.tags, {})), - # tomap({"landingzone" = var.landingzone.key}), - # tomap({ "level" = var.landingzone.level }), - # tomap(try({ "environment" = local.global_settings.environment }, {})), - # tomap({ "rover_version" = var.rover_version }), + try(local.global_settings.tags, {}), + {"landingzone" = var.landingzone.key}, + { "level" = var.landingzone.level }, + try({ "environment" = local.global_settings.environment }, {}), + { "rover_version" = var.rover_version }, var.tags ) diff --git a/caf_solution/variables.tf b/caf_solution/variables.tf index bc129f343..e0b5d9cc9 100644 --- a/caf_solution/variables.tf +++ b/caf_solution/variables.tf @@ -53,7 +53,7 @@ variable "provider_azurerm_features_keyvault" { variable "rover_version" { - default = {} + default = "caf_standalone" } variable "client_config" { From d6bf9924b4581ed58d0988b74aac9a8f8fa3a222 Mon Sep 17 00:00:00 2001 From: Wade Francis Date: Mon, 24 May 2021 08:56:07 +0000 Subject: [PATCH 13/65] Removed duplicate azuread_applications stanza --- caf_solution/local.remote.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index 6704380f8..3c6146228 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -42,9 +42,6 @@ locals { availability_sets = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].availability_sets, {})) } - azuread_applications = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aad_apps, {})) - } azuread_groups = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) } From 700b4a816e6e67154dd88de088bedbd68557a336 Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Fri, 28 May 2021 10:55:42 +0000 Subject: [PATCH 14/65] update devops and azuread_app --- .../add-ons/azure_devops_v1/azdo_service_endpoint.tf | 8 ++++---- .../add-ons/azure_devops_v1/local.azuread.tf | 10 ++++++++++ .../azure_devops_v1/locals.remote_tfstates.tf | 9 +++------ .../add-ons/azure_devops_v1/variables.azuread.tf | 12 ++++++++++++ caf_solution/landingzone.tf | 3 ++- 5 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 caf_solution/add-ons/azure_devops_v1/local.azuread.tf create mode 100644 caf_solution/add-ons/azure_devops_v1/variables.azuread.tf diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf index e5fca29ff..ecca19f33 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf @@ -2,8 +2,8 @@ data "azurerm_key_vault_secret" "client_secret" { for_each = var.service_endpoints - name = format("%s-client-secret", local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].keyvaults[each.value.keyvault_key].secret_name_client_secret) - key_vault_id = local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].keyvaults[each.value.keyvault_key].id + name = each.value.keyvault.secret_name + key_vault_id = local.remote.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.key].id } resource "azuredevops_serviceendpoint_azurerm" "azure" { @@ -12,10 +12,10 @@ resource "azuredevops_serviceendpoint_azurerm" "azure" { project_id = data.azuredevops_project.project[each.value.project_key].id service_endpoint_name = each.value.endpoint_name credentials { - serviceprincipalid = local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].azuread_application.application_id + serviceprincipalid = local.remote.azuread_applications[each.value.azuread_application.lz_key][each.value.azuread_application.key].application_id serviceprincipalkey = data.azurerm_key_vault_secret.client_secret[each.key].value } - azurerm_spn_tenantid = local.remote.aad_apps[each.value.lz_key][each.value.aad_app_key].tenant_id + azurerm_spn_tenantid = local.remote.azuread_applications[each.value.azuread_application.lz_key][each.value.azuread_application.key].tenant_id azurerm_subscription_id = each.value.subscription.id azurerm_subscription_name = each.value.subscription.name } diff --git a/caf_solution/add-ons/azure_devops_v1/local.azuread.tf b/caf_solution/add-ons/azure_devops_v1/local.azuread.tf new file mode 100644 index 000000000..dca335922 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/local.azuread.tf @@ -0,0 +1,10 @@ +locals { + azuread = merge( + var.azuread, + { + azuread_applications = var.azuread_applications + azuread_service_principals = var.azuread_service_principals + azuread_service_principal_passwords = var.azuread_service_principal_passwords + } + ) +} diff --git a/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf index 8b878f87f..c1b2b0d79 100644 --- a/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf +++ b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf @@ -40,23 +40,20 @@ locals { aad_apps = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aad_apps, {})) } - + azuread_applications = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) + } azuread_groups = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) } - keyvaults = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) } - managed_identities = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) } - vnets = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) } } - - } diff --git a/caf_solution/add-ons/azure_devops_v1/variables.azuread.tf b/caf_solution/add-ons/azure_devops_v1/variables.azuread.tf new file mode 100644 index 000000000..3ddca35c4 --- /dev/null +++ b/caf_solution/add-ons/azure_devops_v1/variables.azuread.tf @@ -0,0 +1,12 @@ +variable "azuread" { + default = {} +} +variable "azuread_applications" { + default = {} +} +variable "azuread_service_principals" { + default = {} +} +variable "azuread_service_principal_passwords" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 9cfc3326c..21383d57b 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,7 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" + # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" + source = "../../aztfmod" azuread = local.azuread From 776f2a26002ae6268706fc37d8cbb41c73c2db28 Mon Sep 17 00:00:00 2001 From: Hieu Nguyen Nhu <5441003+hieumoscow@users.noreply.github.com> Date: Sun, 30 May 2021 21:58:47 +0800 Subject: [PATCH 15/65] Added aks_secure_baseline_v2 with Flux provider and refactor aks addons --- .gitignore | 3 +- .../add-ons/aks_applications/app/variables.tf | 2 - .../add-ons/aks_applications/applications.tf | 2 - .../aks_azure_devops_agents/app/module.tf | 47 ------------ .../aks_azure_devops_agents/app/output.tf | 0 .../aks_azure_devops_agents/app/variables.tf | 13 ---- .../aks_azure_devops_agents/applications.tf | 4 +- .../aks_azure_devops_agents/kustomization.tf | 2 +- .../kustomize/kustomization_build.tf | 16 ----- .../aks_azure_devops_agents/kustomize/main.tf | 7 -- .../kustomize/variables.tf | 2 - .../aks-pod-identity-assignment.tf | 67 +++++++++++++++++ .../aks_secure_baseline_v2/backend.azurerm | 4 ++ .../aks_secure_baseline_v2/flux/flux.tf | 71 +++++++++++++++++++ .../flux}/main.tf | 8 +-- .../aks_secure_baseline_v2/flux/variables.tf | 3 + .../locals.remote_tfstates.tf | 53 ++++++++++++++ .../add-ons/aks_secure_baseline_v2/main.tf | 20 ++++++ .../add-ons/aks_secure_baseline_v2/module.tf | 5 ++ .../aks_secure_baseline_v2/providers.tf | 29 ++++++++ .../aks_secure_baseline_v2/variables.tf | 44 ++++++++++++ 21 files changed, 304 insertions(+), 98 deletions(-) delete mode 100644 caf_solution/add-ons/aks_azure_devops_agents/app/module.tf delete mode 100644 caf_solution/add-ons/aks_azure_devops_agents/app/output.tf delete mode 100644 caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf delete mode 100644 caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf delete mode 100644 caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf delete mode 100644 caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/backend.azurerm create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf rename caf_solution/add-ons/{aks_azure_devops_agents/app => aks_secure_baseline_v2/flux}/main.tf (52%) create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/flux/variables.tf create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/locals.remote_tfstates.tf create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/main.tf create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/module.tf create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/providers.tf create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/variables.tf diff --git a/.gitignore b/.gitignore index a043b16a3..9496a3682 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ **/*.log **/backend.azurerm.tf public -aztfmod \ No newline at end of file +aztfmod +*output.json \ No newline at end of file diff --git a/caf_solution/add-ons/aks_applications/app/variables.tf b/caf_solution/add-ons/aks_applications/app/variables.tf index 6c2baef70..cb3a56bfc 100644 --- a/caf_solution/add-ons/aks_applications/app/variables.tf +++ b/caf_solution/add-ons/aks_applications/app/variables.tf @@ -1,5 +1,3 @@ -variable "cluster" {} - variable "namespaces" { default = {} } diff --git a/caf_solution/add-ons/aks_applications/applications.tf b/caf_solution/add-ons/aks_applications/applications.tf index d36d39b0d..d72da73f4 100644 --- a/caf_solution/add-ons/aks_applications/applications.tf +++ b/caf_solution/add-ons/aks_applications/applications.tf @@ -1,7 +1,5 @@ module "app" { source = "./app" - - cluster = local.aks_clusters[var.aks_cluster_key] namespaces = var.namespaces helm_charts = var.helm_charts } diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/module.tf b/caf_solution/add-ons/aks_azure_devops_agents/app/module.tf deleted file mode 100644 index 138110a32..000000000 --- a/caf_solution/add-ons/aks_azure_devops_agents/app/module.tf +++ /dev/null @@ -1,47 +0,0 @@ -resource "kubernetes_namespace" "namespaces" { - for_each = var.namespaces - metadata { - annotations = try(each.value.annotations, null) - labels = try(each.value.labels, null) - name = each.value.name - } - -} - -# https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release -resource "helm_release" "charts" { - for_each = var.helm_charts - - name = each.value.name - repository = each.value.repository - chart = each.value.chart - - namespace = each.value.namespace - wait = try(each.value.wait, true) - timeout = try(each.value.timeout, 900) - skip_crds = try(each.value.skip_crds, false) - create_namespace = try(each.value.create_namespace, false) - values = try(each.value.values, null) - - dynamic "set" { - for_each = try(each.value.sets, {}) - content { - name = set.key - value = set.value - } - } - - dynamic "set_sensitive" { - for_each = try(each.value.sets_sensitive, {}) - content { - name = set_sensitive.key - value = set_sensitive.value - } - } - - - # depends_on = [kubernetes_namespace.namespaces] - # values = [ - # "${file("values.yaml")}" - # ] -} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/output.tf b/caf_solution/add-ons/aks_azure_devops_agents/app/output.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf b/caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf deleted file mode 100644 index 6c2baef70..000000000 --- a/caf_solution/add-ons/aks_azure_devops_agents/app/variables.tf +++ /dev/null @@ -1,13 +0,0 @@ -variable "cluster" {} - -variable "namespaces" { - default = {} -} - -variable "helm_charts" { - default = {} -} - -variable "kuztomization_settings" { - default = {} -} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/applications.tf b/caf_solution/add-ons/aks_azure_devops_agents/applications.tf index d36d39b0d..49c6951c1 100644 --- a/caf_solution/add-ons/aks_azure_devops_agents/applications.tf +++ b/caf_solution/add-ons/aks_azure_devops_agents/applications.tf @@ -1,7 +1,5 @@ module "app" { - source = "./app" - - cluster = local.aks_clusters[var.aks_cluster_key] + source = "../aks_applications/app" namespaces = var.namespaces helm_charts = var.helm_charts } diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf index 5a6a941af..969d1cf82 100644 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf @@ -50,7 +50,7 @@ output "manifests" { # } module "kustomization_azdopat-secret" { - source = "./kustomize" + source = "../aks_applications/kustomize" settings = data.kustomization_overlay.azdopat-secret diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf deleted file mode 100644 index 985bd96f5..000000000 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/kustomization_build.tf +++ /dev/null @@ -1,16 +0,0 @@ -resource "kustomization_resource" "p0" { - for_each = var.settings.ids_prio[0] - manifest = var.settings.manifests[each.value] -} - -resource "kustomization_resource" "p1" { - depends_on = [kustomization_resource.p0] - for_each = var.settings.ids_prio[1] - manifest = var.settings.manifests[each.value] -} - -resource "kustomization_resource" "p2" { - depends_on = [kustomization_resource.p1] - for_each = var.settings.ids_prio[2] - manifest = var.settings.manifests[each.value] -} diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf deleted file mode 100644 index e65c6fa22..000000000 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/main.tf +++ /dev/null @@ -1,7 +0,0 @@ -terraform { - required_providers { - kustomization = { - source = "kbst/kustomization" - } - } -} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf deleted file mode 100644 index f5c321890..000000000 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomize/variables.tf +++ /dev/null @@ -1,2 +0,0 @@ -variable "settings" { -} diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf new file mode 100644 index 000000000..eab82971f --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf @@ -0,0 +1,67 @@ + +# Get the details of the node pool's resource group created by AKS +data "azurerm_resource_group" "noderg" { + for_each = var.aks_clusters + name = local.remote.aks_clusters[each.value.lz_key][each.value.key].node_resource_group +} + +# +# Set permissions to the kubelet and cluster identity +# +resource "azurerm_role_assignment" "kubelet_noderg_miop" { + for_each = var.aks_clusters + + scope = data.azurerm_resource_group.noderg[each.key].id + role_definition_name = "Managed Identity Operator" + principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +} + +resource "azurerm_role_assignment" "kubelet_noderg_vmcontrib" { + for_each = var.aks_clusters + + scope = data.azurerm_resource_group.noderg[each.key].id + role_definition_name = "Virtual Machine Contributor" + principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +} + +# Separate subnet +resource "azurerm_role_assignment" "kubelet_subnets_networkcontrib" { + for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",{vnet=true}) + + scope = try(each.value==true, false) ? local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id : local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].subnets[each.value].id + role_definition_name = "Network Contributor" + principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id +} + +# # Whole vnet +# resource "azurerm_role_assignment" "kubelet_vnet_networkcontrib" { +# for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",null) == null ? var.vnets : {} + +# scope = local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id +# role_definition_name = "Network Contributor" +# principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id +# } + +resource "azurerm_role_assignment" "kubelet_user_msi" { + for_each = local.msi_to_grant_permissions + + scope = each.value.id + role_definition_name = "Managed Identity Operator" + principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].kubelet_identity[0].object_id +} + +locals { + msi_to_grant_permissions = { + for msi in flatten( + [ + for key, value in var.managed_identities : [ + for msi_key in value.msi_keys : { + key = key + msi_key = msi_key + id = local.remote.managed_identities[value.lz_key][msi_key].id + } + ] + ] + ) : format("%s-%s", msi.key, msi.msi_key) => msi + } +} diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/backend.azurerm b/caf_solution/add-ons/aks_secure_baseline_v2/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf new file mode 100644 index 000000000..44da6c6d8 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf @@ -0,0 +1,71 @@ +data "flux_install" "main" { + target_path = var.setting.target_install_path + namespace = var.setting.namespace +} + +data "flux_sync" "main" { + target_path = var.setting.target_sync_path + url = var.setting.url + branch = var.setting.branch + secret = try(var.setting.flux_auth_secret, null) + namespace = var.setting.namespace +} + +data "kubectl_file_documents" "install" { + content = data.flux_install.main.content +} + +data "kubectl_file_documents" "sync" { + content = data.flux_sync.main.content +} + +resource "kubernetes_namespace" "flux_system" { + metadata { + name = var.setting.namespace + } + + lifecycle { + ignore_changes = [ + metadata[0].labels, + ] + } +} + +locals { + install = [for v in data.kubectl_file_documents.install.documents : { + data : yamldecode(v) + content : v + } + ] + sync = [for v in data.kubectl_file_documents.sync.documents : { + data : yamldecode(v) + content : v + } + ] +} + +resource "kubectl_manifest" "install" { + for_each = { for v in local.install : lower(join("/", compact([v.data.apiVersion, v.data.kind, lookup(v.data.metadata, "namespace", ""), v.data.metadata.name]))) => v.content } + depends_on = [kubernetes_namespace.flux_system] + yaml_body = each.value +} + +resource "kubectl_manifest" "sync" { + for_each = { for v in local.sync : lower(join("/", compact([v.data.apiVersion, v.data.kind, lookup(v.data.metadata, "namespace", ""), v.data.metadata.name]))) => v.content } + depends_on = [kubernetes_namespace.flux_system] + yaml_body = each.value +} + + + +resource "kubernetes_secret" "fluxauth" { + depends_on = [kubectl_manifest.install] + + metadata { + name = try(var.setting.flux_auth_secret, "flux-system") + namespace = var.setting.namespace + } + data = try(var.setting.secret.data, null) + type = try(var.setting.secret.type, null) +} + diff --git a/caf_solution/add-ons/aks_azure_devops_agents/app/main.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/main.tf similarity index 52% rename from caf_solution/add-ons/aks_azure_devops_agents/app/main.tf rename to caf_solution/add-ons/aks_secure_baseline_v2/flux/main.tf index c45957ba5..b19246abc 100644 --- a/caf_solution/add-ons/aks_azure_devops_agents/app/main.tf +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/main.tf @@ -3,11 +3,11 @@ terraform { kubernetes = { source = "hashicorp/kubernetes" } - helm = { - source = "hashicorp/helm" + kubectl = { + source = "gavinbunney/kubectl" } - kustomization = { - source = "kbst/kustomization" + flux = { + source = "fluxcd/flux" } } } \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/variables.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/variables.tf new file mode 100644 index 000000000..53a87763f --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/variables.tf @@ -0,0 +1,3 @@ +variable "setting" { + description = "Flux settings" +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/locals.remote_tfstates.tf b/caf_solution/add-ons/aks_secure_baseline_v2/locals.remote_tfstates.tf new file mode 100644 index 000000000..f8312ab8c --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/locals.remote_tfstates.tf @@ -0,0 +1,53 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name + container_name = try(each.value.container, local.landingzone[try(each.value.level, "current")].container_name) + resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id + key = each.value.tfstate + } +} + +locals { + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + + remote = { + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + global_settings = local.global_settings + diagnostics = local.diagnostics + + + aks_clusters = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aks_clusters, {})) + } + managed_identities = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) + } + vnets = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) + } + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/main.tf b/caf_solution/add-ons/aks_secure_baseline_v2/main.tf new file mode 100644 index 000000000..a409122ce --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/main.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.0.2" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.11.1" + } + flux = { + source = "fluxcd/flux" + version = ">= 0.0.13" + } + } + required_version = ">= 0.13" +} + + + diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/module.tf b/caf_solution/add-ons/aks_secure_baseline_v2/module.tf new file mode 100644 index 000000000..2f77e8315 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/module.tf @@ -0,0 +1,5 @@ +module "flux" { + source = "./flux" + for_each = var.flux_settings + setting = each.value +} diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/providers.tf b/caf_solution/add-ons/aks_secure_baseline_v2/providers.tf new file mode 100644 index 000000000..aafa4dd61 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/providers.tf @@ -0,0 +1,29 @@ +provider "azurerm" { + features { + } +} + +provider "kubectl" { + host = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.host, null) + username = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.username, null) + password = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.password, null) + client_key = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_key), null) + client_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_certificate), null) + cluster_ca_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.cluster_ca_certificate), null) + load_config_file = false +} + +provider "kubernetes" { + host = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.host, null) + username = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.username, null) + password = try(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.password, null) + client_key = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_key), null) + client_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.client_certificate), null) + cluster_ca_certificate = try(base64decode(data.azurerm_kubernetes_cluster.kubeconfig.kube_admin_config.0.cluster_ca_certificate), null) +} + +# Get kubeconfig from AKS clusters +data "azurerm_kubernetes_cluster" "kubeconfig" { + name = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_clusters[var.aks_cluster_key].key].cluster_name + resource_group_name = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_clusters[var.aks_cluster_key].key].resource_group_name +} \ No newline at end of file diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/variables.tf b/caf_solution/add-ons/aks_secure_baseline_v2/variables.tf new file mode 100644 index 000000000..0b195b73b --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/variables.tf @@ -0,0 +1,44 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is populated by the rover. subscription id hosting the remote tfstates" +} +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "global_settings" { + default = {} +} + + +variable "landingzone" {} +variable "rover_version" { + default = null +} +variable "tags" { + default = {} +} +variable "namespaces" { + default = {} +} + +variable "helm_charts" { + default = {} +} +variable "aks_clusters" {} + +variable "aks_cluster_key" {} + +variable "aks_cluster_vnet_key" {} + +variable "flux_settings" {} + +variable "vnets" {} +variable "managed_identities" { + description = "Map of the user managed identities." +} \ No newline at end of file From 7e446e02ffd7f8bf2762b40205200b391467b598 Mon Sep 17 00:00:00 2001 From: lolorol Date: Mon, 31 May 2021 03:02:04 +0000 Subject: [PATCH 16/65] Add sas_token to remote_tfstate different tenant --- caf_launchpad/landingzone.tf | 3 ++- caf_launchpad/local.remote.tf | 7 +++++ caf_launchpad/locals.remote_tfstates.tf | 36 +++++++++++++++++++++++++ caf_launchpad/variables.tf | 4 +++ caf_solution/landingzone.tf | 2 +- caf_solution/local.remote.tf | 3 +++ caf_solution/locals.remote_tfstates.tf | 1 + 7 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 caf_launchpad/local.remote.tf create mode 100644 caf_launchpad/locals.remote_tfstates.tf diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index e7ae25669..15dbf3095 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -3,7 +3,7 @@ module "launchpad" { # version = "~>5.3.2" source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" - + # source = "../../aztfmod" azuread_api_permissions = var.azuread_api_permissions azuread_apps = var.azuread_apps @@ -22,6 +22,7 @@ module "launchpad" { logged_aad_app_objectId = var.logged_aad_app_objectId logged_user_objectId = var.logged_user_objectId managed_identities = var.managed_identities + remote_objects = local.remote resource_groups = var.resource_groups role_mapping = var.role_mapping storage_accounts = var.storage_accounts diff --git a/caf_launchpad/local.remote.tf b/caf_launchpad/local.remote.tf new file mode 100644 index 000000000..bb78a9300 --- /dev/null +++ b/caf_launchpad/local.remote.tf @@ -0,0 +1,7 @@ +locals { + remote = { + azuread_service_principals = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_service_principals, {})) + } + } +} \ No newline at end of file diff --git a/caf_launchpad/locals.remote_tfstates.tf b/caf_launchpad/locals.remote_tfstates.tf new file mode 100644 index 000000000..ec34586a0 --- /dev/null +++ b/caf_launchpad/locals.remote_tfstates.tf @@ -0,0 +1,36 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] +} + +locals { + + remote_state = { + + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = value.workspace + key = value.tfstate + resource_group_name = value.resource_group_name + storage_account_name = value.storage_account_name + subscription_id = value.subscription_id + tenant_id = value.tenant_id + sas_token = try(value.sas_token, null) == "" ? var.sas_token : null + } + } + + } + +} \ No newline at end of file diff --git a/caf_launchpad/variables.tf b/caf_launchpad/variables.tf index 6ff3d9dd7..aadfda8cd 100644 --- a/caf_launchpad/variables.tf +++ b/caf_launchpad/variables.tf @@ -16,6 +16,10 @@ variable "tenant_id" {} variable "landingzone" { description = "The landing zone name is used to reference the tfstate in configuration files. Therefore while set it is recommended not to change" } +variable "sas_token" { + description = "SAS Token to access the remote state in another Azure AD tenant." + default = null +} variable "passthrough" { default = false diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 9cfc3326c..639f346c4 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -3,7 +3,7 @@ module "solution" { # version = "~>5.3.2" source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" - + # source = "../../aztfmod" azuread = local.azuread azuread_api_permissions = var.azuread_api_permissions diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index 3c6146228..3d8da0849 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -9,6 +9,9 @@ locals { azuread_groups = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) } + azuread_service_principals = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_service_principals, {})) + } azuread_users = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_users, {})) } diff --git a/caf_solution/locals.remote_tfstates.tf b/caf_solution/locals.remote_tfstates.tf index efe90da88..554b101f0 100644 --- a/caf_solution/locals.remote_tfstates.tf +++ b/caf_solution/locals.remote_tfstates.tf @@ -31,6 +31,7 @@ locals { storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) subscription_id = try(value.subscription_id, var.tfstate_subscription_id) tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + sas_token = try(value.sas_token, null) == "" ? var.sas_token : null } } From 336a9fe54823720adc3ddcd3cb4a46648b494ed2 Mon Sep 17 00:00:00 2001 From: lolorol Date: Mon, 31 May 2021 03:53:33 +0000 Subject: [PATCH 17/65] Fix mtms sas token for remote states --- caf_launchpad/locals.remote_tfstates.tf | 2 +- caf_solution/locals.remote_tfstates.tf | 2 +- caf_solution/variables.tf | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/caf_launchpad/locals.remote_tfstates.tf b/caf_launchpad/locals.remote_tfstates.tf index ec34586a0..88c96e810 100644 --- a/caf_launchpad/locals.remote_tfstates.tf +++ b/caf_launchpad/locals.remote_tfstates.tf @@ -27,7 +27,7 @@ locals { storage_account_name = value.storage_account_name subscription_id = value.subscription_id tenant_id = value.tenant_id - sas_token = try(value.sas_token, null) == "" ? var.sas_token : null + sas_token = try(value.sas_token, null) != null ? var.sas_token : null } } diff --git a/caf_solution/locals.remote_tfstates.tf b/caf_solution/locals.remote_tfstates.tf index 554b101f0..eb66538e1 100644 --- a/caf_solution/locals.remote_tfstates.tf +++ b/caf_solution/locals.remote_tfstates.tf @@ -31,7 +31,7 @@ locals { storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) subscription_id = try(value.subscription_id, var.tfstate_subscription_id) tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) - sas_token = try(value.sas_token, null) == "" ? var.sas_token : null + sas_token = try(value.sas_token, null) != null ? var.sas_token : null } } diff --git a/caf_solution/variables.tf b/caf_solution/variables.tf index e0b5d9cc9..be3fd1255 100644 --- a/caf_solution/variables.tf +++ b/caf_solution/variables.tf @@ -25,6 +25,10 @@ variable "tfstate_key" { variable "tfstate_resource_group_name" { default = null } +variable "sas_token" { + description = "SAS Token to access the remote state in another Azure AD tenant." + default = null +} variable "landingzone" { default = { From 9be14c2c989eb09c48c7e8ebf40e652a1c0c237d Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Hieu <5441003+hieumoscow@users.noreply.github.com> Date: Wed, 2 Jun 2021 15:44:53 +0800 Subject: [PATCH 18/65] Update kustomization.tf --- caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf index 969d1cf82..e3d7bcf2c 100644 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf @@ -203,7 +203,7 @@ data "kustomization_overlay" "placeholderjob" { patch = <<-EOF - op: replace path: /spec/template/spec/containers/0/env/0/value - value: "https://dev.azure.com/afopssre" + value: ${var.agent_pools.org_url} EOF target = { kind = "Job" From ef2728da16231c0ec7051e9a90b10944f83e2aec Mon Sep 17 00:00:00 2001 From: Hieu Nguyen Nhu <5441003+hieumoscow@users.noreply.github.com> Date: Wed, 2 Jun 2021 08:06:18 +0000 Subject: [PATCH 19/65] Add org_url for aks_azure_devops_agents addon --- caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf index 5a6a941af..6645e0d1f 100644 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf @@ -203,7 +203,7 @@ data "kustomization_overlay" "placeholderjob" { patch = <<-EOF - op: replace path: /spec/template/spec/containers/0/env/0/value - value: "https://dev.azure.com/afopssre" + value: ${var.agent_pools.org_url} EOF target = { kind = "Job" From 4b6f9cd9f32cce6917630b9a89a9ea9cc5f155e9 Mon Sep 17 00:00:00 2001 From: Hieu Nguyen Nhu <5441003+hieumoscow@users.noreply.github.com> Date: Wed, 2 Jun 2021 08:08:24 +0000 Subject: [PATCH 20/65] Fix kuztomize agent bug --- caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf index e3d7bcf2c..e1d3b6e5b 100644 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf @@ -88,7 +88,7 @@ data "kustomization_overlay" "azdopat-secret" { } module "kustomization" { - source = "./kustomize" + source = "../aks_applications/kustomize" for_each = try(data.kustomization_overlay.roverjob, {}) settings = each.value @@ -172,7 +172,7 @@ data "kustomization_overlay" "roverjob" { module "kustomization_placeholderagent" { - source = "./kustomize" + source = "../aks_applications/kustomize" for_each = try(data.kustomization_overlay.placeholderjob, {}) settings = each.value From e85e7c7e9bb6cf4bacbd2d6ec409b1046f8b99bc Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Hieu <5441003+hieumoscow@users.noreply.github.com> Date: Thu, 3 Jun 2021 10:01:26 +0800 Subject: [PATCH 21/65] Update kustomization.tf --- caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf index e1d3b6e5b..88033dae2 100644 --- a/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf +++ b/caf_solution/add-ons/aks_azure_devops_agents/kustomization.tf @@ -129,7 +129,7 @@ data "kustomization_overlay" "roverjob" { patch = <<-EOF - op: replace path: /spec/jobTargetRef/template/spec/containers/0/env/0/value - value: "https://dev.azure.com/afopssre" + value: ${var.agent_pools.org_url} EOF target = { kind = "ScaledJob" From 9339a7a22dae4777b366e7940db3889bfec42fbc Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Thu, 3 Jun 2021 14:37:52 -0700 Subject: [PATCH 22/65] Update flux.tf --- caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf index 44da6c6d8..99b9a6367 100644 --- a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf @@ -52,7 +52,7 @@ resource "kubectl_manifest" "install" { resource "kubectl_manifest" "sync" { for_each = { for v in local.sync : lower(join("/", compact([v.data.apiVersion, v.data.kind, lookup(v.data.metadata, "namespace", ""), v.data.metadata.name]))) => v.content } - depends_on = [kubernetes_namespace.flux_system] + depends_on = [kubernetes_namespace.flux_system, kubectl_manifest.install] yaml_body = each.value } From 091af7df9a05c0ef1c771b4f4954218fd23544fc Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Thu, 3 Jun 2021 16:10:44 -0700 Subject: [PATCH 23/65] aks_kubeconfig_cmd to output --- caf_solution/add-ons/aks_secure_baseline_v2/output.tf | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 caf_solution/add-ons/aks_secure_baseline_v2/output.tf diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/output.tf b/caf_solution/add-ons/aks_secure_baseline_v2/output.tf new file mode 100644 index 000000000..7a40f2a75 --- /dev/null +++ b/caf_solution/add-ons/aks_secure_baseline_v2/output.tf @@ -0,0 +1,7 @@ +output "aks_clusters_kubeconfig" { + value = { + aks_kubeconfig_admin_cmd = local.remote.aks_clusters.aks.cluster_re1.aks_kubeconfig_admin_cmd + aks_kubeconfig_cmd = local.remote.aks_clusters.aks.cluster_re1.aks_kubeconfig_cmd + } + sensitive = false +} From a7b6adcd9c8201271c2fbafb6d45030228e3ece8 Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Fri, 4 Jun 2021 05:26:27 +0000 Subject: [PATCH 24/65] added remote objects to support eslz --- caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf b/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf index 48fc5bc32..719130e82 100644 --- a/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf +++ b/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf @@ -46,6 +46,15 @@ locals { managed_identities = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) } + azuread_groups = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) + } + azuread_service_principals = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_service_principals, {})) + } + azuread_applications = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) + } } } From 8286338d5f49bac6fcb5f36230ddc0932d5014cb Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Fri, 4 Jun 2021 10:02:19 -0700 Subject: [PATCH 25/65] Update aks-pod-identity-assignment.tf --- .../aks-pod-identity-assignment.tf | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf index eab82971f..ed6818de6 100644 --- a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf +++ b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf @@ -1,67 +1,67 @@ -# Get the details of the node pool's resource group created by AKS -data "azurerm_resource_group" "noderg" { - for_each = var.aks_clusters - name = local.remote.aks_clusters[each.value.lz_key][each.value.key].node_resource_group -} - -# -# Set permissions to the kubelet and cluster identity -# -resource "azurerm_role_assignment" "kubelet_noderg_miop" { - for_each = var.aks_clusters - - scope = data.azurerm_resource_group.noderg[each.key].id - role_definition_name = "Managed Identity Operator" - principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id -} +# # Get the details of the node pool's resource group created by AKS +# data "azurerm_resource_group" "noderg" { +# for_each = var.aks_clusters +# name = local.remote.aks_clusters[each.value.lz_key][each.value.key].node_resource_group +# } -resource "azurerm_role_assignment" "kubelet_noderg_vmcontrib" { - for_each = var.aks_clusters +# # +# # Set permissions to the kubelet and cluster identity +# # +# resource "azurerm_role_assignment" "kubelet_noderg_miop" { +# for_each = var.aks_clusters - scope = data.azurerm_resource_group.noderg[each.key].id - role_definition_name = "Virtual Machine Contributor" - principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id -} +# scope = data.azurerm_resource_group.noderg[each.key].id +# role_definition_name = "Managed Identity Operator" +# principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +# } -# Separate subnet -resource "azurerm_role_assignment" "kubelet_subnets_networkcontrib" { - for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",{vnet=true}) +# resource "azurerm_role_assignment" "kubelet_noderg_vmcontrib" { +# for_each = var.aks_clusters - scope = try(each.value==true, false) ? local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id : local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].subnets[each.value].id - role_definition_name = "Network Contributor" - principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id -} +# scope = data.azurerm_resource_group.noderg[each.key].id +# role_definition_name = "Virtual Machine Contributor" +# principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +# } -# # Whole vnet -# resource "azurerm_role_assignment" "kubelet_vnet_networkcontrib" { -# for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",null) == null ? var.vnets : {} +# # Separate subnet +# resource "azurerm_role_assignment" "kubelet_subnets_networkcontrib" { +# for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",{vnet=true}) -# scope = local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id +# scope = try(each.value==true, false) ? local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id : local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].subnets[each.value].id # role_definition_name = "Network Contributor" # principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id # } -resource "azurerm_role_assignment" "kubelet_user_msi" { - for_each = local.msi_to_grant_permissions +# # # Whole vnet +# # resource "azurerm_role_assignment" "kubelet_vnet_networkcontrib" { +# # for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",null) == null ? var.vnets : {} - scope = each.value.id - role_definition_name = "Managed Identity Operator" - principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].kubelet_identity[0].object_id -} +# # scope = local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id +# # role_definition_name = "Network Contributor" +# # principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id +# # } -locals { - msi_to_grant_permissions = { - for msi in flatten( - [ - for key, value in var.managed_identities : [ - for msi_key in value.msi_keys : { - key = key - msi_key = msi_key - id = local.remote.managed_identities[value.lz_key][msi_key].id - } - ] - ] - ) : format("%s-%s", msi.key, msi.msi_key) => msi - } -} +# resource "azurerm_role_assignment" "kubelet_user_msi" { +# for_each = local.msi_to_grant_permissions + +# scope = each.value.id +# role_definition_name = "Managed Identity Operator" +# principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].kubelet_identity[0].object_id +# } + +# locals { +# msi_to_grant_permissions = { +# for msi in flatten( +# [ +# for key, value in var.managed_identities : [ +# for msi_key in value.msi_keys : { +# key = key +# msi_key = msi_key +# id = local.remote.managed_identities[value.lz_key][msi_key].id +# } +# ] +# ] +# ) : format("%s-%s", msi.key, msi.msi_key) => msi +# } +# } From 1073290103008cace8188f79fd2e01978eeb9ea5 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Fri, 4 Jun 2021 10:38:26 -0700 Subject: [PATCH 26/65] Update aks-pod-identity-assignment.tf --- .../aks-pod-identity-assignment.tf | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf index ed6818de6..eab82971f 100644 --- a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf +++ b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf @@ -1,67 +1,67 @@ -# # Get the details of the node pool's resource group created by AKS -# data "azurerm_resource_group" "noderg" { -# for_each = var.aks_clusters -# name = local.remote.aks_clusters[each.value.lz_key][each.value.key].node_resource_group -# } +# Get the details of the node pool's resource group created by AKS +data "azurerm_resource_group" "noderg" { + for_each = var.aks_clusters + name = local.remote.aks_clusters[each.value.lz_key][each.value.key].node_resource_group +} -# # -# # Set permissions to the kubelet and cluster identity -# # -# resource "azurerm_role_assignment" "kubelet_noderg_miop" { -# for_each = var.aks_clusters +# +# Set permissions to the kubelet and cluster identity +# +resource "azurerm_role_assignment" "kubelet_noderg_miop" { + for_each = var.aks_clusters -# scope = data.azurerm_resource_group.noderg[each.key].id -# role_definition_name = "Managed Identity Operator" -# principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id -# } + scope = data.azurerm_resource_group.noderg[each.key].id + role_definition_name = "Managed Identity Operator" + principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +} -# resource "azurerm_role_assignment" "kubelet_noderg_vmcontrib" { -# for_each = var.aks_clusters +resource "azurerm_role_assignment" "kubelet_noderg_vmcontrib" { + for_each = var.aks_clusters -# scope = data.azurerm_resource_group.noderg[each.key].id -# role_definition_name = "Virtual Machine Contributor" -# principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id -# } + scope = data.azurerm_resource_group.noderg[each.key].id + role_definition_name = "Virtual Machine Contributor" + principal_id = local.remote.aks_clusters[each.value.lz_key][each.value.key].kubelet_identity[0].object_id +} + +# Separate subnet +resource "azurerm_role_assignment" "kubelet_subnets_networkcontrib" { + for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",{vnet=true}) + + scope = try(each.value==true, false) ? local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id : local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].subnets[each.value].id + role_definition_name = "Network Contributor" + principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id +} -# # Separate subnet -# resource "azurerm_role_assignment" "kubelet_subnets_networkcontrib" { -# for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",{vnet=true}) +# # Whole vnet +# resource "azurerm_role_assignment" "kubelet_vnet_networkcontrib" { +# for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",null) == null ? var.vnets : {} -# scope = try(each.value==true, false) ? local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id : local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].subnets[each.value].id +# scope = local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id # role_definition_name = "Network Contributor" # principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id # } -# # # Whole vnet -# # resource "azurerm_role_assignment" "kubelet_vnet_networkcontrib" { -# # for_each = lookup(var.vnets[var.aks_cluster_vnet_key],"subnet_keys",null) == null ? var.vnets : {} +resource "azurerm_role_assignment" "kubelet_user_msi" { + for_each = local.msi_to_grant_permissions -# # scope = local.remote.vnets[var.vnets[var.aks_cluster_vnet_key].lz_key][var.vnets[var.aks_cluster_vnet_key].key].id -# # role_definition_name = "Network Contributor" -# # principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].identity[0].principal_id -# # } + scope = each.value.id + role_definition_name = "Managed Identity Operator" + principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].kubelet_identity[0].object_id +} -# resource "azurerm_role_assignment" "kubelet_user_msi" { -# for_each = local.msi_to_grant_permissions - -# scope = each.value.id -# role_definition_name = "Managed Identity Operator" -# principal_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_cluster_key].kubelet_identity[0].object_id -# } - -# locals { -# msi_to_grant_permissions = { -# for msi in flatten( -# [ -# for key, value in var.managed_identities : [ -# for msi_key in value.msi_keys : { -# key = key -# msi_key = msi_key -# id = local.remote.managed_identities[value.lz_key][msi_key].id -# } -# ] -# ] -# ) : format("%s-%s", msi.key, msi.msi_key) => msi -# } -# } +locals { + msi_to_grant_permissions = { + for msi in flatten( + [ + for key, value in var.managed_identities : [ + for msi_key in value.msi_keys : { + key = key + msi_key = msi_key + id = local.remote.managed_identities[value.lz_key][msi_key].id + } + ] + ] + ) : format("%s-%s", msi.key, msi.msi_key) => msi + } +} From 580530bc125f2635db957106dbecaab5e245edfe Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Fri, 4 Jun 2021 14:34:50 -0700 Subject: [PATCH 27/65] Rename aks-pod-identity-assignment.tf to aks-pod-identity-assignment.ignore --- ...-identity-assignment.tf => aks-pod-identity-assignment.ignore} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename caf_solution/add-ons/aks_secure_baseline_v2/{aks-pod-identity-assignment.tf => aks-pod-identity-assignment.ignore} (100%) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.ignore similarity index 100% rename from caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf rename to caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.ignore From 645d48992d0c68b3b908971a5e5124fe58f1fdd3 Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Hieu <5441003+hieumoscow@users.noreply.github.com> Date: Sat, 5 Jun 2021 09:19:12 +0800 Subject: [PATCH 28/65] Rename aks-pod-identity-assignment.ignore to aks-pod-identity-assignment.tf --- ...-identity-assignment.ignore => aks-pod-identity-assignment.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename caf_solution/add-ons/aks_secure_baseline_v2/{aks-pod-identity-assignment.ignore => aks-pod-identity-assignment.tf} (100%) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.ignore b/caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf similarity index 100% rename from caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.ignore rename to caf_solution/add-ons/aks_secure_baseline_v2/aks-pod-identity-assignment.tf From 39e5b490b83733df3d19c8235d9d1dcc231ae322 Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Hieu <5441003+hieumoscow@users.noreply.github.com> Date: Sat, 5 Jun 2021 11:14:14 +0800 Subject: [PATCH 29/65] Update flux.tf --- caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf index 99b9a6367..b0ec15a90 100644 --- a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf @@ -1,10 +1,10 @@ data "flux_install" "main" { - target_path = var.setting.target_install_path + target_path = var.setting.target_path namespace = var.setting.namespace } data "flux_sync" "main" { - target_path = var.setting.target_sync_path + target_path = var.setting.target_path url = var.setting.url branch = var.setting.branch secret = try(var.setting.flux_auth_secret, null) From b9b021a5e8b3536d3f14e313d42c90168757aa21 Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Hieu <5441003+hieumoscow@users.noreply.github.com> Date: Sat, 5 Jun 2021 11:18:10 +0800 Subject: [PATCH 30/65] Update flux.tf --- caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf index b0ec15a90..f3cae3f16 100644 --- a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf @@ -26,7 +26,7 @@ resource "kubernetes_namespace" "flux_system" { lifecycle { ignore_changes = [ - metadata[0].labels, + metadata[0].labels, metadata[0].annotations ] } } From 92701044092cf7544e50e41c5491a20aaf0b1e60 Mon Sep 17 00:00:00 2001 From: lolorol Date: Sat, 5 Jun 2021 06:51:48 +0000 Subject: [PATCH 31/65] Update to mtms module --- caf_solution/landingzone.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 0c776ae02..332ed3550 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" - source = "../../aztfmod" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" azuread = local.azuread cloud = local.cloud From 87889ae4a40adc6c1d25a553e5b0ce0eae3c3c1e Mon Sep 17 00:00:00 2001 From: lolorol Date: Sun, 6 Jun 2021 10:01:39 +0000 Subject: [PATCH 32/65] Add azuread_password_policies --- caf_solution/landingzone.tf | 4 ++-- caf_solution/local.azuread.tf | 1 + caf_solution/variables.azuread.tf | 21 ++++++++++----------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 332ed3550..411ddaa03 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - # source = "../../aztfmod" + # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + source = "../../aztfmod" azuread = local.azuread cloud = local.cloud diff --git a/caf_solution/local.azuread.tf b/caf_solution/local.azuread.tf index 098386d3d..c05966caf 100644 --- a/caf_solution/local.azuread.tf +++ b/caf_solution/local.azuread.tf @@ -6,6 +6,7 @@ locals { azuread_applications = var.azuread_applications azuread_apps = var.azuread_apps azuread_groups = var.azuread_groups + azuread_password_policies = var.azuread_password_policies azuread_roles = var.azuread_roles azuread_service_principal_passwords = var.azuread_service_principal_passwords azuread_service_principals = var.azuread_service_principals diff --git a/caf_solution/variables.azuread.tf b/caf_solution/variables.azuread.tf index 6692ef214..ac3ec0b88 100644 --- a/caf_solution/variables.azuread.tf +++ b/caf_solution/variables.azuread.tf @@ -1,31 +1,30 @@ variable "azuread" { default = {} } +variable "azuread_api_permissions" { + default = {} +} variable "azuread_applications" { default = {} } variable "azuread_apps" { default = {} } -variable "azuread_service_principals" { +variable "azuread_groups" { default = {} } -variable "azuread_service_principal_passwords" { +variable "azuread_password_policies" { default = {} } - -variable "azuread_groups" { +variable "azuread_roles" { default = {} } - -variable "azuread_roles" { +variable "azuread_service_principals" { default = {} } - -variable "azuread_users" { +variable "azuread_service_principal_passwords" { default = {} } - -variable "azuread_api_permissions" { +variable "azuread_users" { default = {} -} \ No newline at end of file +} From fbd2fb239b7071d1449dea03afc774e006e8c7d0 Mon Sep 17 00:00:00 2001 From: lolorol Date: Sun, 6 Jun 2021 10:18:07 +0000 Subject: [PATCH 33/65] Update launchpad with azuread variables --- caf_launchpad/landingzone.tf | 21 ++++++++++++++------- caf_launchpad/variables.tf | 13 +++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index 15dbf3095..626a5b3ba 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -2,14 +2,9 @@ module "launchpad" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=azuread_serviceprincipal" - # source = "../../aztfmod" + # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + source = "../../aztfmod" - azuread_api_permissions = var.azuread_api_permissions - azuread_apps = var.azuread_apps - azuread_groups = var.azuread_groups - azuread_roles = var.azuread_roles - azuread_users = var.azuread_users current_landingzone_key = var.landingzone.key custom_role_definitions = var.custom_role_definitions enable = var.enable @@ -31,6 +26,18 @@ module "launchpad" { tenant_id = var.tenant_id user_type = var.user_type + azuread = { + azuread_api_permissions = var.azuread_api_permissions + azuread_applications = var.azuread_applications + azuread_apps = var.azuread_apps + azuread_groups = var.azuread_groups + azuread_password_policies = var.azuread_password_policies + azuread_roles = var.azuread_roles + azuread_service_principal_passwords = var.azuread_service_principal_passwords + azuread_service_principals = var.azuread_service_principals + azuread_users = var.azuread_users + } + diagnostics = { diagnostics_definition = try(var.diagnostics.diagnostics_definition, var.diagnostics_definition) diagnostics_destinations = try(var.diagnostics.diagnostics_destinations, var.diagnostics_destinations) diff --git a/caf_launchpad/variables.tf b/caf_launchpad/variables.tf index 2b5c56f20..05ef0710b 100644 --- a/caf_launchpad/variables.tf +++ b/caf_launchpad/variables.tf @@ -104,6 +104,15 @@ variable "azuread_users" { variable "azuread_roles" { default = {} } +variable "azuread_password_policies" { + default = {} +} +variable "azuread_service_principals" { + default = {} +} +variable "azuread_service_principal_passwords" { + default = {} +} variable "managed_identities" { default = {} } @@ -156,6 +165,10 @@ variable "azuread_api_permissions" { default = {} } +variable "azuread_applications" { + default = {} +} + variable "environment" { type = string description = "This variable is set by the rover during the deployment based on the -env or -environment flags. Default to sandpit" From ee78cd24dd03f1e697804fa8d4faa5e7d851218f Mon Sep 17 00:00:00 2001 From: lolorol Date: Sun, 6 Jun 2021 15:40:43 +0000 Subject: [PATCH 34/65] Fix mtms bootstrap --- caf_launchpad/landingzone.tf | 2 +- caf_launchpad/variables.tf | 2 +- caf_solution/local.azuread.tf | 3 ++- caf_solution/locals.remote_tfstates.tf | 6 +++--- caf_solution/main.tf | 2 +- caf_solution/variables.azuread.tf | 5 ++++- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index 626a5b3ba..fe3115875 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -30,8 +30,8 @@ module "launchpad" { azuread_api_permissions = var.azuread_api_permissions azuread_applications = var.azuread_applications azuread_apps = var.azuread_apps + azuread_credential_policies = var.azuread_credential_policies azuread_groups = var.azuread_groups - azuread_password_policies = var.azuread_password_policies azuread_roles = var.azuread_roles azuread_service_principal_passwords = var.azuread_service_principal_passwords azuread_service_principals = var.azuread_service_principals diff --git a/caf_launchpad/variables.tf b/caf_launchpad/variables.tf index 05ef0710b..55482506c 100644 --- a/caf_launchpad/variables.tf +++ b/caf_launchpad/variables.tf @@ -104,7 +104,7 @@ variable "azuread_users" { variable "azuread_roles" { default = {} } -variable "azuread_password_policies" { +variable "azuread_credential_policies" { default = {} } variable "azuread_service_principals" { diff --git a/caf_solution/local.azuread.tf b/caf_solution/local.azuread.tf index c05966caf..51f5d7bae 100644 --- a/caf_solution/local.azuread.tf +++ b/caf_solution/local.azuread.tf @@ -5,8 +5,9 @@ locals { azuread_api_permissions = var.azuread_api_permissions azuread_applications = var.azuread_applications azuread_apps = var.azuread_apps + azuread_credential_policies = var.azuread_credential_policies + azuread_credentials = var.azuread_credentials azuread_groups = var.azuread_groups - azuread_password_policies = var.azuread_password_policies azuread_roles = var.azuread_roles azuread_service_principal_passwords = var.azuread_service_principal_passwords azuread_service_principals = var.azuread_service_principals diff --git a/caf_solution/locals.remote_tfstates.tf b/caf_solution/locals.remote_tfstates.tf index 7b78ed381..867e1452b 100644 --- a/caf_solution/locals.remote_tfstates.tf +++ b/caf_solution/locals.remote_tfstates.tf @@ -75,16 +75,16 @@ locals { ) } # Get the remote existing diagnostics objects - storage_accounts = coalesce( + storage_accounts = merge( try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.diagnostics.storage_accounts, null), try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.storage_accounts, null) ) - log_analytics = coalesce( + log_analytics = merge( try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.diagnostics.log_analytics, null), try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.log_analytics, null) ) - event_hub_namespaces = coalesce( + event_hub_namespaces = merge( try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.diagnostics.event_hub_namespaces, null), try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics.event_hub_namespaces, null) ) diff --git a/caf_solution/main.tf b/caf_solution/main.tf index e2aaad5b2..a8ec361aa 100644 --- a/caf_solution/main.tf +++ b/caf_solution/main.tf @@ -48,7 +48,7 @@ locals { } ) , - data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates + try(data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates, {}) ) diff --git a/caf_solution/variables.azuread.tf b/caf_solution/variables.azuread.tf index ac3ec0b88..6230c47f8 100644 --- a/caf_solution/variables.azuread.tf +++ b/caf_solution/variables.azuread.tf @@ -10,10 +10,13 @@ variable "azuread_applications" { variable "azuread_apps" { default = {} } +variable "azuread_credentials" { + default = {} +} variable "azuread_groups" { default = {} } -variable "azuread_password_policies" { +variable "azuread_credential_policies" { default = {} } variable "azuread_roles" { From 59160f46c42e765d65b220741083015869370629 Mon Sep 17 00:00:00 2001 From: lolorol Date: Sun, 6 Jun 2021 15:45:12 +0000 Subject: [PATCH 35/65] Update source --- caf_launchpad/landingzone.tf | 4 ++-- caf_solution/landingzone.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index fe3115875..5b90984a0 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -2,8 +2,8 @@ module "launchpad" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - source = "../../aztfmod" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" current_landingzone_key = var.landingzone.key custom_role_definitions = var.custom_role_definitions diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 411ddaa03..332ed3550 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - source = "../../aztfmod" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" azuread = local.azuread cloud = local.cloud From b8f325eef41b5274de31b9452ec54d9c0e7147ac Mon Sep 17 00:00:00 2001 From: Hieu Nguyen Nhu <5441003+hieumoscow@users.noreply.github.com> Date: Mon, 7 Jun 2021 23:40:45 +0000 Subject: [PATCH 36/65] Add flux_install_yaml_documents_without_namespace --- caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf index f3cae3f16..102b9c5d2 100644 --- a/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf +++ b/caf_solution/add-ons/aks_secure_baseline_v2/flux/flux.tf @@ -32,7 +32,8 @@ resource "kubernetes_namespace" "flux_system" { } locals { - install = [for v in data.kubectl_file_documents.install.documents : { + flux_install_yaml_documents_without_namespace = [for x in data.kubectl_file_documents.install.documents: x if length(regexall("kind: Namespace", x)) == 0] + install = [for v in local.flux_install_yaml_documents_without_namespace : { data : yamldecode(v) content : v } From e0bb4b72fc9a3af193349f652117e5f4e87f2655 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Tue, 8 Jun 2021 18:20:15 +1000 Subject: [PATCH 37/65] hahicorp vault add on --- .../dynamic_vault_secrets/backend.azurerm | 4 + .../local.remote_tfstates.tf | 72 ++++++++++++ .../add-ons/dynamic_vault_secrets/main.tf | 66 +++++++++++ .../dynamic_vault_secrets/providers.tf | 10 ++ .../dynamic_vault_secrets/secret/variables.tf | 6 + .../secret/vault_secret.tf | 15 +++ .../add-ons/dynamic_vault_secrets/solution.tf | 36 ++++++ .../dynamic_vault_secrets/variables.tf | 103 ++++++++++++++++++ .../add-ons/dynamic_vault_secrets/vault.tf | 13 +++ 9 files changed, 325 insertions(+) create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/backend.azurerm create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/main.tf create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/providers.tf create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/secret/variables.tf create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/solution.tf create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/variables.tf create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/vault.tf diff --git a/caf_solution/add-ons/dynamic_vault_secrets/backend.azurerm b/caf_solution/add-ons/dynamic_vault_secrets/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf b/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf new file mode 100644 index 000000000..230e41470 --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf @@ -0,0 +1,72 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name + container_name = local.landingzone[try(each.value.level, "current")].container_name + resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name + subscription_id = var.tfstate_subscription_id + key = each.value.tfstate + } +} + +locals { + landingzone_tag = { + "landingzone" = var.landingzone.key + } + + tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) + + global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings + diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics + + combined = { + aad_apps = merge(local.remote.aad_apps, tomap({ (var.landingzone.key) = module.caf.aad_apps })) + azuread_groups = merge(local.remote.azuread_groups, tomap({ (var.landingzone.key) = module.caf.azuread_groups })) + keyvaults = merge(local.remote.keyvaults, tomap({ (var.landingzone.key) = module.caf.keyvaults })) + managed_identities = merge(local.remote.managed_identities, tomap({ (var.landingzone.key) = module.caf.managed_identities })) + objects = merge(local.remote.objects, tomap({ (var.landingzone.key) = module.caf })) + + } + + remote = { + objects = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key], {})) + } + aad_apps = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aad_apps, {})) + } + + azuread_groups = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) + } + + keyvaults = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) + } + + managed_identities = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) + } + + vnets = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) + } + } +} diff --git a/caf_solution/add-ons/dynamic_vault_secrets/main.tf b/caf_solution/add-ons/dynamic_vault_secrets/main.tf new file mode 100644 index 000000000..b0197911a --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/main.tf @@ -0,0 +1,66 @@ +terraform { + required_providers { + azuread = { + source = "hashicorp/azuread" + version = "~> 1.4.0" + } + random = { + source = "hashicorp/random" + version = "~> 2.2.1" + } + null = { + source = "hashicorp/null" + version = "~> 2.1.0" + } + external = { + source = "hashicorp/external" + version = "~> 1.2.0" + } + azuredevops = { + source = "microsoft/azuredevops" + version = "~> 0.1.0" + } + tls = { + source = "hashicorp/tls" + version = "~> 2.2.0" + } + azurecaf = { + source = "aztfmod/azurecaf" + version = "~> 1.2.0" + } + vault = { + source = "hashicorp/vault" + version = "2.17.0" + } + } + required_version = ">= 0.13" +} + + +locals { + + # Update the tfstates map + tfstates = merge( + tomap( + { + (var.landingzone.key) = local.backend[var.landingzone.backend_type] + } + ) + , + data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates + ) + + + backend = { + azurerm = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + key = var.tfstate_key + level = var.landingzone.level + tenant_id = var.tenant_id + subscription_id = data.azurerm_client_config.current.subscription_id + } + } + +} diff --git a/caf_solution/add-ons/dynamic_vault_secrets/providers.tf b/caf_solution/add-ons/dynamic_vault_secrets/providers.tf new file mode 100644 index 000000000..efbfdc213 --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/providers.tf @@ -0,0 +1,10 @@ +provider "vault" {} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = true + } + } +} +data "azurerm_client_config" "current" {} diff --git a/caf_solution/add-ons/dynamic_vault_secrets/secret/variables.tf b/caf_solution/add-ons/dynamic_vault_secrets/secret/variables.tf new file mode 100644 index 000000000..e607d024b --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/secret/variables.tf @@ -0,0 +1,6 @@ +variable "secrets" {} +variable "path" {} +variable "disable_read" {} +variable "objects" { + default = {} +} \ No newline at end of file diff --git a/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf b/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf new file mode 100644 index 000000000..a20887e4c --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf @@ -0,0 +1,15 @@ +locals { + transposed = { + for key, value in var.secrets : key => coalesce( + try(value.value, null), + try(var.objects[value.lz_key][value.output_key][value.resource_key][value.attribute_key], null), + try(var.objects[value.lz_key][value.output_key][value.attribute_key], null) + ) } +} +#try(each.value.value, null) == null ? try(var.objects[each.value.output_key][each.value.resource_key][each.value.attribute_key], var.objects[each.value.output_key][each.value.attribute_key]) : each.value.value + +resource "vault_generic_secret" "azuresecrets" { + path = var.path + disable_read = try(var.disable_read, false) + data_json = jsonencode(local.transposed) +} \ No newline at end of file diff --git a/caf_solution/add-ons/dynamic_vault_secrets/solution.tf b/caf_solution/add-ons/dynamic_vault_secrets/solution.tf new file mode 100644 index 000000000..070af9c68 --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/solution.tf @@ -0,0 +1,36 @@ +module "caf" { + source = "aztfmod/caf/azurerm" + version = "~>5.3.0" + + current_landingzone_key = var.landingzone.key + tenant_id = var.tenant_id + tfstates = local.tfstates + tags = local.tags + global_settings = local.global_settings + diagnostics = local.diagnostics + diagnostic_storage_accounts = var.diagnostic_storage_accounts + logged_user_objectId = var.logged_user_objectId + logged_aad_app_objectId = var.logged_aad_app_objectId + resource_groups = var.resource_groups + storage_accounts = var.storage_accounts + azuread_groups = var.azuread_groups + keyvaults = var.keyvaults + keyvault_access_policies = var.keyvault_access_policies + keyvault_access_policies_azuread_apps = var.keyvault_access_policies_azuread_apps + role_mapping = var.role_mapping + custom_role_definitions = var.custom_role_definitions + azuread_apps = var.azuread_apps + compute = { + virtual_machines = var.virtual_machines + } + storage = { + storage_account_blobs = var.storage_account_blobs + } + + remote_objects = { + keyvaults = local.remote.keyvaults + vnets = local.remote.vnets + managed_identities = local.remote.managed_identities + azuread_groups = local.remote.azuread_groups + } +} diff --git a/caf_solution/add-ons/dynamic_vault_secrets/variables.tf b/caf_solution/add-ons/dynamic_vault_secrets/variables.tf new file mode 100644 index 000000000..9d17b7cab --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/variables.tf @@ -0,0 +1,103 @@ +# Map of the remote data state for lower level +variable "lower_storage_account_name" {} +variable "lower_container_name" {} +variable "lower_resource_group_name" {} + +variable "tfstate_storage_account_name" {} +variable "tfstate_container_name" {} +variable "tfstate_key" {} +variable "tfstate_resource_group_name" {} + +variable "tfstate_subscription_id" { + description = "This value is propulated by the rover. subscription id hosting the remote tfstates" +} + +variable "global_settings" { + default = {} +} +variable "tenant_id" {} +variable "landingzone" { +} +variable "rover_version" { + default = null +} + +variable "logged_user_objectId" { + default = null +} +variable "logged_aad_app_objectId" { + default = null +} +variable "tags" { + default = null +} +variable "app_service_environments" { + default = {} +} +variable "app_service_plans" { + default = {} +} +variable "app_services" { + default = {} +} +variable "diagnostics_definition" { + default = {} +} +variable "resource_groups" { + default = {} +} +variable "network_security_group_definition" { + default = {} +} +variable "vnets" { + default = {} +} +variable "azurerm_redis_caches" { + default = {} +} +variable "mssql_servers" { + default = {} +} +variable "storage_accounts" { + default = {} +} +variable "storage_account_blobs" { + default = {} +} +variable "azuread_groups" { + default = {} +} +variable "keyvaults" { + default = {} +} +variable "keyvault_access_policies" { + default = {} +} +variable "keyvault_access_policies_azuread_apps" { + default = {} +} +variable "virtual_machines" { + default = {} +} +variable "diagnostic_storage_accounts" { + default = {} +} +variable "virtual_machine_extension_scripts" { + default = {} +} +variable "azure_devops" { + default = {} +} +variable "role_mapping" { + default = {} +} +variable "custom_role_definitions" { + default = {} +} +variable "azuread_apps" { + default = {} +} +variable "dynamic_keyvault_secrets" { + default = {} +} +variable "dynamic_vault_secrets" {} \ No newline at end of file diff --git a/caf_solution/add-ons/dynamic_vault_secrets/vault.tf b/caf_solution/add-ons/dynamic_vault_secrets/vault.tf new file mode 100644 index 000000000..d306e03ba --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/vault.tf @@ -0,0 +1,13 @@ +module "vault_secret" { + source = "./secret" + depends_on = [module.caf] + for_each = { + for key, value in var.dynamic_vault_secrets : key => value + if try(value.secrets, null) != null && try(value.secrets, null) != "" + } + secrets = each.value.secrets + path = each.value.path + disable_read = try(each.value.disable_read, false) + objects = local.combined.objects +} + From 15d641b155a4de509ff29e3205a5eb080532193e Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Tue, 8 Jun 2021 18:25:29 +1000 Subject: [PATCH 38/65] added uniform module name --- caf_solution/add-ons/dynamic_vault_secrets/vault.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/vault.tf b/caf_solution/add-ons/dynamic_vault_secrets/vault.tf index d306e03ba..1451481bd 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/vault.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/vault.tf @@ -1,4 +1,4 @@ -module "vault_secret" { +module "dynamic_vault_secret" { source = "./secret" depends_on = [module.caf] for_each = { From b4268f3bce464f193724d0791811567baeafd709 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 00:15:37 +1000 Subject: [PATCH 39/65] unload caf solution and removed unwanted variables --- .../local.remote_tfstates.tf | 30 ---------------- .../add-ons/dynamic_vault_secrets/main.tf | 31 +--------------- .../add-ons/dynamic_vault_secrets/solution.tf | 36 ------------------- .../dynamic_vault_secrets/variables.tf | 1 - .../add-ons/dynamic_vault_secrets/vault.tf | 10 +++--- 5 files changed, 5 insertions(+), 103 deletions(-) delete mode 100644 caf_solution/add-ons/dynamic_vault_secrets/solution.tf diff --git a/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf b/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf index 230e41470..473213da2 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf @@ -34,39 +34,9 @@ locals { tags = merge(local.global_settings.tags, local.landingzone_tag, { "level" = var.landingzone.level }, { "environment" = local.global_settings.environment }, { "rover_version" = var.rover_version }, var.tags) global_settings = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].global_settings - diagnostics = data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.objects[var.landingzone.global_settings_key].diagnostics - - combined = { - aad_apps = merge(local.remote.aad_apps, tomap({ (var.landingzone.key) = module.caf.aad_apps })) - azuread_groups = merge(local.remote.azuread_groups, tomap({ (var.landingzone.key) = module.caf.azuread_groups })) - keyvaults = merge(local.remote.keyvaults, tomap({ (var.landingzone.key) = module.caf.keyvaults })) - managed_identities = merge(local.remote.managed_identities, tomap({ (var.landingzone.key) = module.caf.managed_identities })) - objects = merge(local.remote.objects, tomap({ (var.landingzone.key) = module.caf })) - - } - remote = { objects = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key], {})) } - aad_apps = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].aad_apps, {})) - } - - azuread_groups = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_groups, {})) - } - - keyvaults = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].keyvaults, {})) - } - - managed_identities = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].managed_identities, {})) - } - - vnets = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) - } } } diff --git a/caf_solution/add-ons/dynamic_vault_secrets/main.tf b/caf_solution/add-ons/dynamic_vault_secrets/main.tf index b0197911a..8a5abbae5 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/main.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/main.tf @@ -34,33 +34,4 @@ terraform { } } required_version = ">= 0.13" -} - - -locals { - - # Update the tfstates map - tfstates = merge( - tomap( - { - (var.landingzone.key) = local.backend[var.landingzone.backend_type] - } - ) - , - data.terraform_remote_state.remote[var.landingzone.global_settings_key].outputs.tfstates - ) - - - backend = { - azurerm = { - storage_account_name = var.tfstate_storage_account_name - container_name = var.tfstate_container_name - resource_group_name = var.tfstate_resource_group_name - key = var.tfstate_key - level = var.landingzone.level - tenant_id = var.tenant_id - subscription_id = data.azurerm_client_config.current.subscription_id - } - } - -} +} \ No newline at end of file diff --git a/caf_solution/add-ons/dynamic_vault_secrets/solution.tf b/caf_solution/add-ons/dynamic_vault_secrets/solution.tf deleted file mode 100644 index 070af9c68..000000000 --- a/caf_solution/add-ons/dynamic_vault_secrets/solution.tf +++ /dev/null @@ -1,36 +0,0 @@ -module "caf" { - source = "aztfmod/caf/azurerm" - version = "~>5.3.0" - - current_landingzone_key = var.landingzone.key - tenant_id = var.tenant_id - tfstates = local.tfstates - tags = local.tags - global_settings = local.global_settings - diagnostics = local.diagnostics - diagnostic_storage_accounts = var.diagnostic_storage_accounts - logged_user_objectId = var.logged_user_objectId - logged_aad_app_objectId = var.logged_aad_app_objectId - resource_groups = var.resource_groups - storage_accounts = var.storage_accounts - azuread_groups = var.azuread_groups - keyvaults = var.keyvaults - keyvault_access_policies = var.keyvault_access_policies - keyvault_access_policies_azuread_apps = var.keyvault_access_policies_azuread_apps - role_mapping = var.role_mapping - custom_role_definitions = var.custom_role_definitions - azuread_apps = var.azuread_apps - compute = { - virtual_machines = var.virtual_machines - } - storage = { - storage_account_blobs = var.storage_account_blobs - } - - remote_objects = { - keyvaults = local.remote.keyvaults - vnets = local.remote.vnets - managed_identities = local.remote.managed_identities - azuread_groups = local.remote.azuread_groups - } -} diff --git a/caf_solution/add-ons/dynamic_vault_secrets/variables.tf b/caf_solution/add-ons/dynamic_vault_secrets/variables.tf index 9d17b7cab..a45870b27 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/variables.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/variables.tf @@ -21,7 +21,6 @@ variable "landingzone" { variable "rover_version" { default = null } - variable "logged_user_objectId" { default = null } diff --git a/caf_solution/add-ons/dynamic_vault_secrets/vault.tf b/caf_solution/add-ons/dynamic_vault_secrets/vault.tf index 1451481bd..e46c77e8b 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/vault.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/vault.tf @@ -1,13 +1,11 @@ module "dynamic_vault_secret" { - source = "./secret" - depends_on = [module.caf] + source = "./secret" for_each = { for key, value in var.dynamic_vault_secrets : key => value - if try(value.secrets, null) != null && try(value.secrets, null) != "" + if try(value.secrets, null) != null && try(value.secrets, null) != "" && try(value.path, null) != null && try(value.path, null) != "" } secrets = each.value.secrets path = each.value.path disable_read = try(each.value.disable_read, false) - objects = local.combined.objects -} - + objects = local.remote.objects +} \ No newline at end of file From b9588dc7b11cc90b82f1b3f5e8544f1791fef138 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 00:23:11 +1000 Subject: [PATCH 40/65] removed unused variables --- .../add-ons/dynamic_vault_secrets/main.tf | 4 ++ .../dynamic_vault_secrets/variables.tf | 69 ------------------- 2 files changed, 4 insertions(+), 69 deletions(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/main.tf b/caf_solution/add-ons/dynamic_vault_secrets/main.tf index 8a5abbae5..e38daa2e3 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/main.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/main.tf @@ -1,5 +1,9 @@ terraform { required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.43" + } azuread = { source = "hashicorp/azuread" version = "~> 1.4.0" diff --git a/caf_solution/add-ons/dynamic_vault_secrets/variables.tf b/caf_solution/add-ons/dynamic_vault_secrets/variables.tf index a45870b27..3d9784f56 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/variables.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/variables.tf @@ -30,73 +30,4 @@ variable "logged_aad_app_objectId" { variable "tags" { default = null } -variable "app_service_environments" { - default = {} -} -variable "app_service_plans" { - default = {} -} -variable "app_services" { - default = {} -} -variable "diagnostics_definition" { - default = {} -} -variable "resource_groups" { - default = {} -} -variable "network_security_group_definition" { - default = {} -} -variable "vnets" { - default = {} -} -variable "azurerm_redis_caches" { - default = {} -} -variable "mssql_servers" { - default = {} -} -variable "storage_accounts" { - default = {} -} -variable "storage_account_blobs" { - default = {} -} -variable "azuread_groups" { - default = {} -} -variable "keyvaults" { - default = {} -} -variable "keyvault_access_policies" { - default = {} -} -variable "keyvault_access_policies_azuread_apps" { - default = {} -} -variable "virtual_machines" { - default = {} -} -variable "diagnostic_storage_accounts" { - default = {} -} -variable "virtual_machine_extension_scripts" { - default = {} -} -variable "azure_devops" { - default = {} -} -variable "role_mapping" { - default = {} -} -variable "custom_role_definitions" { - default = {} -} -variable "azuread_apps" { - default = {} -} -variable "dynamic_keyvault_secrets" { - default = {} -} variable "dynamic_vault_secrets" {} \ No newline at end of file From f4b3916129fa9f26efd94f0f942268935765a409 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 00:48:24 +1000 Subject: [PATCH 41/65] added readme and example to deploy --- .../add-ons/dynamic_vault_secrets/README.md | 41 ++++++++++++++ .../configuration.tfvars | 53 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/README.md create mode 100644 caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets/configuration.tfvars diff --git a/caf_solution/add-ons/dynamic_vault_secrets/README.md b/caf_solution/add-ons/dynamic_vault_secrets/README.md new file mode 100644 index 000000000..b1ce7f71f --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/README.md @@ -0,0 +1,41 @@ +# CAF landing zones for Terraform - Dynamic Hashicorp Vault Secrets Add-on + +Deploys dynamic hashicop vault secrets to a given path + + +## Prerequisites + +Before running the add-on please make sure you set your vault backend, you can do this by exporting below environment variables. + +``` bash +export VAULT_ADDR = "vault address" +export VAULT_TOKEN= "vault token" + +``` + +## Example + +The example configurations to deploy this add-on can be found [here](./scenario/100-simple-vault-secrets/configuration.tfvars) + +Ensure the below is set prior to apply or destroy. + +```bash +# Login the Azure subscription +rover login -t [TENANT_ID/TENANT_NAME] -s [SUBSCRIPTION_GUID] +# Environment is needed to be defined, otherwise the below LZs will land into sandpit which someone else is working on +export environment=[YOUR_ENVIRONMENT] +``` + +## Run vault dynamic serets deployment + +```bash +rover \ + -lz /tf/caf/landingzones/caf_solution/add-ons/dynamic_vault_secrets \ + -var-folder /tf/caf/landingzones/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets \ + -tfstate vault.tfstate \ + -env ${environment}} \ + -level level1 \ + -parallelism 50 \ + -a plan + +``` \ No newline at end of file diff --git a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets/configuration.tfvars b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets/configuration.tfvars new file mode 100644 index 000000000..e75c43635 --- /dev/null +++ b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets/configuration.tfvars @@ -0,0 +1,53 @@ +landingzone = { + backend_type = "azurerm" + global_settings_key = "launchpad" + level = "level1" + key = "vault" + tfstates = { + launchpad = { + level = "lower" + tfstate = "caf_launchpad.tfstate" + } + } +} +dynamic_vault_secrets = { + vmadmin-username = { + path = "secret/test" + secrets = { + username = { + value = "vmadmin" + } + password = { + value = "password" + } + subscription_id = { + lz_key = "launchpad" + output_key = "client_config" + attribute_key = "subscription_id" + } + lower_rg = { + lz_key = "launchpad" + output_key = "resource_groups" + resource_key = "level0" + attribute_key = "name" + } + lower_rg_1 = { + lz_key = "launchpad" + output_key = "resource_groups" + resource_key = "level1" + attribute_key = "name" + } + } + } + vmadmin-password = { + path = "secret/password" + secrets = { + username = { + value = "vmadmin" + } + password = { + value = "Welcome@1" + } + } + } +} \ No newline at end of file From bc4b20400f18104bb56494469be4ec0fccf3c230 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 00:57:33 +1000 Subject: [PATCH 42/65] modified readme and folder name --- caf_solution/add-ons/dynamic_vault_secrets/README.md | 6 +++--- .../configuration.tfvars | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename caf_solution/add-ons/dynamic_vault_secrets/scenario/{100-simple-vault-secrets => 100-simple-dynamic-vault-secrets}/configuration.tfvars (100%) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/README.md b/caf_solution/add-ons/dynamic_vault_secrets/README.md index b1ce7f71f..c72dc91c1 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/README.md +++ b/caf_solution/add-ons/dynamic_vault_secrets/README.md @@ -1,6 +1,6 @@ # CAF landing zones for Terraform - Dynamic Hashicorp Vault Secrets Add-on -Deploys dynamic hashicop vault secrets to a given path +Deploys dynamic hashicop vault secrets. ## Prerequisites @@ -15,7 +15,7 @@ export VAULT_TOKEN= "vault token" ## Example -The example configurations to deploy this add-on can be found [here](./scenario/100-simple-vault-secrets/configuration.tfvars) +The example configurations to deploy this add-on can be found [here](./scenario/100-simple-dynamic-vault-secrets/configuration.tfvars) Ensure the below is set prior to apply or destroy. @@ -31,7 +31,7 @@ export environment=[YOUR_ENVIRONMENT] ```bash rover \ -lz /tf/caf/landingzones/caf_solution/add-ons/dynamic_vault_secrets \ - -var-folder /tf/caf/landingzones/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets \ + -var-folder /tf/caf/landingzones/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets \ -tfstate vault.tfstate \ -env ${environment}} \ -level level1 \ diff --git a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets/configuration.tfvars b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars similarity index 100% rename from caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-vault-secrets/configuration.tfvars rename to caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars From 2c4a2843c0fc1b53c074d6f6a4ddbb75cbd2646e Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 08:28:56 +1000 Subject: [PATCH 43/65] readme updated and generic example --- caf_solution/add-ons/dynamic_vault_secrets/README.md | 4 ++-- .../configuration.tfvars | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/README.md b/caf_solution/add-ons/dynamic_vault_secrets/README.md index c72dc91c1..9c24489e4 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/README.md +++ b/caf_solution/add-ons/dynamic_vault_secrets/README.md @@ -5,7 +5,7 @@ Deploys dynamic hashicop vault secrets. ## Prerequisites -Before running the add-on please make sure you set your vault backend, you can do this by exporting below environment variables. +Before running the add-on please make sure you are [authenticated](https://learn.hashicorp.com/tutorials/vault/getting-started-authentication) to vault, one of the way is to set the below environment variables. For more information visit : https://www.vaultproject.io/docs/commands#environment-variables ``` bash export VAULT_ADDR = "vault address" @@ -15,7 +15,7 @@ export VAULT_TOKEN= "vault token" ## Example -The example configurations to deploy this add-on can be found [here](./scenario/100-simple-dynamic-vault-secrets/configuration.tfvars) +An example of the configurations to deploy this add-on feature can be found [here](./scenario/100-simple-dynamic-vault-secrets/configuration.tfvars) Ensure the below is set prior to apply or destroy. diff --git a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars index e75c43635..637f029e7 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars +++ b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars @@ -11,10 +11,11 @@ landingzone = { } } dynamic_vault_secrets = { - vmadmin-username = { - path = "secret/test" + secret1 = { + path = "secret/test" + disable_read = true # optional secrets = { - username = { + username = { # key will be used for secret name in vault value = "vmadmin" } password = { @@ -39,8 +40,9 @@ dynamic_vault_secrets = { } } } - vmadmin-password = { - path = "secret/password" + secret2 = { + path = "secret/password" + disable_read = true # optional secrets = { username = { value = "vmadmin" From 7ba8bd64b5f228397bbdb4a6eb91ee831484e2e7 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 11:03:29 +1000 Subject: [PATCH 44/65] removed unwanted providers --- .../add-ons/dynamic_vault_secrets/main.tf | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/main.tf b/caf_solution/add-ons/dynamic_vault_secrets/main.tf index e38daa2e3..c025e40ac 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/main.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/main.tf @@ -4,34 +4,6 @@ terraform { source = "hashicorp/azurerm" version = "~> 2.43" } - azuread = { - source = "hashicorp/azuread" - version = "~> 1.4.0" - } - random = { - source = "hashicorp/random" - version = "~> 2.2.1" - } - null = { - source = "hashicorp/null" - version = "~> 2.1.0" - } - external = { - source = "hashicorp/external" - version = "~> 1.2.0" - } - azuredevops = { - source = "microsoft/azuredevops" - version = "~> 0.1.0" - } - tls = { - source = "hashicorp/tls" - version = "~> 2.2.0" - } - azurecaf = { - source = "aztfmod/azurecaf" - version = "~> 1.2.0" - } vault = { source = "hashicorp/vault" version = "2.17.0" From d05a82973dee99314cec15b9271f0a8a03aceeb0 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 16:07:55 +1000 Subject: [PATCH 45/65] functionality to fetch secrets from key vault --- .../configuration.tfvars | 5 +++++ .../dynamic_vault_secrets/secret/vault_secret.tf | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars index 637f029e7..f6129d394 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars +++ b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars @@ -38,6 +38,11 @@ dynamic_vault_secrets = { resource_key = "level1" attribute_key = "name" } + client_secret = { + secretname = "sp-client-secret" + lz_key = "launchpad" + keyvault_key = "level1" + } } } secret2 = { diff --git a/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf b/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf index a20887e4c..f6d4c956c 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf @@ -3,13 +3,22 @@ locals { for key, value in var.secrets : key => coalesce( try(value.value, null), try(var.objects[value.lz_key][value.output_key][value.resource_key][value.attribute_key], null), - try(var.objects[value.lz_key][value.output_key][value.attribute_key], null) + try(var.objects[value.lz_key][value.output_key][value.attribute_key], null), + try(data.azurerm_key_vault_secret.client_secret[key].value, null) ) } } -#try(each.value.value, null) == null ? try(var.objects[each.value.output_key][each.value.resource_key][each.value.attribute_key], var.objects[each.value.output_key][each.value.attribute_key]) : each.value.value + +data "azurerm_key_vault_secret" "client_secret" { + for_each = { + for key, value in var.secrets : key => value + if try(value.secretname, null) != null && try(value.secretname, null) != "" + } + name = each.value.secretname + key_vault_id = var.objects[each.value.lz_key].keyvaults[each.value.keyvault_key].id +} resource "vault_generic_secret" "azuresecrets" { path = var.path disable_read = try(var.disable_read, false) data_json = jsonencode(local.transposed) -} \ No newline at end of file +} From a5cafe17376e71f6173ae23cd498ab72e9681399 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 16:22:07 +1000 Subject: [PATCH 46/65] added comment for information --- .../100-simple-dynamic-vault-secrets/configuration.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars index f6129d394..9a2a702ca 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars +++ b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars @@ -38,7 +38,7 @@ dynamic_vault_secrets = { resource_key = "level1" attribute_key = "name" } - client_secret = { + client_secret = { # scenario to push secrets from key vault to hashicorp vault applicable mostly for Service Principal paswords. secretname = "sp-client-secret" lz_key = "launchpad" keyvault_key = "level1" From 6293ca089f74cd57bf0400357884b2fd68100a73 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 16:22:43 +1000 Subject: [PATCH 47/65] typo fixed --- .../100-simple-dynamic-vault-secrets/configuration.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars index 9a2a702ca..a6f4823f4 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars +++ b/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars @@ -38,7 +38,7 @@ dynamic_vault_secrets = { resource_key = "level1" attribute_key = "name" } - client_secret = { # scenario to push secrets from key vault to hashicorp vault applicable mostly for Service Principal paswords. + client_secret = { # scenario to push secrets from key vault to hashicorp vault applicable mostly for Service Principal passwords. secretname = "sp-client-secret" lz_key = "launchpad" keyvault_key = "level1" From 705b2a590350ff83e8b983d3941d6f122b84000a Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Wed, 9 Jun 2021 16:30:02 +1000 Subject: [PATCH 48/65] typo fixed --- caf_solution/add-ons/dynamic_vault_secrets/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/main.tf b/caf_solution/add-ons/dynamic_vault_secrets/main.tf index c025e40ac..30138148b 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/main.tf +++ b/caf_solution/add-ons/dynamic_vault_secrets/main.tf @@ -6,7 +6,7 @@ terraform { } vault = { source = "hashicorp/vault" - version = "2.17.0" + version = "~> 2.17.0" } } required_version = ">= 0.13" From 21ae22fe9244e23647ae132cb42a40b4606f70ed Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Thu, 10 Jun 2021 01:18:33 +0000 Subject: [PATCH 49/65] chnage naming convention for add-on --- .../README.md | 4 ++-- .../backend.azurerm | 0 .../local.remote_tfstates.tf | 0 .../main.tf | 0 .../providers.tf | 0 .../100-simple-hashicorp-vault-secrets}/configuration.tfvars | 2 +- .../secret/variables.tf | 0 .../secret/vault_secret.tf | 0 .../variables.tf | 2 +- .../vault.tf | 4 ++-- 10 files changed, 6 insertions(+), 6 deletions(-) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/README.md (85%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/backend.azurerm (100%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/local.remote_tfstates.tf (100%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/main.tf (100%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/providers.tf (100%) rename caf_solution/add-ons/{dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets => hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets}/configuration.tfvars (98%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/secret/variables.tf (100%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/secret/vault_secret.tf (100%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/variables.tf (95%) rename caf_solution/add-ons/{dynamic_vault_secrets => hashicorp_vault_secrets}/vault.tf (77%) diff --git a/caf_solution/add-ons/dynamic_vault_secrets/README.md b/caf_solution/add-ons/hashicorp_vault_secrets/README.md similarity index 85% rename from caf_solution/add-ons/dynamic_vault_secrets/README.md rename to caf_solution/add-ons/hashicorp_vault_secrets/README.md index 9c24489e4..d0b2e8ba7 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/README.md +++ b/caf_solution/add-ons/hashicorp_vault_secrets/README.md @@ -30,8 +30,8 @@ export environment=[YOUR_ENVIRONMENT] ```bash rover \ - -lz /tf/caf/landingzones/caf_solution/add-ons/dynamic_vault_secrets \ - -var-folder /tf/caf/landingzones/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets \ + -lz /tf/caf/landingzones/caf_solution/add-ons/hashicorp_vault_secrets \ + -var-folder /tf/caf/landingzones/caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets \ -tfstate vault.tfstate \ -env ${environment}} \ -level level1 \ diff --git a/caf_solution/add-ons/dynamic_vault_secrets/backend.azurerm b/caf_solution/add-ons/hashicorp_vault_secrets/backend.azurerm similarity index 100% rename from caf_solution/add-ons/dynamic_vault_secrets/backend.azurerm rename to caf_solution/add-ons/hashicorp_vault_secrets/backend.azurerm diff --git a/caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf b/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf similarity index 100% rename from caf_solution/add-ons/dynamic_vault_secrets/local.remote_tfstates.tf rename to caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf diff --git a/caf_solution/add-ons/dynamic_vault_secrets/main.tf b/caf_solution/add-ons/hashicorp_vault_secrets/main.tf similarity index 100% rename from caf_solution/add-ons/dynamic_vault_secrets/main.tf rename to caf_solution/add-ons/hashicorp_vault_secrets/main.tf diff --git a/caf_solution/add-ons/dynamic_vault_secrets/providers.tf b/caf_solution/add-ons/hashicorp_vault_secrets/providers.tf similarity index 100% rename from caf_solution/add-ons/dynamic_vault_secrets/providers.tf rename to caf_solution/add-ons/hashicorp_vault_secrets/providers.tf diff --git a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars b/caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets/configuration.tfvars similarity index 98% rename from caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars rename to caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets/configuration.tfvars index a6f4823f4..3d0624543 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/scenario/100-simple-dynamic-vault-secrets/configuration.tfvars +++ b/caf_solution/add-ons/hashicorp_vault_secrets/scenario/100-simple-hashicorp-vault-secrets/configuration.tfvars @@ -10,7 +10,7 @@ landingzone = { } } } -dynamic_vault_secrets = { +hashicorp_vault_secrets = { secret1 = { path = "secret/test" disable_read = true # optional diff --git a/caf_solution/add-ons/dynamic_vault_secrets/secret/variables.tf b/caf_solution/add-ons/hashicorp_vault_secrets/secret/variables.tf similarity index 100% rename from caf_solution/add-ons/dynamic_vault_secrets/secret/variables.tf rename to caf_solution/add-ons/hashicorp_vault_secrets/secret/variables.tf diff --git a/caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf b/caf_solution/add-ons/hashicorp_vault_secrets/secret/vault_secret.tf similarity index 100% rename from caf_solution/add-ons/dynamic_vault_secrets/secret/vault_secret.tf rename to caf_solution/add-ons/hashicorp_vault_secrets/secret/vault_secret.tf diff --git a/caf_solution/add-ons/dynamic_vault_secrets/variables.tf b/caf_solution/add-ons/hashicorp_vault_secrets/variables.tf similarity index 95% rename from caf_solution/add-ons/dynamic_vault_secrets/variables.tf rename to caf_solution/add-ons/hashicorp_vault_secrets/variables.tf index 3d9784f56..e5f003557 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/variables.tf +++ b/caf_solution/add-ons/hashicorp_vault_secrets/variables.tf @@ -30,4 +30,4 @@ variable "logged_aad_app_objectId" { variable "tags" { default = null } -variable "dynamic_vault_secrets" {} \ No newline at end of file +variable "hashicorp_vault_secrets" {} \ No newline at end of file diff --git a/caf_solution/add-ons/dynamic_vault_secrets/vault.tf b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf similarity index 77% rename from caf_solution/add-ons/dynamic_vault_secrets/vault.tf rename to caf_solution/add-ons/hashicorp_vault_secrets/vault.tf index e46c77e8b..cdb4a004a 100644 --- a/caf_solution/add-ons/dynamic_vault_secrets/vault.tf +++ b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf @@ -1,7 +1,7 @@ -module "dynamic_vault_secret" { +module "hashicorp_vault_secrets" { source = "./secret" for_each = { - for key, value in var.dynamic_vault_secrets : key => value + for key, value in var.hashicorp_vault_secrets : key => value if try(value.secrets, null) != null && try(value.secrets, null) != "" && try(value.path, null) != null && try(value.path, null) != "" } secrets = each.value.secrets From 6f3c6578d3bb162a0972641e6c93e43ba25d4b49 Mon Sep 17 00:00:00 2001 From: Jor Seng Date: Fri, 11 Jun 2021 10:55:24 +0000 Subject: [PATCH 50/65] Add remote azurerm_firewall_policies --- caf_solution/landingzone.tf | 5 +++++ caf_solution/local.remote.tf | 3 +++ 2 files changed, 8 insertions(+) diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 332ed3550..d8ea73732 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,13 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" +<<<<<<< Updated upstream source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" # source = "../../aztfmod" +======= + # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master" + source = "/tf/caf/aztfmod" +>>>>>>> Stashed changes azuread = local.azuread cloud = local.cloud diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index d56c68076..eab0603f5 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -56,6 +56,9 @@ locals { azuread_users = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_users, {})) } + azurerm_firewall_policies = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azurerm_firewall_policies, {})) + } azurerm_firewalls = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azurerm_firewalls, {})) } From 1a4eb6208bbd9c1e07ba679570050a3a2a7b600d Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Mon, 14 Jun 2021 02:33:07 +0000 Subject: [PATCH 51/65] fix source in caf solutions --- caf_solution/landingzone.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index d8ea73732..332ed3550 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,13 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" -<<<<<<< Updated upstream source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" # source = "../../aztfmod" -======= - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=master" - source = "/tf/caf/aztfmod" ->>>>>>> Stashed changes azuread = local.azuread cloud = local.cloud From 6ddb1ab36f6ebb14335dda6eebcfe08fccec4cf8 Mon Sep 17 00:00:00 2001 From: lolorol Date: Tue, 15 Jun 2021 01:27:47 +0000 Subject: [PATCH 52/65] Add fw policy rule collection groups --- caf_launchpad/landingzone.tf | 4 ++-- caf_solution/landingzone.tf | 4 ++-- caf_solution/local.networking.tf | 3 ++- caf_solution/variables.networking.tf | 3 +++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index 5b90984a0..fe3115875 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -2,8 +2,8 @@ module "launchpad" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - # source = "../../aztfmod" + # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + source = "../../aztfmod" current_landingzone_key = var.landingzone.key custom_role_definitions = var.custom_role_definitions diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 332ed3550..411ddaa03 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - # source = "../../aztfmod" + # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + source = "../../aztfmod" azuread = local.azuread cloud = local.cloud diff --git a/caf_solution/local.networking.tf b/caf_solution/local.networking.tf index d4a17cc9b..32bbda803 100644 --- a/caf_solution/local.networking.tf +++ b/caf_solution/local.networking.tf @@ -10,6 +10,7 @@ locals { azurerm_firewall_nat_rule_collection_definition = var.azurerm_firewall_nat_rule_collection_definition azurerm_firewall_network_rule_collection_definition = var.azurerm_firewall_network_rule_collection_definition azurerm_firewall_policies = var.azurerm_firewall_policies + azurerm_firewall_policy_rule_collection_groups = var.azurerm_firewall_policy_rule_collection_groups azurerm_firewalls = var.azurerm_firewalls azurerm_routes = var.azurerm_routes ddos_services = var.ddos_services @@ -34,10 +35,10 @@ locals { virtual_hub_connections = var.virtual_hub_connections virtual_hub_er_gateway_connections = var.virtual_hub_er_gateway_connections virtual_hub_route_tables = var.virtual_hub_route_tables + virtual_hubs = var.virtual_hubs virtual_network_gateway_connections = var.virtual_network_gateway_connections virtual_network_gateways = var.virtual_network_gateways virtual_wans = var.virtual_wans - virtual_hubs = var.virtual_hubs vnet_peerings = var.vnet_peerings vnets = var.vnets } diff --git a/caf_solution/variables.networking.tf b/caf_solution/variables.networking.tf index b8d416089..1b7b5d3d0 100644 --- a/caf_solution/variables.networking.tf +++ b/caf_solution/variables.networking.tf @@ -26,6 +26,9 @@ variable "azurerm_firewall_network_rule_collection_definition" { variable "azurerm_firewall_policies" { default = {} } +variable "azurerm_firewall_policy_rule_collection_groups" { + default = {} +} variable "azurerm_routes" { default = {} } From 3d025ca0b87a614ee8a70d39c0966ee23f07f38c Mon Sep 17 00:00:00 2001 From: lolorol Date: Tue, 15 Jun 2021 01:28:24 +0000 Subject: [PATCH 53/65] Add cross-tenant vnet to vhub --- .../backend.azurerm | 4 + .../hub_connection.tf | 81 +++++++++++++++++++ .../cross_tenant_hub_connection/main.tf | 23 ++++++ .../cross_tenant_hub_connection/providers.tf | 54 +++++++++++++ .../scripts/wait.sh | 23 ++++++ .../cross_tenant_hub_connection/variables.tf | 58 +++++++++++++ 6 files changed, 243 insertions(+) create mode 100644 caf_solution/add-ons/cross_tenant_hub_connection/backend.azurerm create mode 100644 caf_solution/add-ons/cross_tenant_hub_connection/hub_connection.tf create mode 100644 caf_solution/add-ons/cross_tenant_hub_connection/main.tf create mode 100644 caf_solution/add-ons/cross_tenant_hub_connection/providers.tf create mode 100755 caf_solution/add-ons/cross_tenant_hub_connection/scripts/wait.sh create mode 100644 caf_solution/add-ons/cross_tenant_hub_connection/variables.tf diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/backend.azurerm b/caf_solution/add-ons/cross_tenant_hub_connection/backend.azurerm new file mode 100644 index 000000000..5d026b233 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/backend.azurerm @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} \ No newline at end of file diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/hub_connection.tf b/caf_solution/add-ons/cross_tenant_hub_connection/hub_connection.tf new file mode 100644 index 000000000..c35ac7b4d --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/hub_connection.tf @@ -0,0 +1,81 @@ + + +data "azurerm_virtual_hub" "vhub-glrtss-phb-sanhub" { + for_each = var.virtual_hub_connections + + name = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].name + resource_group_name = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].object.resource_group_name + + provider = azurerm.virtual_hub +} + +data "azurerm_virtual_network" "vnw-uglife-dev-az1-aks" { + for_each = var.virtual_hub_connections + + name = data.terraform_remote_state.remote[each.value.vnet.lz_key].outputs.objects[each.value.vnet.lz_key].vnets[each.value.vnet.vnet_key].name + resource_group_name = data.terraform_remote_state.remote[each.value.vnet.lz_key].outputs.objects[each.value.vnet.lz_key].vnets[each.value.vnet.vnet_key].resource_group_name + + provider = azurerm.vnet +} + +resource "null_resource" "wait_for_virtual_hub_state" { + for_each = var.virtual_hub_connections + + triggers = { + routing = try(jsonencode(each.value.routing), null) + } + + provisioner "local-exec" { + command = format("%s/scripts/wait.sh", path.module) + + environment = { + VIRTUAL_HUB_ID = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].id + } + } +} + +resource "azurerm_virtual_hub_connection" "conn" { + for_each = var.virtual_hub_connections + depends_on = [null_resource.wait_for_virtual_hub_state] + + name = each.value.name + virtual_hub_id = data.terraform_remote_state.remote[each.value.virtual_hub.lz_key].outputs.objects[each.value.virtual_hub.lz_key].virtual_hubs[each.value.virtual_hub.key].id + remote_virtual_network_id = data.terraform_remote_state.remote[each.value.vnet.lz_key].outputs.objects[each.value.vnet.lz_key].vnets[each.value.vnet.vnet_key].id + internet_security_enabled = try(each.value.internet_security_enabled, false) + + dynamic "routing" { + for_each = try(each.value.routing, null) == null ? [] : [1] + + content { + + associated_route_table_id = data.terraform_remote_state.remote[each.value.routing.virtual_hub_route_table.lz_key].outputs.objects[each.value.routing.virtual_hub_route_table.lz_key].virtual_hub_route_table[each.value.routing.virtual_hub_route_table.key].id + + dynamic "propagated_route_table" { + for_each = try(each.value.routing.propagated_route_table, null) == null ? [] : [1] + + content { + route_table_ids = flatten( + [ + [ + try(each.value.routing.propagated_route_table.ids, []), + ], + [ + for key, value in try(each.value.routing.propagated_route_table.route_tables, []) : + [ + data.terraform_remote_state.remote[value.lz_key].outputs.objects[value.lz_key].virtual_hub_route_table[value.key].id + ] + ] + ] + ) + labels = try(each.value.routing.propagated_route_table.labels, null) + } + } + } + } + + provider = azurerm.virtual_hub +} + +output azurerm_virtual_hub_connection { + value = azurerm_virtual_hub_connection.conn +} diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/main.tf b/caf_solution/add-ons/cross_tenant_hub_connection/main.tf new file mode 100644 index 000000000..0ccb8f1f4 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/main.tf @@ -0,0 +1,23 @@ + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 2.55.0" + } + null = { + source = "hashicorp/null" + version = "~> 2.1.0" + } + } + required_version = ">= 0.13" +} + + +# provider "azurerm" { +# features {} +# skip_provider_registration = true +# } + +# data "azurerm_client_config" "current" {} + diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/providers.tf b/caf_solution/add-ons/cross_tenant_hub_connection/providers.tf new file mode 100644 index 000000000..b11dee7d9 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/providers.tf @@ -0,0 +1,54 @@ +locals { + landingzone = { + current = { + storage_account_name = var.tfstate_storage_account_name + container_name = var.tfstate_container_name + resource_group_name = var.tfstate_resource_group_name + } + lower = { + storage_account_name = var.lower_storage_account_name + container_name = var.lower_container_name + resource_group_name = var.lower_resource_group_name + } + } +} + +data "terraform_remote_state" "remote" { + for_each = try(var.landingzone.tfstates, {}) + + backend = var.landingzone.backend_type + config = { + container_name = try(each.value.workspace, local.landingzone[try(each.value.level, "current")].container_name) + key = each.value.tfstate + resource_group_name = try(each.value.resource_group_name, local.landingzone[try(each.value.level, "current")].resource_group_name) + sas_token = try(each.value.sas_token, null) != null ? var.sas_token : null + storage_account_name = try(each.value.storage_account_name, local.landingzone[try(each.value.level, "current")].storage_account_name) + subscription_id = try(each.value.subscription_id, data.azurerm_client_config.current.subscription_id) + tenant_id = try(each.value.tenant_id, data.azurerm_client_config.current.tenant_id) + use_azuread_auth = try(each.value.use_azuread_auth, true) + } +} + +data "azurerm_client_config" "current" { + provider = azurerm.vnet +} + +provider "azurerm" { + alias = "virtual_hub" + features {} + skip_provider_registration = true + subscription_id = var.virtual_hub_subscription_id + tenant_id = var.virtual_hub_tenant_id + + # Source tenants for virtual networks. + # Client ID must have permissions on those virtual_networks + auxiliary_tenant_ids = var.landingzone.tfstates[var.virtual_hub_lz_key].auxiliary_tenant_ids +} +provider "azurerm" { + features {} + alias = "vnet" + skip_provider_registration = true + subscription_id = var.virtual_network_subscription_id + tenant_id = var.virtual_network_tenant_id +} + diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/scripts/wait.sh b/caf_solution/add-ons/cross_tenant_hub_connection/scripts/wait.sh new file mode 100755 index 000000000..ae281ff58 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/scripts/wait.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +function canonicalize() { + echo $@ | sed 's|//subscriptions|/subscriptions|' +} + +API_VERSION=2021-02-01 + +RAW_URL=${resourceManager}/${VIRTUAL_HUB_ID}?api-version=${API_VERSION} + +URL=$(canonicalize ${RAW_URL}) + +PROVISIONING_STATE=$(az rest --method GET --uri ${URL} --query properties.provisioningState -o tsv) + +while ${PROVISIONING_STATE} != "Succeeded" +do + sleep 30 + PROVISIONING_STATE=$(az rest --method GET --uri ${URL} --query properties.provisioningState -o tsv) +done + +echo "Virtual Hub is ready" diff --git a/caf_solution/add-ons/cross_tenant_hub_connection/variables.tf b/caf_solution/add-ons/cross_tenant_hub_connection/variables.tf new file mode 100644 index 000000000..be428ef51 --- /dev/null +++ b/caf_solution/add-ons/cross_tenant_hub_connection/variables.tf @@ -0,0 +1,58 @@ +# Map of the remote data state +variable "lower_storage_account_name" { + description = "This value is propulated by the rover" +} +variable "lower_container_name" { + description = "This value is propulated by the rover" +} +variable "lower_resource_group_name" { + description = "This value is propulated by the rover" +} + + +variable "tfstate_subscription_id" { + description = "This value is propulated by the rover. subscription id hosting the remote tfstates" + default = null +} +variable "tfstate_storage_account_name" { + description = "This value is propulated by the rover" +} +variable "tfstate_container_name" { + description = "This value is propulated by the rover" +} +variable "tfstate_resource_group_name" { + description = "This value is propulated by the rover" +} +variable "sas_token" { + default = null +} + +variable "landingzone" {} +variable "virtual_hub_connections" { + default = {} +} + +variable "virtual_hub_subscription_id" { + type = string + description = "Subscription ID of the Virtual Hub." +} + +variable "virtual_hub_tenant_id" { + type = string + description = "Tenant ID of the Virtual Hub." +} + +variable "virtual_network_subscription_id" { + type = string + description = "Subscription ID of the Virtual Network." +} + +variable "virtual_network_tenant_id" { + type = string + description = "Tenant ID of the Virtual Network." +} + +variable "virtual_hub_lz_key" { + type = string + description = "Virtual Hub landingzone key in var.landingzone.tfstate." +} From 2c78df6dc55cf63884a41fb44ad97f24d3a2b2f5 Mon Sep 17 00:00:00 2001 From: lolorol Date: Tue, 15 Jun 2021 17:22:21 +0000 Subject: [PATCH 54/65] Fix module source --- caf_launchpad/landingzone.tf | 4 ++-- caf_solution/landingzone.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index fe3115875..5b90984a0 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -2,8 +2,8 @@ module "launchpad" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - source = "../../aztfmod" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" current_landingzone_key = var.landingzone.key custom_role_definitions = var.custom_role_definitions diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 411ddaa03..332ed3550 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - source = "../../aztfmod" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" azuread = local.azuread cloud = local.cloud From b264765e4660e5af199a14fd0d89ad496e386fc4 Mon Sep 17 00:00:00 2001 From: Carl Johnston Date: Tue, 8 Jun 2021 06:13:29 +0000 Subject: [PATCH 55/65] Added support for vpn_sites --- caf_solution/local.networking.tf | 1 + caf_solution/local.remote.tf | 3 +++ caf_solution/variables.networking.tf | 3 +++ 3 files changed, 7 insertions(+) diff --git a/caf_solution/local.networking.tf b/caf_solution/local.networking.tf index d4a17cc9b..4a124362e 100644 --- a/caf_solution/local.networking.tf +++ b/caf_solution/local.networking.tf @@ -40,6 +40,7 @@ locals { virtual_hubs = var.virtual_hubs vnet_peerings = var.vnet_peerings vnets = var.vnets + vpn_sites = var.vpn_sites } ) } diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index 17d2c7159..bdbb6ada1 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -160,6 +160,9 @@ locals { vnets = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) } + vpn_sites = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vpn_sites, {})) + } wvd_host_pools = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].wvd_host_pools, {})) } diff --git a/caf_solution/variables.networking.tf b/caf_solution/variables.networking.tf index b8d416089..5e8052f6f 100644 --- a/caf_solution/variables.networking.tf +++ b/caf_solution/variables.networking.tf @@ -116,4 +116,7 @@ variable "virtual_hub_route_tables" { } variable "virtual_hub_connections" { default = {} +} +variable "vpn_sites" { + default = {} } \ No newline at end of file From b0233bd1f8359e419a6a8313b941f9897e3cd905 Mon Sep 17 00:00:00 2001 From: Carl Johnston Date: Wed, 16 Jun 2021 04:36:45 +0000 Subject: [PATCH 56/65] Added support for vpn_gateway_connections --- caf_solution/local.networking.tf | 1 + caf_solution/local.remote.tf | 5 ++++- caf_solution/variables.networking.tf | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/caf_solution/local.networking.tf b/caf_solution/local.networking.tf index 4a124362e..cefe49902 100644 --- a/caf_solution/local.networking.tf +++ b/caf_solution/local.networking.tf @@ -41,6 +41,7 @@ locals { vnet_peerings = var.vnet_peerings vnets = var.vnets vpn_sites = var.vpn_sites + vpn_gateway_connections = var.vpn_gateway_connections } ) } diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index bdbb6ada1..0625fc1c7 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -163,6 +163,9 @@ locals { vpn_sites = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vpn_sites, {})) } + vpn_gateway_connections = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vpn_gateway_connections, {})) + } wvd_host_pools = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].wvd_host_pools, {})) } @@ -173,4 +176,4 @@ locals { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].wvd_workspaces, {})) } } -} \ No newline at end of file +} diff --git a/caf_solution/variables.networking.tf b/caf_solution/variables.networking.tf index 5e8052f6f..ff830c628 100644 --- a/caf_solution/variables.networking.tf +++ b/caf_solution/variables.networking.tf @@ -119,4 +119,7 @@ variable "virtual_hub_connections" { } variable "vpn_sites" { default = {} -} \ No newline at end of file +} +variable "vpn_gateway_connections" { + default = {} +} From d6c54ae84e98f5291dcee844deea53dd2a716ac3 Mon Sep 17 00:00:00 2001 From: lolorol Date: Fri, 18 Jun 2021 01:31:03 +0000 Subject: [PATCH 57/65] Add create namespace to helm chart --- caf_solution/add-ons/helm-charts/charts.tf | 15 ++++++++------- caf_solution/landingzone.tf | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/caf_solution/add-ons/helm-charts/charts.tf b/caf_solution/add-ons/helm-charts/charts.tf index 74b16be73..be33a7906 100644 --- a/caf_solution/add-ons/helm-charts/charts.tf +++ b/caf_solution/add-ons/helm-charts/charts.tf @@ -11,11 +11,12 @@ resource "helm_release" "chart" { depends_on = [kubernetes_namespace.gitlab_runners] for_each = var.helm_charts - chart = each.value.chart - name = each.value.name - namespace = each.value.namespace - repository = try(each.value.repository, null) - timeout = try(each.value.timeout, 4000) - values = [file(each.value.value_file)] - wait = try(each.value.wait, true) + chart = each.value.chart + create_namespace = try(each.value.create_namespace, false) + name = each.value.name + namespace = each.value.namespace + repository = try(each.value.repository, null) + timeout = try(each.value.timeout, 4000) + values = [file(each.value.value_file)] + wait = try(each.value.wait, true) } \ No newline at end of file diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 332ed3550..411ddaa03 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - # source = "../../aztfmod" + # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + source = "../../aztfmod" azuread = local.azuread cloud = local.cloud From 9640b4d33f0a98ce32458e985ed598ed28d068fb Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Thu, 17 Jun 2021 19:08:58 +0000 Subject: [PATCH 58/65] fix remote tfstate issue for eslz --- .../caf_eslz/locals.remote_tfstates.tf | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf b/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf index 54e93e04d..7b7b28c28 100644 --- a/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf +++ b/caf_solution/add-ons/caf_eslz/locals.remote_tfstates.tf @@ -17,16 +17,23 @@ data "terraform_remote_state" "remote" { for_each = try(var.landingzone.tfstates, {}) backend = var.landingzone.backend_type - config = { - storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name - container_name = local.landingzone[try(each.value.level, "current")].container_name - resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name - subscription_id = var.tfstate_subscription_id - key = each.value.tfstate - } + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] } locals { + remote_state = { + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = try(value.workspace, local.landingzone[try(value.level, "current")].container_name) + key = value.tfstate + resource_group_name = try(value.resource_group_name, local.landingzone[try(value.level, "current")].resource_group_name) + storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) + subscription_id = try(value.subscription_id, var.tfstate_subscription_id) + tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + } + } + } + landingzone_tag = { "landingzone" = var.landingzone.key } From 3962b7ec560d188c136af11c16026160cdd35500 Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Thu, 17 Jun 2021 20:45:02 +0000 Subject: [PATCH 59/65] fix source caf solutions --- caf_solution/landingzone.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 411ddaa03..332ed3550 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,8 +2,8 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - source = "../../aztfmod" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" azuread = local.azuread cloud = local.cloud From 3c747ad894401a00225a2d1720bea89bd93cff33 Mon Sep 17 00:00:00 2001 From: Jor Seng Date: Mon, 21 Jun 2021 00:56:51 +0000 Subject: [PATCH 60/65] Add cross subscription access support for kv secret --- caf_launchpad/landingzone.tf | 2 +- .../aad-pod-identity/aad-msi-binding.yaml | 5 ++++- .../aad-pod-identity/aad_pod_identity.tf | 2 ++ .../aks-pod-identity-assignment.tf | 4 ++-- caf_solution/add-ons/azure_devops_v1/azdo.tf | 18 ++++++++++++------ .../azure_devops_v1/azdo_service_endpoint.tf | 16 +++++++++++----- caf_solution/landingzone.tf | 2 +- caf_solution/local.remote.tf | 3 --- 8 files changed, 33 insertions(+), 19 deletions(-) diff --git a/caf_launchpad/landingzone.tf b/caf_launchpad/landingzone.tf index fe3115875..59bddf9a6 100644 --- a/caf_launchpad/landingzone.tf +++ b/caf_launchpad/landingzone.tf @@ -3,7 +3,7 @@ module "launchpad" { # version = "~>5.3.2" # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - source = "../../aztfmod" + source = "/tf/caf/aztfmod" current_landingzone_key = var.landingzone.key custom_role_definitions = var.custom_role_definitions diff --git a/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml b/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml index c430b2385..c8e5a083f 100644 --- a/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml +++ b/caf_solution/add-ons/aad-pod-identity/aad-msi-binding.yaml @@ -1,3 +1,5 @@ +# https://github.com/Azure/aad-pod-identity/blob/b3ee1d07209f26c47a96abf3ba20749932763de6/website/content/en/docs/Concepts/azureidentity.md + apiVersion: aadpodidentity.k8s.io/v1 kind: AzureIdentity metadata: @@ -13,4 +15,5 @@ metadata: name: podmi-gitlab-runner-binding spec: azureIdentity: podmi-caf-rover-platform-level0 - selector: podmi-caf-rover-platform-level0 \ No newline at end of file + selector: podmi-caf-rover-platform-level0 + diff --git a/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf b/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf index 05f60b4d1..f95d17c5f 100644 --- a/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf +++ b/caf_solution/add-ons/aad-pod-identity/aad_pod_identity.tf @@ -1,3 +1,5 @@ +# https://github.com/Azure/aad-pod-identity/blob/b3ee1d07209f26c47a96abf3ba20749932763de6/website/content/en/docs/Concepts/azureidentity.md + resource "kubernetes_namespace" "ns" { metadata { name = var.aad_pod_identity.namespace diff --git a/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf b/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf index 6635d9494..444eed636 100644 --- a/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf +++ b/caf_solution/add-ons/aks-secure-baseline/aks-pod-identity-assignment.tf @@ -58,11 +58,11 @@ locals { } resource "azurerm_key_vault_access_policy" "keyvault_policy" { - provider = azurerm.launchpad + # provider = azurerm.launchpad for_each = var.keyvaults key_vault_id = local.remote.keyvaults[each.value.lz_key][each.value.key].id tenant_id = data.azurerm_client_config.current.tenant_id object_id = local.remote.aks_clusters[var.aks_clusters[var.aks_cluster_key].lz_key][var.aks_clusters[var.aks_cluster_key].key].kubelet_identity[0].object_id secret_permissions = each.value.secret_permissions -} \ No newline at end of file +} diff --git a/caf_solution/add-ons/azure_devops_v1/azdo.tf b/caf_solution/add-ons/azure_devops_v1/azdo.tf index 77e39c70b..90094bccf 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo.tf @@ -1,11 +1,17 @@ # The PAT token must be provisioned in a different deployment provider "azuredevops" { org_service_url = var.azure_devops.url - personal_access_token = data.azurerm_key_vault_secret.pat.value + personal_access_token = data.external.pat.result.value } -data "azurerm_key_vault_secret" "pat" { - name = var.azure_devops.pats["admin"].secret_name - key_vault_id = local.remote.keyvaults[var.azure_devops.pats["admin"].lz_key][var.azure_devops.pats["admin"].keyvault_key].id - -} +# To support cross subscrpition reference +data "external" "pat" { + program = [ + "bash", "-c", + format( + "az keyvault secret show --id '%s'secrets/'%s' --query '{value: value}' -o json", + local.remote.keyvaults[var.azure_devops.pats["admin"].lz_key][var.azure_devops.pats["admin"].keyvault_key].vault_uri, + var.azure_devops.pats["admin"].secret_name + ) + ] +} \ No newline at end of file diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf index ecca19f33..1991a5d6d 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_service_endpoint.tf @@ -1,9 +1,15 @@ -data "azurerm_key_vault_secret" "client_secret" { +# To support cross subscription +data "external" "client_secret" { for_each = var.service_endpoints - - name = each.value.keyvault.secret_name - key_vault_id = local.remote.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.key].id + program = [ + "bash", "-c", + format( + "az keyvault secret show --id '%s'secrets/'%s' --query '{value: value}' -o json", + local.remote.keyvaults[each.value.keyvault.lz_key][each.value.keyvault.key].vault_uri, + each.value.keyvault.secret_name + ) + ] } resource "azuredevops_serviceendpoint_azurerm" "azure" { @@ -13,7 +19,7 @@ resource "azuredevops_serviceendpoint_azurerm" "azure" { service_endpoint_name = each.value.endpoint_name credentials { serviceprincipalid = local.remote.azuread_applications[each.value.azuread_application.lz_key][each.value.azuread_application.key].application_id - serviceprincipalkey = data.azurerm_key_vault_secret.client_secret[each.key].value + serviceprincipalkey = data.external.client_secret[each.key].result.value } azurerm_spn_tenantid = local.remote.azuread_applications[each.value.azuread_application.lz_key][each.value.azuread_application.key].tenant_id azurerm_subscription_id = each.value.subscription.id diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 411ddaa03..f49a157bc 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -3,7 +3,7 @@ module "solution" { # version = "~>5.3.2" # source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - source = "../../aztfmod" + source = "/tf/caf/aztfmod" azuread = local.azuread cloud = local.cloud diff --git a/caf_solution/local.remote.tf b/caf_solution/local.remote.tf index eab0603f5..3131a9071 100644 --- a/caf_solution/local.remote.tf +++ b/caf_solution/local.remote.tf @@ -50,9 +50,6 @@ locals { availability_sets = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].availability_sets, {})) } - azuread_applications = { - for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_applications, {})) - } azuread_users = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].azuread_users, {})) } From 2cda697d8ed433ea09e58791505e133c6aa739d3 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Tue, 22 Jun 2021 01:06:59 +0000 Subject: [PATCH 61/65] added output to debug --- caf_solution/add-ons/hashicorp_vault_secrets/vault.tf | 5 ++++- caf_solution/landingzone.tf | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf index cdb4a004a..ba0bce242 100644 --- a/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf +++ b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf @@ -8,4 +8,7 @@ module "hashicorp_vault_secrets" { path = each.value.path disable_read = try(each.value.disable_read, false) objects = local.remote.objects -} \ No newline at end of file +} +output name { + value = local.remote.objects +} diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 332ed3550..086cf6627 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,10 +2,10 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" - # source = "../../aztfmod" + #source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=integration518" + source = "../../aztfmod" - azuread = local.azuread + # azuread = local.azuread cloud = local.cloud compute = local.compute current_landingzone_key = var.landingzone.key From e1a701da65745b6e309f90abf494e1d7df01ffe3 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Tue, 22 Jun 2021 03:14:20 +0000 Subject: [PATCH 62/65] chnages for remote states --- .../local.remote_tfstates.tf | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf b/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf index 473213da2..ec8506c71 100644 --- a/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf +++ b/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf @@ -17,16 +17,22 @@ data "terraform_remote_state" "remote" { for_each = try(var.landingzone.tfstates, {}) backend = var.landingzone.backend_type - config = { - storage_account_name = local.landingzone[try(each.value.level, "current")].storage_account_name - container_name = local.landingzone[try(each.value.level, "current")].container_name - resource_group_name = local.landingzone[try(each.value.level, "current")].resource_group_name - subscription_id = var.tfstate_subscription_id - key = each.value.tfstate - } + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] } locals { + remote_state = { + azurerm = { + for key, value in try(var.landingzone.tfstates, {}) : key => { + container_name = try(value.workspace, local.landingzone[try(value.level, "current")].container_name) + key = value.tfstate + resource_group_name = try(value.resource_group_name, local.landingzone[try(value.level, "current")].resource_group_name) + storage_account_name = try(value.storage_account_name, local.landingzone[try(value.level, "current")].storage_account_name) + subscription_id = try(value.subscription_id, var.tfstate_subscription_id) + tenant_id = try(value.tenant_id, data.azurerm_client_config.current.tenant_id) + } + } + } landingzone_tag = { "landingzone" = var.landingzone.key } From 528780c931e17ce064aa8dea012aed37eeda0f60 Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Tue, 22 Jun 2021 03:23:33 +0000 Subject: [PATCH 63/65] remove output from module --- .../add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf | 2 +- caf_solution/add-ons/hashicorp_vault_secrets/vault.tf | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf b/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf index ec8506c71..bb8847493 100644 --- a/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf +++ b/caf_solution/add-ons/hashicorp_vault_secrets/local.remote_tfstates.tf @@ -17,7 +17,7 @@ data "terraform_remote_state" "remote" { for_each = try(var.landingzone.tfstates, {}) backend = var.landingzone.backend_type - config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] + config = local.remote_state[try(each.value.backend_type, var.landingzone.backend_type, "azurerm")][each.key] } locals { diff --git a/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf index ba0bce242..66b03b258 100644 --- a/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf +++ b/caf_solution/add-ons/hashicorp_vault_secrets/vault.tf @@ -9,6 +9,4 @@ module "hashicorp_vault_secrets" { disable_read = try(each.value.disable_read, false) objects = local.remote.objects } -output name { - value = local.remote.objects -} + From 2cda869176fa98f2a18e7ce3ed2ea8d0892eefab Mon Sep 17 00:00:00 2001 From: Papun Senapati Date: Tue, 22 Jun 2021 03:37:13 +0000 Subject: [PATCH 64/65] revert back to mtms --- caf_solution/landingzone.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/caf_solution/landingzone.tf b/caf_solution/landingzone.tf index 086cf6627..332ed3550 100644 --- a/caf_solution/landingzone.tf +++ b/caf_solution/landingzone.tf @@ -2,10 +2,10 @@ module "solution" { # source = "aztfmod/caf/azurerm" # version = "~>5.3.2" - #source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=integration518" - source = "../../aztfmod" + source = "git::https://github.com/aztfmod/terraform-azurerm-caf.git?ref=mtms" + # source = "../../aztfmod" - # azuread = local.azuread + azuread = local.azuread cloud = local.cloud compute = local.compute current_landingzone_key = var.landingzone.key From aa39c468c14d314a0ff9ebed4390cfeca17a844e Mon Sep 17 00:00:00 2001 From: Abdullah Khairi Date: Wed, 23 Jun 2021 07:24:31 +0000 Subject: [PATCH 65/65] add remote objects feature for var group --- .../add-ons/azure_devops_v1/azdo_variable_groups.tf | 13 +++++++++++++ .../azure_devops_v1/locals.remote_tfstates.tf | 3 +++ 2 files changed, 16 insertions(+) diff --git a/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf index 73edb1f5a..2a8099203 100644 --- a/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf +++ b/caf_solution/add-ons/azure_devops_v1/azdo_variable_groups.tf @@ -27,6 +27,7 @@ resource "azuredevops_variable_group" "variable_group" { name = key == "name" ? variable : key value = key == "name" ? null : variable } + if try(each.value.remote_objects, false) == false } content { @@ -36,4 +37,16 @@ resource "azuredevops_variable_group" "variable_group" { } } + dynamic "variable" { + for_each = { + for key, value in each.value.variables : key => value + if try(each.value.remote_objects, false) == true + } + + content { + name = variable.value.name + value = local.remote[variable.value.output_key][variable.value.lz_key][variable.value.resource_key][variable.value.attribute_key] + } + } + } diff --git a/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf index 92912e8f8..14904c3bc 100644 --- a/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf +++ b/caf_solution/add-ons/azure_devops_v1/locals.remote_tfstates.tf @@ -63,5 +63,8 @@ locals { vnets = { for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].vnets, {})) } + subscriptions = { + for key, value in try(var.landingzone.tfstates, {}) : key => merge(try(data.terraform_remote_state.remote[key].outputs.objects[key].subscriptions, {})) + } } }