Skip to content

Commit

Permalink
Support storage network access and worm removal in remove test resour…
Browse files Browse the repository at this point in the history 
…ces script (#8558)

* Support storage network access and worm removal in remove test resources script

* Move storage network access script to common resource helpers file

* Improve storage container deletion resilience

* Plumb through pool variable to live test cleanup template

* Add sleep for network rule application
  • Loading branch information
@benbp
benbp authored Jul 8, 2024
1 parent 1ec6beb commit 86c522c
Show file tree
Hide file tree
Showing 9 changed files with 515 additions and 396 deletions.
294 changes: 3 additions & 291 deletions eng/common/TestResources/New-TestResources.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,8 @@ param (
$NewTestResourcesRemainingArguments
)

. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1)
. $PSScriptRoot/TestResources-Helpers.ps1
. $PSScriptRoot/SubConfig-Helpers.ps1

if (!$ServicePrincipalAuth) {
Expand All @@ -131,272 +133,6 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
$ErrorActionPreference = 'Stop'
}

function Log($Message)
{
Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
}

# vso commands are specially formatted log lines that are parsed by Azure Pipelines
# to perform additional actions, most commonly marking values as secrets.
# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands
function LogVsoCommand([string]$message)
{
if (!$CI -or $SuppressVsoCommands) {
return
}
Write-Host $message
}

function Retry([scriptblock] $Action, [int] $Attempts = 5)
{
$attempt = 0
$sleep = 5

while ($attempt -lt $Attempts) {
try {
$attempt++
return $Action.Invoke()
} catch {
if ($attempt -lt $Attempts) {
$sleep *= 2

Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..."
Start-Sleep -Seconds $sleep
} else {
throw
}
}
}
}

# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type.
# This is necessary to work around breaking changes introduced in Az version 7.0.0:
# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/
function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName)
{
if ((Get-Module Az.Resources).Version -eq "5.3.0") {
# https://github.com/Azure/azure-powershell/issues/17040
# New-AzAdServicePrincipal calls will fail with:
# "You cannot call a method on a null-valued expression."
Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1"
Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1"
exit 1
}

try {
$servicePrincipal = Retry {
New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName
}
} catch {
# The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying
# to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following:
# "Cannot bind argument to parameter 'ObjectId' because it is an empty string."
# Provide a more helpful diagnostic prompt to the user if appropriate:
$totalApps = (Get-AzADApplication -OwnedApplication).Length
$msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + `
"`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + `
" or by running the following command to remove apps created by this script:" + `
"`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + `
"`nNOTE: You may need to wait for the quota number to be updated after removing unused applications."
Write-Warning $msg
throw
}

$spPassword = ""
$appId = ""
if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) {
Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API"
# Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0
$spPassword = $servicePrincipal.Secret
$appId = $servicePrincipal.ApplicationId
} else {
if ((Get-Module Az.Resources).Version -eq "5.1.0") {
Write-Verbose "Creating password and credential for service principal via MS Graph API"
Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'"
# Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately.
# Submitting a password credential object without specifying a password will result in one being generated on the server side.
$password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential"
$password.DisplayName = "Password for $displayName"
$credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' }
$spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force
$appId = $servicePrincipal.AppId
} else {
Write-Verbose "Creating service principal credential via MS Graph API"
# In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the
# parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter.
$credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' }
$spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force
$appId = $servicePrincipal.AppId
}
}

return @{
AppId = $appId
ApplicationId = $appId
# This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion
Id = $servicePrincipal.Id
DisplayName = $servicePrincipal.DisplayName
Secret = $spPassword
}
}

function LoadCloudConfig([string] $env)
{
$configPath = "$PSScriptRoot/clouds/$env.json"
if (!(Test-Path $configPath)) {
Write-Warning "Could not find cloud configuration for environment '$env'"
return @{}
}

$config = Get-Content $configPath | ConvertFrom-Json -AsHashtable
return $config
}

function MergeHashes([hashtable] $source, [psvariable] $dest)
{
foreach ($key in $source.Keys) {
if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) {
Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " +
"to new value '$($source[$key])'")
}
$dest.Value[$key] = $source[$key]
}
}

function BuildBicepFile([System.IO.FileSystemInfo] $file)
{
if (!(Get-Command bicep -ErrorAction Ignore)) {
Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install"
throw
}

$tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath()
$templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json"

# Az can deploy bicep files natively, but by compiling here it becomes easier to parse the
# outputted json for mismatched parameter declarations.
bicep build $file.FullName --outfile $templateFilePath
if ($LASTEXITCODE) {
Write-Error "Failure building bicep file '$($file.FullName)'"
throw
}

return $templateFilePath
}

function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) {
$serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
# Add default values
$deploymentOutputs = [Ordered]@{
"${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id;
"${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName;
"${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location;
"${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name;
"${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority;
"${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl;
"${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl;
"AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant();
}

if ($ServicePrincipalAuth) {
$deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId;
$deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret;
$deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id;
}

MergeHashes $environmentVariables $(Get-Variable deploymentOutputs)

foreach ($key in $deployment.Outputs.Keys) {
$variable = $deployment.Outputs[$key]

# Work around bug that makes the first few characters of environment variables be lowercase.
$key = $key.ToUpperInvariant()

if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') {
$deploymentOutputs[$key] = $variable.Value
}
}

# Force capitalization of all keys to avoid Azure Pipelines confusion with
# variable auto-capitalization and OS env var capitalization differences
$capitalized = @{}
foreach ($item in $deploymentOutputs.GetEnumerator()) {
$capitalized[$item.Name.ToUpperInvariant()] = $item.Value
}

return $capitalized
}

function SetDeploymentOutputs(
[string]$serviceName,
[object]$azContext,
[object]$deployment,
[object]$templateFile,
[hashtable]$environmentVariables = @{}
) {
$deploymentEnvironmentVariables = $environmentVariables.Clone()
$deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables

if ($OutFile) {
if (!$IsWindows) {
Write-Host 'File option is supported only on Windows'
}

$outputFile = "$($templateFile.originalFilePath).env"

$environmentText = $deploymentOutputs | ConvertTo-Json;
$bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
$protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)

Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force

Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile"
} else {
if (!$CI) {
# Write an extra new line to isolate the environment variables for easy reading.
Log "Persist the following environment variables based on your detected shell ($shell):`n"
}

# Write overwrite warnings first, since local execution prints a runnable command to export variables
foreach ($key in $deploymentOutputs.Keys) {
if ([Environment]::GetEnvironmentVariable($key)) {
Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'"
}
}

# Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep
# file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default).
# This variable supports a second check on not marking previously allowed keys/values as secret.
$notSecretValues = @()
foreach ($key in $deploymentOutputs.Keys) {
$value = $deploymentOutputs[$key]
$deploymentEnvironmentVariables[$key] = $value

if ($CI) {
if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) {
# Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
# In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.
LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value"
Write-Host "Setting variable as secret '$key'"
} else {
Write-Host "Setting variable '$key': $value"
$notSecretValues += $value
}
LogVsoCommand "##vso[task.setvariable variable=$key;]$value"
} else {
Write-Host ($shellExportFormat -f $key, $value)
}
}

if ($key) {
# Isolate the environment variables for easy reading.
Write-Host "`n"
$key = $null
}
}

return $deploymentEnvironmentVariables, $deploymentOutputs
}

# Support actions to invoke on exit.
$exitActions = @({
Expand Down Expand Up @@ -843,31 +579,7 @@ try {
-templateFile $templateFile `
-environmentVariables $EnvironmentVariables

$storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" }
# Add client IP to storage account when running as local user. Pipeline's have their own vnet with access
if ($storageAccounts) {
foreach ($account in $storageAccounts) {
$rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name
if ($rules -and $rules.DefaultAction -eq "Allow") {
Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default"
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny }
if ($CI -and $env:PoolSubnet) {
Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)"
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet }
} elseif ($AllowIpRanges) {
Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges"
$ipRanges = $AllowIpRanges | ForEach-Object {
@{ Action = 'allow'; IPAddressOrRange = $_ }
}
Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null }
} elseif (!$CI) {
Write-Host "Enabling access to '$($account.Name)' from client IP"
$clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site
Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null }
}
}
}
}
SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI

$postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-post.ps1"
if (Test-Path $postDeploymentScript) {
Expand Down
19 changes: 19 additions & 0 deletions eng/common/TestResources/Remove-TestResources.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,19 @@ param (
[Parameter()]
[switch] $ServicePrincipalAuth,

# List of CIDR ranges to add to specific resource firewalls, e.g. @(10.100.0.0/16, 10.200.0.0/16)
[Parameter()]
[ValidateCount(0,399)]
[Validatescript({
foreach ($range in $PSItem) {
if ($range -like '*/31' -or $range -like '*/32') {
throw "Firewall IP Ranges cannot contain a /31 or /32 CIDR"
}
}
return $true
})]
[array] $AllowIpRanges = @(),

[Parameter()]
[switch] $Force,

Expand All @@ -69,6 +82,9 @@ param (
$RemoveTestResourcesRemainingArguments
)

. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1)
. (Join-Path $PSScriptRoot TestResources-Helpers.ps1)

# By default stop for any error.
if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
$ErrorActionPreference = 'Stop'
Expand Down Expand Up @@ -241,6 +257,9 @@ $verifyDeleteScript = {
# Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty.
$purgeableResources = Get-PurgeableGroupResources $ResourceGroupName

SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI
Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName -CI:$CI

Log "Deleting resource group '$ResourceGroupName'"
if ($Force -and !$purgeableResources) {
Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob
Expand Down
Loading

0 comments on commit 86c522c

Please sign in to comment.