Code ingress with git

deployment/administration/SHM_Teardown.ps1

@@ -71,11 +71,6 @@ $adDnsRecordname = "@"
 Add-LogMessage -Level Info "[ ] Removing '$adDnsRecordname' TXT record from SHM $shmId DNS zone ($shmDomain)"
 Remove-AzDnsRecordSet -Name $adDnsRecordname -RecordType TXT -ZoneName $shmDomain -ResourceGroupName $dnsResourceGroup
 $success = $?
-# # RDS ACME record
-# $rdsAcmeDnsRecordname = "_acme-challenge"
-# Add-LogMessage -Level Info "[ ] Removing '$rdsAcmeDnsRecordname' TXT record from SRE $sreId DNS zone ($shmDomain)"
-# Remove-AzDnsRecordSet -Name $rdsAcmeDnsRecordname -RecordType TXT -ZoneName $shmDomain -ResourceGroupName $dnsResourceGroup
-# $success = $success -and $?
 # Print success/failure message
 if ($success) {
     Add-LogMessage -Level Success "Record removal succeeded"

deployment/administration/SRE_Upload_Git_Repo_to_GitlabReview.ps1

@@ -0,0 +1,137 @@
+param(
+    [Parameter(Mandatory = $true, HelpMessage = "Enter SRE ID (usually a number e.g enter '9' for DSG9)")]
+    [string]$sreId,
+    [Parameter( Mandatory = $true, HelpMessage = "Enter the git URL of the source repository")]
+    [string]$sourceGitURL,
+    ## interpret the basename of the final path segment in a (possibly encoded) URI as the name of the repository
+    [Parameter( Mandatory = $false, HelpMessage = "Enter the name of the repository as it should appear within SRE GITLAB (default is the basename of the final path segment of the git URL)")]
+    [string]$targetRepoName = [uri]::UnescapeDataString((Split-Path -Path ([uri]$sourceGitURL).Segments[-1] -LeafBase)),
+    [Parameter( Mandatory = $true, HelpMessage = "Enter the full commit hash of the commit in the source repository to snapshot")]
+    [string]$sourceCommitHash,
+    [Parameter( Mandatory = $true, HelpMessage = "Enter the desired branch name where the snapshot should be placed (in the repository inside SRE GITLAB)")]
+    [string]$targetBranchName
+)
+
+Import-Module Az
+Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
+Import-Module $PSScriptRoot/../common/Security.psm1 -Force
+Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
+Import-Module $PSScriptRoot/../common/Deployments.psm1 -Force
+Import-Module $PSScriptRoot/../common/GenerateSasToken.psm1 -Force
+
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-SreConfig $sreId
+$originalContext = Get-AzContext
+$_ = Set-AzContext -SubscriptionId $config.sre.subscriptionName
+
+# Create local zip file
+# ---------------------
+
+# The zipfile is called "repo.zip", with the following contents:
+#
+# repo/
+#   sourceGitURL
+#   targetRepoName
+#   sourceCommitHash
+#   targetBranchName
+#   snapshot/
+#     ... repository contents
+
+
+Add-LogMessage -Level Info "Creating zipfilepath."
+## $zipFileName = "${targetRepoName}_${sourceCommitHash}_${targetBranchName}.zip"
+$zipFileName = "repo.zip"
+
+$tempDir = New-Item -ItemType Directory -Path (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()))
+
+$repoPath = Join-Path $tempDir "repo"
+New-Item -ItemType Directory $repoPath
+
+## 
+$workingDir = Get-Location
+Set-Location $repoPath
+
+Add-LogMessage -Level Info "About to git clone "
+git clone $sourceGitURL snapshot
+
+Set-Location "snapshot"
+
+git checkout $sourceCommitHash
+# Remove the .git directory
+Remove-Item -Path ".git" -Recurse -Force
+
+## Record some metadata about the repository
+Set-Location $repoPath
+$sourceGitURL > sourceGitURL
+$targetRepoName > targetRepoName
+$sourceCommitHash > sourceCommitHash
+$targetBranchName > targetBranchName
+
+# Zip contents and meta
+Set-Location $tempDir
+
+$zipFilePath = Join-Path $tempDir $zipFileName
+Compress-Archive -CompressionLevel NoCompression -Path $repoPath -DestinationPath $zipFilePath
+if ($?) {
+    Add-LogMessage -Level Success "Zip file creation succeeded! $zipFilePath"
+} else {
+    Add-LogMessage -Level Fatal "Zip file creation failed!"
+}
+Set-Location $workingDir
+
+
+# Upload the zip file to the VM, via blob storage
+# -----------------------------------------------
+
+$gitlabReviewVmName = $config.sre.webapps.gitlabreview.vmName
+# Go via blob storage - first create storage account if not already there
+$storageResourceGroupName = $config.sre.storage.artifacts.rg
+$sreStorageAccountName = $config.sre.storage.artifacts.accountName
+$sreStorageAccount = Deploy-StorageAccount -Name $sreStorageAccountName -ResourceGroupName $storageResourceGroupName -Location $config.sre.location
+
+# Create a temporary storage container
+$containerName = $config.sre.storage.artifacts.containers.gitlabAirlockName + "-" + [Guid]::NewGuid().ToString()
+
+# Ensure an empty storage container of the given name exists
+$_ = Deploy-StorageContainer -Name $containerName -StorageAccount $sreStorageAccount
+
+# copy zipfile to blob storage
+# ----------------------------
+Add-LogMessage -Level Info "Upload zipfile to storage..."
+
+Set-AzStorageBlobContent -Container $containerName -Context $sreStorageAccount.Context -File $zipFilePath -Blob $zipFileName -Force
+
+# Download zipfile onto the remote machine
+# ----------------------------------------
+# Get a SAS token and construct URL
+$sasToken = New-ReadOnlyAccountSasToken -ResourceGroup $storageResourceGroupName -AccountName $sreStorageAccount.StorageAccountName -SubscriptionName $config.sre.subscriptionName
+$remoteUrl = "https://$($sreStorageAccount.StorageAccountName).blob.core.windows.net/${containerName}/${zipFileName}${sasToken}"
+Add-LogMessage -Level Info "Got SAS token and URL $remoteUrl"
+
+$sreAdminUsername = Resolve-KeyVaultSecret -VaultName $config.sre.keyVault.Name -SecretName $config.sre.keyVault.secretNames.adminUsername -DefaultValue "sre$($config.sre.id)admin".ToLower()
+
+# Create remote script (make a subdirectory of /tmp/zipfiles and run CURL to download blob to there)
+$script = @"
+#!/bin/bash
+mkdir -p /tmp/zipfiles/
+tmpdir=`$(mktemp -d /tmp/zipfiles/XXXXXXXXXXXXXXXXXXXX)
+curl -X GET -o `$tmpdir/${zipFileName} "${remoteUrl}"
+
+chown -R ${sreAdminUsername}:${sreAdminUsername} /tmp/zipfiles/
+"@
+
+$resourceGroupName = $config.sre.webapps.rg
+Add-LogMessage -Level Info "[ ] Running remote script to download zipfile onto $gitlabReviewVmName"
+$result = Invoke-RemoteScript -Shell "UnixShell" -Script $script -VMName $gitlabReviewVmName -ResourceGroupName $resourceGroupName
+
+# clean up - remove the zipfile from local machine.
+Add-LogMessage -Level Info "[ ] Removing original zipfile $zipFilePath"
+Remove-Item -Path $zipFilePath
+
+# Remove the temporary storage container
+Remove-AzStorageContainer -Name $containerName
+
+# Switch back to original subscription
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext

deployment/common/Configuration.psm1

@@ -300,6 +300,7 @@ function Add-SreConfig {
     $dataAdministratorsGroup = "SG $($config.sre.domain.netbiosName) Data Administrators"
     $systemAdministratorsGroup = "SG $($config.sre.domain.netbiosName) System Administrators"
     $researchUsersGroup = "SG $($config.sre.domain.netbiosName) Research Users"
+    $reviewUsersGroup = "SG $($config.sre.domain.netbiosName) Review Users"
     $config.sre.domain.securityGroups = [ordered]@{
         dataAdministrators = [ordered]@{
             name = $dataAdministratorsGroup
@@ -313,6 +314,10 @@ function Add-SreConfig {
             name = $researchUsersGroup
             description = $researchUsersGroup
         }
+        reviewUsers = [ordered]@{
+            name = $reviewUsersGroup
+            description = $reviewUsersGroup
+        }
     }
 
     # --- Network config ---
@@ -340,12 +345,20 @@ function Add-SreConfig {
                 prefix = "${sreBasePrefix}.$([int]$sreThirdOctet + 3)"
                 nsg = "databases"
             }
+            airlock = [ordered]@{
+                name = "AirlockSubnet"
+                prefix = "${sreBasePrefix}.$([int]$sreThirdOctet + 4)"
+                nsg = "airlock"
+            }
         }
         nsg = [ordered]@{
             data = [ordered]@{}
             databases = [ordered]@{
                 name = "NSG_SRE_$($config.sre.id)_DATABASES".ToUpper()
             }
+            airlock = [ordered]@{
+                name = "NSG_SRE_$($config.sre.id)_AIRLOCK".ToUpper()
+            }
         }
     }
     # Construct the CIDR for each subnet based on the prefix. Using '/24' gives 256 address for each subnet
@@ -360,6 +373,9 @@ function Add-SreConfig {
         artifacts = [ordered]@{
             rg = $storageRg
             accountName = "sre$($shm.id)artifacts${storageSuffix}".ToLower() | TrimToLength 24
+            containers = [ordered]@{
+                gitlabAirlockName = "gitlabairlock"
+            }
         }
         bootdiagnostics = [ordered]@{
             rg = $storageRg
@@ -383,8 +399,14 @@ function Add-SreConfig {
             gitlabLdapPassword = "$($config.sre.shortName)-gitlab-ldap-password"
             gitlabRootPassword = "$($config.sre.shortName)-gitlab-root-password"
             gitlabUserPassword = "$($config.sre.shortName)-gitlab-user-password"
+            gitlabUsername = "$($config.sre.shortName)-gitlab-username"
+            gitlabPassword = "$($config.sre.shortName)-gitlab-password"
+            gitlabAPIToken = "$($config.sre.shortName)-gitlab-api-token"
             hackmdLdapPassword = "$($config.sre.shortName)-hackmd-ldap-password"
             hackmdUserPassword = "$($config.sre.shortName)-hackmd-user-password"
+            gitlabReviewUsername = "$($config.sre.shortName)-gitlab-review-username"
+            gitlabReviewPassword = "$($config.sre.shortName)-gitlab-review-password"
+            gitlabReviewAPIToken = "$($config.sre.shortName)-gitlab-review-api-token"
             letsEncryptCertificate = "$($config.sre.shortName)-lets-encrypt-certificate"
             npsSecret = "$($config.sre.shortName)-nps-secret"
             postgresDbAdminUsername = "$($config.sre.shortName)-postgresdb-admin-username"
@@ -434,7 +456,7 @@ function Add-SreConfig {
         researchers = [ordered]@{
             test = [ordered]@{
                 name = "$($config.sre.domain.netbiosName) Test Researcher"
-                samAccountName = "testresrch$($sreConfigBase.sreId)".ToLower() | TrimToLength 20
+                samAccountName = "researcher$($sreConfigBase.sreId)".ToLower() | TrimToLength 20
             }
         }
     }
@@ -458,6 +480,11 @@ function Add-SreConfig {
             vmSize = "Standard_DS2_v2"
             nsg = "NSG_SRE_$($config.sre.id)_RDS_SESSION_HOSTS".ToUpper()
         }
+        sessionHost3 = [ordered]@{
+            vmName = "REV-SRE-$($config.sre.id)".ToUpper() | TrimToLength 15
+            vmSize = "Standard_DS2_v2"
+            nsg = "NSG_SRE_$($config.sre.id)_RDS_SESSION_HOSTS".ToUpper()
+        }
     }
 
     # Set which IPs can access the Safe Haven: if 'default' is given then apply sensible defaults
@@ -491,6 +518,9 @@ function Add-SreConfig {
     $config.sre.rds.sessionHost2.hostname = $config.sre.rds.sessionHost2.vmName
     $config.sre.rds.sessionHost2.fqdn = "$($config.sre.rds.sessionHost2.hostname).$($config.shm.domain.fqdn)"
     $config.sre.rds.sessionHost2.ip = "$($config.sre.network.subnets.rds.prefix).248"
+    $config.sre.rds.sessionHost3.hostname = $config.sre.rds.sessionHost3.vmName
+    $config.sre.rds.sessionHost3.fqdn = "$($config.sre.rds.sessionHost3.hostname).$($config.shm.domain.fqdn)"
+    $config.sre.rds.sessionHost3.ip = "$($config.sre.network.subnets.rds.prefix).247"
 
     # --- Secure servers ---
 
@@ -512,17 +542,24 @@ function Add-SreConfig {
         rg = "RG_SRE_WEBAPPS"
         nsg = "NSG_SRE_$($config.sre.id)_WEBAPPS".ToUpper()
         gitlab = [ordered]@{
-            vmName = "GITLAB-SRE-$($config.sre.id)".ToUpper()
+                vmName = "GITLAB-$($config.sre.id)".ToUpper()
+                vmSize = "Standard_D2s_v3"
+        }
+        gitlabreview = [ordered]@{
+            vmName = "GITLAB-REVIEW-$($config.sre.id)".ToUpper()
             vmSize = "Standard_D2s_v3"
         }
         hackmd = [ordered]@{
-            vmName = "HACKMD-SRE-$($config.sre.id)".ToUpper()
+            vmName = "HACKMD-$($config.sre.id)".ToUpper()
             vmSize = "Standard_D2s_v3"
         }
     }
     $config.sre.webapps.gitlab.hostname = $config.sre.webapps.gitlab.vmName
     $config.sre.webapps.gitlab.fqdn = "$($config.sre.webapps.gitlab.hostname).$($config.shm.domain.fqdn)"
     $config.sre.webapps.gitlab.ip = "$($config.sre.network.subnets.data.prefix).151"
+    $config.sre.webapps.gitlabreview.hostname = $config.sre.webapps.gitlabreview.vmName
+    $config.sre.webapps.gitlabreview.fqdn = "$($config.sre.webapps.gitlabreview.hostname).$($config.shm.domain.fqdn)"
+    $config.sre.webapps.gitlabreview.ip = "$($config.sre.network.subnets.airlock.prefix).151"
     $config.sre.webapps.hackmd.hostname = $config.sre.webapps.hackmd.vmName
     $config.sre.webapps.hackmd.fqdn = "$($config.sre.webapps.hackmd.hostname).$($config.shm.domain.fqdn)"
     $config.sre.webapps.hackmd.ip = "$($config.sre.network.subnets.data.prefix).152"
@@ -603,7 +640,6 @@ function Add-SreConfig {
     }
 
     $jsonOut = ($config | ConvertTo-Json -Depth 10)
-    # Write-Host $jsonOut
     Out-File -FilePath $sreFullConfigPath -Encoding "UTF8" -InputObject $jsonOut
 }
 Export-ModuleMember -Function Add-SreConfig

deployment/common/Deployments.psm1

@@ -319,6 +319,35 @@ function Deploy-StorageContainer {
 Export-ModuleMember -Function Deploy-StorageContainer
 
 
+# Ensure the specified storage container is empty
+# -----------------------------------------------
+function Clear-StorageContainer {
+    param(
+        [Parameter(Mandatory = $true, HelpMessage = "Name of storage container to clear")]
+        $Name,
+        [Parameter(Mandatory = $true, HelpMessage = "Name of storage account where the container exists")]
+        $StorageAccount
+    )
+    # delete existing blobs in the container
+    $blobs = @(Get-AzStorageBlob -Container $Name -Context $StorageAccount.Context)
+    $numBlobs = $blobs.Length
+    if ($numBlobs -gt 0) {
+        Add-LogMessage -Level Info "[ ] deleting $numBlobs blobs aready in container '$Name'..."
+        $blobs | ForEach-Object { Remove-AzStorageBlob -Blob $_.Name -Container $Name -Context $StorageAccount.Context -Force }
+        while ($numBlobs -gt 0) {
+            Start-Sleep -Seconds 5
+            $numBlobs = (Get-AzStorageBlob -Container $Name -Context $StorageAccount.Context).Length
+        }
+        if ($?) {
+            Add-LogMessage -Level Success "Blob deletion succeeded"
+        } else {
+            Add-LogMessage -Level Fatal "Blob deletion failed!"
+        }
+    }
+}
+Export-ModuleMember -Function Clear-StorageContainer
+
+
 # Create Linux virtual machine if it does not exist
 # -------------------------------------------------
 function Deploy-UbuntuVirtualMachine {
@@ -375,7 +404,7 @@ function Deploy-UbuntuVirtualMachine {
         # Add optional data disks
         $lun = 0
         foreach ($diskId in $DataDiskIds) {
-            $lun += 1
+            $lun += 1  # NB. this line means that our first disk gets deployed at lun1 and we do not use lun0. Consider changing this.
             $vmConfig = Add-AzVMDataDisk -VM $vmConfig -ManagedDiskId $diskId -CreateOption Attach -Lun $lun
         }
         Add-LogMessage -Level Info "[ ] Creating virtual machine '$Name'"