Add vhdbuilder and publisher

.pipelines/.vsts-vhd-builder-windows.yaml

@@ -0,0 +1,102 @@
+# Required pipeline variables:
+# - AZURE_LOCATION - Azure region to create resources in
+# - AZURE_RESOURCE_GROUP_NAME - Resource group to create resources in
+# - BUILD_POOL - Azure DevOps build pool to use
+# - CLIENT_ID - Service principal ID
+# - CLIENT_SECRET - Service principal secret
+# - COPY_VHD - Set to 'True' if VHD should be copied to classic storage account in preparation for publishing
+# - DEIS_GO_DEV_IMAGE - Dev container image ID
+# - PACKER_VM_SIZE - VM Size to be used during packer build operation
+# - STORAGE_ACCT_BLOB_URL - Url to blob storage container used to copy image to in preparation for publishing
+# - STORAGE_ACCT_SAS_TOKEN - SAS token with permission to write to CLASSIC_BLOB container
+# - SUBSCRIPTION_ID - Subscription ID used by the pipeline
+# - TENANT_ID - Tenant Id used for service principal login
+
+trigger: none
+pr: none
+
+jobs:
+  - job: build_vhd_windows
+    timeoutInMinutes: 120
+    strategy:
+      maxParallel: 0
+    pool:
+      name: $(BUILD_POOL)
+    steps:
+      - script: |
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \
+          -e AZURE_LOCATION=${AZURE_LOCATION} \
+          -e AZURE_VM_SIZE=${PACKER_VM_SIZE} \
+          -e GIT_BRANCH=$(Build.SourceBranchName) \
+          -e GIT_REPO=$(Build.Repository.Uri) \
+          -e GIT_VERSION=$(Build.SourceVersion) \
+          -e BUILD_ID=$(Build.BuildId) \
+          -e BUILD_NUMBER=$(Build.BuildNumber) \
+          ${DEIS_GO_DEV_IMAGE} make -f packer.mk run-packer-windows
+        displayName: Building windows VHD
+      - task: PublishPipelineArtifact@1
+        inputs:
+          artifact: 'vhd-release-notes'
+          path: 'release-notes.txt'
+      - script: |
+          OS_DISK_SAS="$(cat packer-output | grep "OSDiskUriReadOnlySas:" | cut -d " " -f 2)" && \
+          VHD_NAME="$(echo $OS_DISK_SAS | cut -d "/" -f 8 | cut -d "?" -f 1)" && \
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e CLASSIC_BLOB=${STORAGE_ACCT_BLOB_URL} \
+          -e CLASSIC_SAS_TOKEN="$(STORAGE_ACCT_SAS_TOKEN)" \
+          -e OS_DISK_SAS=${OS_DISK_SAS} \
+          -e VHD_NAME=${VHD_NAME} \
+          ${DEIS_GO_DEV_IMAGE} make -f packer.mk az-copy
+        displayName: Copying resource to Classic Storage Account
+        condition: eq(variables.COPY_VHD, 'True')
+      - script: |
+          OS_DISK_SAS="$(cat packer-output | grep "OSDiskUriReadOnlySas:" | cut -d " " -f 2)" && \
+          VHD_NAME="$(echo $OS_DISK_SAS | cut -d "/" -f 8 | cut -d "?" -f 1)" && \
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e CLASSIC_SA_CONNECTION_STRING="$(CLASSIC_SA_CONNECTION_STRING)" \
+          -e STORAGE_ACCT_BLOB_URL=${STORAGE_ACCT_BLOB_URL} \
+          -e VHD_NAME=${VHD_NAME} \
+          -e OS_NAME="Windows" \
+          -e SKU_NAME="2019-Datacenter-Core-smalldisk" \
+          -e OFFER_NAME="Windows" \
+          ${DEIS_GO_DEV_IMAGE} make -f packer.mk generate-sas
+        displayName: Generating publishing info for Windows VHD
+        condition: eq(variables.COPY_VHD, 'True')
+      - task: PublishPipelineArtifact@1
+        inputs:
+          artifact: 'publishing-info'
+          path: 'vhd-publishing-info.json'
+        condition: eq(variables.COPY_VHD, 'True')
+      - script: |
+          SA_NAME="$(cat packer-output | grep "storage name:" | cut -d " " -f 3)" && \
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e SA_NAME=${SA_NAME} \
+          -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \
+          ${DEIS_GO_DEV_IMAGE} make -f packer.mk delete-sa
+        displayName: cleanup - delete temp storage account
+        condition: always()
+      - script: |
+          sudo chown -R $USER:$USER .
+        displayName: cleanup - chown all files in work directory
+        condition: always()

.pipelines/.vsts-vhd-builder.yaml

@@ -0,0 +1,88 @@
+trigger: none
+
+# steps:
+# - create an VHD in Packer to normal storage account
+# - copy from Packer storage account to classic storage account using AzCopy
+# - generate SAS link from azure CLI
+# - POST a new SKU to azure marketplace
+
+variables:
+  CONTAINER_IMAGE:  'quay.io/deis/go-dev:v1.25.2'
+
+phases:
+  - phase: build_vhd
+    queue:
+      name: Hosted Ubuntu 1604
+      timeoutInMinutes: 120
+    steps:
+      - script: |
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e AZURE_VM_SIZE=${AZURE_VM_SIZE} \
+          -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \
+          -e AZURE_LOCATION=${AZURE_LOCATION} \
+          -e FEATURE_FLAGS=${FEATURE_FLAGS} \
+          -e GIT_VERSION=$(Build.SourceVersion) \
+          -e BUILD_ID=$(Build.BuildId) \
+          -e BUILD_NUMBER=$(Build.BuildNumber) \
+          -e UBUNTU_SKU=${UBUNTU_SKU} \
+          ${CONTAINER_IMAGE} make  -f packer.mk run-packer
+        displayName: Building VHD
+      - task: PublishPipelineArtifact@0
+        inputs:
+          artifactName: 'vhd-release-notes'
+          targetPath: 'release-notes.txt'
+      - script: |
+          OS_DISK_SAS="$(cat packer-output | grep "OSDiskUriReadOnlySas:" | cut -d " " -f 2)" && \
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e CLASSIC_BLOB=${CLASSIC_BLOB} \
+          -e CLASSIC_SAS_TOKEN="$(SAS_TOKEN)" \
+          -e OS_DISK_SAS=${OS_DISK_SAS} \
+          ${CONTAINER_IMAGE} make -f packer.mk az-copy
+        displayName: Copying resource to Classic Storage Account
+        condition: eq(variables.DRY_RUN, 'False')
+      - script: |
+          SA_NAME="$(cat packer-output | grep "storage name:" | cut -d " " -f 3)" && \
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e SA_NAME=${SA_NAME} \
+          -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \
+          ${CONTAINER_IMAGE} make -f packer.mk delete-sa
+        displayName: Clean-up Storage Account
+        condition: always()
+      - script: |
+          OS_DISK_SAS="$(cat packer-output | grep "OSDiskUriReadOnlySas:" | cut -d " " -f 2)" && \
+          VHD_NAME="$(echo $OS_DISK_SAS | cut -d "/" -f 8 | cut -d "?" -f 1)" && \
+          docker run --rm \
+          -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+          -w /go/src/github.com/Azure/AgentBaker \
+          -e CLIENT_ID=${CLIENT_ID} \
+          -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+          -e TENANT_ID=${TENANT_ID} \
+          -e CLASSIC_SA_CONNECTION_STRING="$(CLASSIC_SA_CONNECTION_STRING)" \
+          -e STORAGE_ACCT_BLOB_URL=${CLASSIC_BLOB} \
+          -e VHD_NAME=${VHD_NAME} \
+          -e OS_NAME="Linux" \
+          -e SKU_NAME=${UBUNTU_SKU} \
+          -e OFFER_NAME="Ubuntu" \
+          ${CONTAINER_IMAGE} make  -f packer.mk generate-sas
+        displayName: Getting Shared Access Signature URI
+        condition: eq(variables.DRY_RUN, 'False')
+      - task: PublishPipelineArtifact@1
+        inputs:
+          artifactName: 'publishing-info'
+          targetPath: 'vhd-publishing-info.json'
+        condition: eq(variables.DRY_RUN, 'False')

packer.mk

@@ -0,0 +1,26 @@
+build-packer:
+	@packer build -var-file=vhdbuilder/packer/settings.json vhdbuilder/packer/vhd-image-builder.json
+
+build-packer-windows:
+	@packer build -var-file=vhdbuilder/packer/settings.json vhdbuilder/packer/windows-vhd-builder.json
+
+init-packer:
+	@./vhdbuilder/packer/init-variables.sh
+
+az-login:
+	az login --service-principal -u ${CLIENT_ID} -p ${CLIENT_SECRET} --tenant ${TENANT_ID}
+
+run-packer: az-login
+	@packer version && ($(MAKE) -f packer.mk init-packer | tee packer-output) && ($(MAKE) -f packer.mk build-packer | tee -a packer-output)
+
+run-packer-windows: az-login
+	@packer version && ($(MAKE) -f packer.mk init-packer | tee packer-output) && ($(MAKE) -f packer.mk build-packer-windows | tee -a packer-output)
+
+az-copy: az-login
+	azcopy-preview copy "${OS_DISK_SAS}" "${CLASSIC_BLOB}${CLASSIC_SAS_TOKEN}" --recursive=true
+
+delete-sa: az-login
+	az storage account delete -n ${SA_NAME} -g ${AZURE_RESOURCE_GROUP_NAME} --yes
+
+generate-sas: az-login
+	@./vhdbuilder/packer/generate-vhd-publishing-info.sh