Add Azure support

cmd/kops-controller/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
         "//cmd/kops-controller/pkg/server:go_default_library",
         "//pkg/nodeidentity:go_default_library",
         "//pkg/nodeidentity/aws:go_default_library",
+        "//pkg/nodeidentity/azure:go_default_library",
         "//pkg/nodeidentity/do:go_default_library",
         "//pkg/nodeidentity/gce:go_default_library",
         "//pkg/nodeidentity/openstack:go_default_library",

cmd/kops-controller/main.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/kops/cmd/kops-controller/pkg/server"
 	"k8s.io/kops/pkg/nodeidentity"
 	nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
+	nodeidentityazure "k8s.io/kops/pkg/nodeidentity/azure"
 	nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do"
 	nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce"
 	nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
@@ -174,6 +175,12 @@ func addNodeController(mgr manager.Manager, opt *config.Options) error {
 			return fmt.Errorf("error building identifier: %v", err)
 		}
 
+	case "azure":
+		identifier, err = nodeidentityazure.New(opt.CacheNodeidentityInfo)
+		if err != nil {
+			return fmt.Errorf("error building identifier: %v", err)
+		}
+
 	case "":
 		return fmt.Errorf("must specify cloud")
 

cmd/kops/create_cluster.go

@@ -300,6 +300,14 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)")
 	cmd.Flags().StringVar(&options.GCEServiceAccount, "gce-service-account", options.GCEServiceAccount, "Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.")
 
+	if featureflag.Azure.Enabled() {
+		cmd.Flags().StringVar(&options.AzureSubscriptionID, "azure-subscription-id", options.AzureSubscriptionID, "Azure subscription where the cluster is created.")
+		cmd.Flags().StringVar(&options.AzureTenantID, "azure-tenant-id", options.AzureTenantID, "Azure tenant where the cluster is created.")
+		cmd.Flags().StringVar(&options.AzureResourceGroupName, "azure-resource-group-name", options.AzureResourceGroupName, "Azure resource group name where the cluster is created. The resource group will be created if it doesn't already exist. Defaults to the cluster name.")
+		cmd.Flags().StringVar(&options.AzureRouteTableName, "azure-route-table-name", options.AzureRouteTableName, "Azure route table name where the cluster is created.")
+		cmd.Flags().StringVar(&options.AzureAdminUser, "azure-admin-user", options.AzureAdminUser, "Azure admin user of VM ScaleSet.")
+	}
+
 	if featureflag.Spotinst.Enabled() {
 		// Spotinst flags
 		cmd.Flags().StringVar(&options.SpotinstProduct, "spotinst-product", options.SpotinstProduct, "Set the product description (valid values: Linux/UNIX, Linux/UNIX (Amazon VPC), Windows and Windows (Amazon VPC))")

cmd/kops/delete_cluster.go

@@ -133,7 +133,7 @@ func RunDeleteCluster(ctx context.Context, f *util.Factory, out io.Writer, optio
 			}
 		}
 
-		allResources, err := resourceops.ListResources(cloud, clusterName, options.Region)
+		allResources, err := resourceops.ListResources(cloud, cluster, options.Region)
 		if err != nil {
 			return err
 		}

cmd/kops/toolbox_dump.go

@@ -130,7 +130,7 @@ func RunToolboxDump(ctx context.Context, f *util.Factory, out io.Writer, options
 	}
 
 	region := "" // Use default
-	resourceMap, err := resourceops.ListResources(cloud, options.ClusterName, region)
+	resourceMap, err := resourceops.ListResources(cloud, cluster, region)
 	if err != nil {
 		return err
 	}

docs/getting_started/azure.md

@@ -0,0 +1,233 @@
+# Getting Started with kOps on Azure
+
+Azure support on kOps is currently in alpha. The original issue
+ticket is [#3957](https://github.com/kubernetes/kops/issues/3957) and
+the remaining items are tracked in
+[#10412](https://github.com/kubernetes/kops/issues/10412).
+
+# Create Creation Steps
+
+## Step 1. Install Azure CLI
+
+First, install Azure CLI.
+
+```bash
+$ curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+```
+
+Then type the following command to login to Azure. This will redirect
+you to the browser login.
+
+```bash
+$ az login
+
+...
+
+You have logged in. Now let us find all the subscriptions to which you have access...
+[
+  {
+	"cloudName": "AzureCloud",
+	"homeTenantId": "76253...",
+	"id": "7e232...",
+	"isDefault": true,
+	"managedByTenants": [],
+	"name": "Your name...",
+	"state": "Enabled",
+	"tenantId": "76253...",
+	"user": {
+	  "name": "...",
+	  "type": "user"
+	}
+  },
+ ...
+]
+```
+
+One Azure account has one or more than one “subscription”, which
+serves as a single billing unit for Azure resources. Set the env var
+`AZURE_SUBSCRIPTION_ID` to the ID of the subscription you want to
+use.
+
+```bash
+$ export AZURE_SUBSCRIPTION_ID=7e232...
+```
+
+## Step 2. Create a Container in Azure Blob
+
+Next, create an Azure Blob storage container for the kOps cluster store.
+
+First, you need to create a resource group, which provides an isolated
+namespace for resources.
+
+```bash
+$ az group create --name kops-test --location eastus
+{
+  "id": "/subscriptions/7e232.../resourceGroups/kops-test",
+  "location": "eastus",
+  "managedBy": null,
+  "name": "kops-test",
+  "properties": {
+	"provisioningState": "Succeeded"
+  },
+  "tags": null,
+  "type": "Microsoft.Resources/resourceGroups"
+}
+```
+
+Then create a storage account for the resource group. The storage
+account provides an isolated namespace for all storage resources. The
+name must be unique across all Azure accounts.
+
+```bash
+$ az storage account create --name kopstest --resource-group kops-test
+```
+
+Set the env var `AZURE_STORAGE_ACCOUNT` to the storage account name for later use.
+
+```bash
+$ export AZURE_STORAGE_ACCOUNT=kopstest
+```
+
+Get an access key of the account and set it in env var `AZURE_STORAGE_KEY` for later use.
+
+```bash
+$ az storage account keys list --account-name kopstest
+[
+  {
+	"keyName": "key1",
+	"permissions": "Full",
+	"value": "RHWWn..."
+  },
+  {
+	"keyName": "key2",
+	"permissions": "Full",
+	"value": "..."
+  }
+
+]
+
+$ export AZURE_STORAGE_KEY="RHWWn...“
+```
+
+
+Then create a blob container.
+
+```bash
+$ az storage container create --name cluster-configs
+{
+  "created": true
+}
+```
+
+You can confirm that the container has been successfully created from
+Storage Exporter or via `az storage container list`.
+
+```bash
+$ az storage container list --output table
+Name             Lease Status    Last Modified
+---------------  --------------  -------------------------
+cluster-configs  unlocked        2020-10-06T21:12:36+00:00
+```
+
+Set the env var `KOPS_STATE_STORE` to the container name URL using kOps' `azureblob://` protocol.
+The URL may include a path within the container.
+kOps stores all of its cluster configuration within this path.
+
+```bash
+export KOPS_STATE_STORE=azureblob://cluster-configs
+```
+
+## Step 3. Set up Credentials for kOps
+
+Use the following commands to generate kOps credentials.
+
+First, create a service principal in Active Directory.
+
+```bash
+$ az ad sp create-for-rbac --name kops-test --role owner --sdk-auth
+
+{
+  "clientId": "8c6fddb5...",
+  "clientSecret": "dUFzX1...",
+  "subscriptionId": "7e232...",
+  "tenantId": "76253...",
+  ...
+}
+```
+
+Set corresponding env vars:
+
+- Set `AZURE_TENANT_ID` to the `tenantId` of the output
+- Set `AZURE_CLIENT_ID` to the `clienteId` of the output
+- Set `AZURE_CLIENT_SECRET` to the `clientSecret` of the output.
+
+```bash
+$ export AZURE_TENANT_ID="76253..."
+$ export AZURE_CLIENT_ID="8c6fddb5..."
+$ export AZURE_CLIENT_SECRET="dUFzX1..."
+```
+
+## Step 4. Run kOps Commands
+
+Use the following command to create cluster configs in the blob container.
+The command line flags prefixed with `--azure-` are for
+Azure-specific configurations.
+
+```bash
+$ export KOPS_FEATURE_FLAGS=AlphaAllowAzure
+
+$ kops create cluster \
+  --cloud azure \
+  --name my-azure.k8s.local \
+  --zones eastus-1 \
+  --network-cidr 172.16.0.0/16 \
+  --networking calico \
+  --azure-subscription-id "${AZURE_SUBSCRIPTION_ID}" \
+  --azure-tenant-id "${AZURE_TENANT_ID}" \
+  --azure-resource-group-name kops-test \
+  --azure-route-table-name kops-test \
+  --azure-admin-user ubuntu
+```
+
+Confirm that config files are created in Blob storage.
+
+```bash
+$ az storage blob list --container-name cluster-configs --output table
+```
+
+Use the following command to preview the Azure resources
+kOps will create for the k8s cluster.
+
+```bash
+$ kops update cluster  \
+  --name my-azure.k8s.local
+```
+
+Now add the `--yes` flag to have kOps provision the resources
+and create the cluster. This will also add a kubeconfig context
+for the cluster.
+
+```bash
+$ kops update cluster  \
+  --name my-azure.k8s.local \
+  --yes
+```
+
+It may take a few minutes for the cluster's API server to become
+reachable. Please run basic kubectl commands like `kubectl get
+namespaces` to verify the API server is reachable.
+
+Currently kOps creates the following resources in Azure:
+
+- Virtual Machine Scale Sets (equivalent to AWS Auto Scaling Groups)
+- Managed Disks (equivalent to AWS Elastic Volume Storage)
+- Virtual network
+- Subnet
+- Route Table
+- Role Assignment
+
+By default, kOps create two VM Scale Sets - one for the k8s master and the
+other for worker nodes. Managed Disks are used as etcd volumes ("main"
+database and "event" database) and attached to the K8s master
+VMs. Role assignments are needed to grant API access and Blob storage
+access to the VMs.

go.mod

@@ -52,7 +52,11 @@ replace github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud
 require (
 	cloud.google.com/go v0.54.0
 	github.com/Azure/azure-pipeline-go v0.2.3
+	github.com/Azure/azure-sdk-for-go v48.2.0+incompatible
 	github.com/Azure/azure-storage-blob-go v0.10.0
+	github.com/Azure/go-autorest/autorest v0.11.9
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.3
+	github.com/Azure/go-autorest/autorest/to v0.2.0
 	github.com/MakeNowJust/heredoc/v2 v2.0.1
 	github.com/Masterminds/sprig/v3 v3.1.0
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.264

go.sum

@@ -34,25 +34,30 @@ github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVt
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v48.2.0+incompatible h1:+t2P1j1r5N6lYgPiiz7ZbEVZFkWjVe9WhHbMm0gg8hw=
+github.com/Azure/azure-sdk-for-go v48.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs=
 github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
-github.com/Azure/go-autorest/autorest v0.11.1 h1:eVvIXUKiTgv++6YnWb42DUA1YL7qDugnKP0HljexdnQ=
 github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU=
+github.com/Azure/go-autorest/autorest v0.11.9 h1:P0ZF0dEYoUPUVDQo3mA1CvH5b8mKev7DDcmTwauuNME=
+github.com/Azure/go-autorest/autorest v0.11.9/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
 github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/azure/auth v0.5.3 h1:lZifaPRAk1bqg5vGqreL6F8uLC5V0fDpY8nFvc3boFc=
+github.com/Azure/go-autorest/autorest/azure/auth v0.5.3/go.mod h1:4bJZhUhcq8LB20TruwHbAQsmUs2Xh+QR7utuJpLXX3A=
+github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY=
+github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
 github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
@@ -63,7 +68,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN
 github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/to v0.2.0 h1:nQOZzFCudTh+TvquAtCRjM01VEYx85e9qbwt5ncW4L8=
 github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc=
+github.com/Azure/go-autorest/autorest/validation v0.1.0 h1:ISSNzGUh+ZSzizJWOWzs8bwpXIePbGLW4z/AmUFGH5A=
 github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE=
@@ -146,7 +153,6 @@ github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.28.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.31.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.35.24 h1:U3GNTg8+7xSM6OAJ8zksiSM4bRqxBWmVwwehvOSNG3A=
 github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
 github.com/aws/aws-sdk-go v1.36.0 h1:CscTrS+szX5iu34zk2bZrChnGO/GMtUYgMK1Xzs2hYo=
 github.com/aws/aws-sdk-go v1.36.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
@@ -194,7 +200,6 @@ github.com/cilium/ebpf v0.0.0-20200507155900-a9f01edf17e3/go.mod h1:XT+cAw5wfvso
 github.com/cilium/ebpf v0.0.0-20200601085316-9f1617e5c574/go.mod h1:XT+cAw5wfvsodedcijoh1l9cf7v1x9FlFB/3VmF/O8s=
 github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
-github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
@@ -208,7 +213,6 @@ github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on
 github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
 github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.3.3 h1:LoIzb5y9x5l8VKAlyrbusNPXqBY0+kviRloxFUMFwKc=
 github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI=
 github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
@@ -222,7 +226,6 @@ github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kw
 github.com/containerd/typeurl v1.0.0/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
 github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/coredns/corefile-migration v1.0.10/go.mod h1:RMy/mXdeDlYwzt0vdMEJvT2hGJ2I86/eO0UdXmH9XNI=
-github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -262,6 +265,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/digitalocean/godo v1.54.0 h1:KP0Nv87pgViR8k/7De3VrmflCL5pJqXbNnkcw0bwG10=
 github.com/digitalocean/godo v1.54.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU=
+github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4=
+github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20191216044856-a8371794149d/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=