From 59096d4428eb5a467940f2e57c0db1d1477d0cc7 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 13 Dec 2019 21:32:45 +0100 Subject: [PATCH] Added --configure-for-azure option for running in Azure cloud --- README.md | 5 +- aws_helper_test.go | 2 +- azure.go | 169 +++++++++++++++++++++++++++++++++++++++++++ config_test.go | 14 ++-- gcp_helper_test.go | 2 +- main.go | 29 ++++++-- multiuser_windows.go | 26 +++---- usage.go | 5 +- workermanager.go | 9 +++ 9 files changed, 228 insertions(+), 33 deletions(-) create mode 100644 azure.go diff --git a/README.md b/README.md index aa670c68..529f8178 100644 --- a/README.md +++ b/README.md @@ -453,7 +453,7 @@ and reports back results to the queue. Usage: generic-worker run [--config CONFIG-FILE] - [--configure-for-aws | --configure-for-gcp] + [--configure-for-aws | --configure-for-gcp | --configure-for-azure] generic-worker show-payload-schema generic-worker new-ed25519-keypair --file ED25519-PRIVATE-KEY-FILE generic-worker --help @@ -487,6 +487,9 @@ and reports back results to the queue. to self-configure, based on AWS metadata, information from the provisioner, and the worker type definition that the provisioner holds for the worker type. + --configure-for-azure This will create the CONFIG-FILE for an Azure + installation by querying the Azure environment + and setting appropriate values. --configure-for-gcp This will create the CONFIG-FILE for a GCP installation by querying the GCP environment and setting appropriate values. diff --git a/aws_helper_test.go b/aws_helper_test.go index afd89928..2b19a376 100644 --- a/aws_helper_test.go +++ b/aws_helper_test.go @@ -266,7 +266,7 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) (teardown func(), er configFile := &gwconfig.File{ Path: filepath.Join(testdataDir, t.Name(), "generic-worker.config"), } - configProvider, err = loadConfig(configFile, true, false) + configProvider, err = loadConfig(configFile, AWS_PROVIDER) return func() { td() err := s.Shutdown(context.Background()) diff --git a/azure.go b/azure.go new file mode 100644 index 00000000..b7cafa88 --- /dev/null +++ b/azure.go @@ -0,0 +1,169 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + + "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/httpbackoff" +) + +var ( + // not a const, because in testing we swap this out + AzureMetadataBaseURL = "http://169.254.169.254" +) + +type ( + AzureConfigProvider struct { + } + + AzureWorkerLocation struct { + Cloud string `json:"cloud"` + Region string `json:"region"` + } + + AzureMetaData struct { + Compute struct { + CustomData string `json:"customData"` + Location string `json:"location"` + VMID string `json:"vmId"` + VMSize string `json:"vmSize"` + } `json:"compute"` + Network struct { + Interface []struct { + IPV4 struct { + IPAddress []struct { + PrivateIPAddress string `json:"privateIpAddress"` + PublicIPAddress string `json:"publicIpAddress"` + } `json:"ipAddress"` + } `json:"ipv4"` + } `json:"interface"` + } `json:"network"` + } + + AttestedDocument struct { + Encoding string `json:"encoding"` + Signature string `json:"signature"` + } +) + +func queryAzureMetaData(client *http.Client, path string, apiVersion string) ([]byte, error) { + req, err := http.NewRequest("GET", AzureMetadataBaseURL+path+"?api-version="+apiVersion, nil) + + if err != nil { + return nil, err + } + + req.Header.Add("Metadata", "true") + + resp, _, err := httpbackoff.ClientDo(client, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return ioutil.ReadAll(resp.Body) +} + +func (g *AzureConfigProvider) UpdateConfig(c *gwconfig.Config) error { + log.Print("Querying Azure Metadata to get default worker type config settings...") + + client := &http.Client{} + + instanceMetaData, err := queryAzureMetaData(client, "/metadata/instance", "2019-04-30") + if err != nil { + return fmt.Errorf("Could not query taskcluster configuration: %v", err) + } + var azureMetaData AzureMetaData + err = json.Unmarshal(instanceMetaData, &azureMetaData) + if err != nil { + return fmt.Errorf("Could not unmarshal instance metadata %q into AzureMetaData struct - is it valid JSON? %v", string(instanceMetaData), err) + } + + taskclusterConfig, err := base64.StdEncoding.DecodeString(azureMetaData.Compute.CustomData) + if err != nil { + return fmt.Errorf("Custom data %q is not valid base64: %v", azureMetaData.Compute.CustomData, err) + } + + userData := new(WorkerManagerUserData) + err = json.Unmarshal(taskclusterConfig, userData) + if err != nil { + return fmt.Errorf("Could not parse base64 decoded custom data %q as JSON: %v", taskclusterConfig, err) + } + + c.WorkerTypeMetadata["azure"] = map[string]interface{}{ + "location": azureMetaData.Compute.Location, + "vmId": azureMetaData.Compute.VMID, + "vmSize": azureMetaData.Compute.VMSize, + } + c.WorkerID = azureMetaData.Compute.VMID + if len(azureMetaData.Network.Interface) == 1 { + iface := azureMetaData.Network.Interface[0] + if len(iface.IPV4.IPAddress) == 1 { + addr := iface.IPV4.IPAddress[0] + c.PublicIP = net.ParseIP(addr.PublicIPAddress) + c.PrivateIP = net.ParseIP(addr.PrivateIPAddress) + } + } + c.InstanceID = azureMetaData.Compute.VMID + c.InstanceType = azureMetaData.Compute.VMSize + c.AvailabilityZone = azureMetaData.Compute.Location + c.Region = azureMetaData.Compute.Location + + attestedDoc, err := queryAzureMetaData(client, "/metadata/attested/document", "2019-04-30") + if err != nil { + return fmt.Errorf("Could not query taskcluster configuration: %v", err) + } + var doc AttestedDocument + err = json.Unmarshal(attestedDoc, &doc) + if err != nil { + return fmt.Errorf("Could not unmarshal attested document %q into AttestedDocument struct - is it valid JSON? %v", string(attestedDoc), err) + } + if doc.Encoding != "pkcs7" { + return fmt.Errorf(`Attested document has unsupported encoding (%q) - generic-worker only supports "pkcs7"`, doc.Encoding) + } + + // TODO: when Azure Provider is completed in Worker Manager, we'll be able + // to do the following, but that isn't available yet: + // + // providerType := tcworkermanager.AzureProviderType{ + // Document: doc.Signature, + // } + + providerType := map[string]string{ + "document": doc.Signature, + } + + err = userData.UpdateConfig(c, providerType) + if err != nil { + return err + } + + // Don't override WorkerLocation if configuration specifies an explicit + // value. + // + // See: + // * https://github.com/taskcluster/taskcluster-rfcs/blob/master/rfcs/0148-taskcluster-worker-location.md + // * https://github.com/taskcluster/taskcluster-worker-runner#google + if c.WorkerLocation == "" { + workerLocation := &AzureWorkerLocation{ + Cloud: "azure", + Region: c.Region, + } + + workerLocationJSON, err := json.Marshal(workerLocation) + if err != nil { + return fmt.Errorf("Error encoding worker location %#v as JSON: %v", workerLocation, err) + } + c.WorkerLocation = string(workerLocationJSON) + } + return nil +} + +func (g *AzureConfigProvider) NewestDeploymentID() (string, error) { + return WMDeploymentID() +} diff --git a/config_test.go b/config_test.go index ed05c532..4fada6b1 100644 --- a/config_test.go +++ b/config_test.go @@ -15,7 +15,7 @@ func TestMissingIPConfig(t *testing.T) { Path: filepath.Join("testdata", "config", "noip.json"), } const setting = "publicIP" - _, err := loadConfig(file, false, false) + _, err := loadConfig(file, NO_PROVIDER) if err != nil { t.Fatalf("%v", err) } @@ -39,7 +39,7 @@ func TestValidConfig(t *testing.T) { } const ipaddr = "2.1.2.1" const workerType = "some-worker-type" - _, err := loadConfig(file, false, false) + _, err := loadConfig(file, NO_PROVIDER) if err != nil { t.Fatalf("%v", err) } @@ -59,7 +59,7 @@ func TestInvalidIPConfig(t *testing.T) { file := &gwconfig.File{ Path: filepath.Join("testdata", "config", "invalid-ip.json"), } - _, err := loadConfig(file, false, false) + _, err := loadConfig(file, NO_PROVIDER) if err == nil { t.Fatal("Was expecting to get an error back due to an invalid IP address, but didn't get one!") } @@ -73,7 +73,7 @@ func TestInvalidJsonConfig(t *testing.T) { file := &gwconfig.File{ Path: filepath.Join("testdata", "config", "invalid-json.json"), } - _, err := loadConfig(file, false, false) + _, err := loadConfig(file, NO_PROVIDER) if err == nil { t.Fatal("Was expecting to get an error back due to an invalid JSON config, but didn't get one!") } @@ -87,7 +87,7 @@ func TestMissingConfigFile(t *testing.T) { file := &gwconfig.File{ Path: filepath.Join("testdata", "config", "non-existent-json.json"), } - _, err := loadConfig(file, false, false) + _, err := loadConfig(file, NO_PROVIDER) if err == nil { t.Fatal("Was expecting an error when loading non existent config file without --configure-for-{aws,gcp} set") } @@ -100,7 +100,7 @@ func TestWorkerTypeMetadata(t *testing.T) { file := &gwconfig.File{ Path: filepath.Join("testdata", "config", "worker-type-metadata.json"), } - _, err := loadConfig(file, false, false) + _, err := loadConfig(file, NO_PROVIDER) if err != nil { t.Fatalf("%v", err) } @@ -126,7 +126,7 @@ func TestBoolAsString(t *testing.T) { file := &gwconfig.File{ Path: filepath.Join("testdata", "config", "bool-as-string.json"), } - _, err := loadConfig(file, false, false) + _, err := loadConfig(file, NO_PROVIDER) if err == nil { t.Fatal("Was expecting to get an error back due to a bool being specified as a string, but didn't get one!") } diff --git a/gcp_helper_test.go b/gcp_helper_test.go index a11e7f09..4df35f5f 100644 --- a/gcp_helper_test.go +++ b/gcp_helper_test.go @@ -126,7 +126,7 @@ func (m *MockGCPProvisionedEnvironment) Setup(t *testing.T) func() { configFile := &gwconfig.File{ Path: filepath.Join(testdataDir, t.Name(), "generic-worker.config"), } - configProvider, err = loadConfig(configFile, false, true) + configProvider, err = loadConfig(configFile, GCP_PROVIDER) if err != nil { t.Fatalf("Error: %v", err) } diff --git a/main.go b/main.go index d368d649..c0ee3a4e 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,8 @@ var ( configureForAWS bool // Whether we are running in GCP configureForGCP bool + // Whether we are running in Azure + configureForAzure bool // General platform independent user settings, such as home directory, username... // Platform specific data should be managed in plat_.go files taskContext = &TaskContext{} @@ -120,6 +122,7 @@ func main() { case arguments["run"]: configureForAWS = arguments["--configure-for-aws"].(bool) configureForGCP = arguments["--configure-for-gcp"].(bool) + configureForAzure = arguments["--configure-for-azure"].(bool) configFileAbs, err := filepath.Abs(arguments["--config"].(string)) exitOnError(CANT_LOAD_CONFIG, err, "Cannot determine absolute path location for generic-worker config file '%v'", arguments["--config"]) @@ -128,7 +131,17 @@ func main() { Path: configFileAbs, } - configProvider, err = loadConfig(configFile, configureForAWS, configureForGCP) + var provider Provider = NO_PROVIDER + switch { + case configureForAWS: + provider = AWS_PROVIDER + case configureForGCP: + provider = GCP_PROVIDER + case configureForAzure: + provider = AZURE_PROVIDER + } + + configProvider, err = loadConfig(configFile, provider) // We need to persist the generic-worker config file if we fetched it // over the network, for example if the config is fetched from the AWS @@ -200,9 +213,9 @@ func main() { } } -func loadConfig(configFile *gwconfig.File, queryAWSUserData bool, queryGCPMetaData bool) (gwconfig.Provider, error) { +func loadConfig(configFile *gwconfig.File, provider Provider) (gwconfig.Provider, error) { - configProvider, err := ConfigProvider(configFile, queryAWSUserData, queryGCPMetaData) + configProvider, err := ConfigProvider(configFile, provider) if err != nil { return nil, err } @@ -278,17 +291,19 @@ func loadConfig(configFile *gwconfig.File, queryAWSUserData bool, queryGCPMetaDa return configProvider, nil } -func ConfigProvider(configFile *gwconfig.File, queryAWSUserData bool, queryGCPMetaData bool) (gwconfig.Provider, error) { +func ConfigProvider(configFile *gwconfig.File, provider Provider) (gwconfig.Provider, error) { var configProvider gwconfig.Provider - switch { - case queryAWSUserData: + switch provider { + case AWS_PROVIDER: var err error configProvider, err = InferAWSConfigProvider() if err != nil { return nil, err } - case queryGCPMetaData: + case GCP_PROVIDER: configProvider = &GCPConfigProvider{} + case AZURE_PROVIDER: + configProvider = &AzureConfigProvider{} default: configProvider = configFile } diff --git a/multiuser_windows.go b/multiuser_windows.go index b7410270..099ed693 100644 --- a/multiuser_windows.go +++ b/multiuser_windows.go @@ -220,10 +220,14 @@ func install(arguments map[string]interface{}) (err error) { case arguments["service"]: nssm := convertNilToEmptyString(arguments["--nssm"]) serviceName := convertNilToEmptyString(arguments["--service-name"]) - configureForAWS := arguments["--configure-for-aws"].(bool) - configureForGCP := arguments["--configure-for-gcp"].(bool) + extraOpts := "" + for k, _ := range arguments { + if strings.HasPrefix(k, "--configure-for-") { + extraOpts += " " + k + } + } dir := filepath.Dir(exePath) - return deployService(configFile, nssm, serviceName, exePath, dir, configureForAWS, configureForGCP) + return deployService(configFile, nssm, serviceName, exePath, dir, extraOpts) } log.Fatal("Unknown install target - only 'service' is allowed") return nil @@ -387,22 +391,14 @@ func platformTargets(arguments map[string]interface{}) ExitCode { return INTERNAL_ERROR } -func CreateRunGenericWorkerBatScript(batScriptFilePath string, configureForAWS bool, configureForGCP bool) error { - runCommand := `.\generic-worker.exe run` - if configureForAWS { - runCommand += ` --configure-for-aws` - } - if configureForGCP { - runCommand += ` --configure-for-gcp` - } - runCommand += ` > .\generic-worker.log 2>&1` +func CreateRunGenericWorkerBatScript(batScriptFilePath string, extraOpts string) error { batScriptContents := []byte(strings.Join([]string{ `:: Run generic-worker`, ``, `:: step inside folder containing this script`, `pushd %~dp0`, ``, - runCommand, + `.\generic-worker.exe run` + extraOpts + ` > .\generic-worker.log 2>&1`, ``, `:: Possible exit codes:`, `:: 0: all tasks completed - only occurs when numberOfTasksToRun > 0`, @@ -425,9 +421,9 @@ func CreateRunGenericWorkerBatScript(batScriptFilePath string, configureForAWS b // is required to install the service, specified as a file system path. The // serviceName is the service name given to the newly created service. if the // service already exists, it is simply updated. -func deployService(configFile, nssm, serviceName, exePath, dir string, configureForAWS bool, configureForGCP bool) error { +func deployService(configFile, nssm, serviceName, exePath, dir string, extraOpts string) error { targetScript := filepath.Join(filepath.Dir(exePath), "run-generic-worker.bat") - err := CreateRunGenericWorkerBatScript(targetScript, configureForAWS, configureForGCP) + err := CreateRunGenericWorkerBatScript(targetScript, extraOpts) if err != nil { return err } diff --git a/usage.go b/usage.go index 22835a93..35a37335 100644 --- a/usage.go +++ b/usage.go @@ -31,7 +31,7 @@ and reports back results to the queue. Usage: generic-worker run [--config CONFIG-FILE] - [--configure-for-aws | --configure-for-gcp]` + installServiceSummary() + ` + [--configure-for-aws | --configure-for-gcp | --configure-for-azure]` + installServiceSummary() + ` generic-worker show-payload-schema generic-worker new-ed25519-keypair --file ED25519-PRIVATE-KEY-FILE` + customTargetsSummary() + ` generic-worker --help @@ -65,6 +65,9 @@ and reports back results to the queue. to self-configure, based on AWS metadata, information from the provisioner, and the worker type definition that the provisioner holds for the worker type. + --configure-for-azure This will create the CONFIG-FILE for an Azure + installation by querying the Azure environment + and setting appropriate values. --configure-for-gcp This will create the CONFIG-FILE for a GCP installation by querying the GCP environment and setting appropriate values.` + platformCommandLineParameters() + ` diff --git a/workermanager.go b/workermanager.go index dd8f7205..59cdc45c 100644 --- a/workermanager.go +++ b/workermanager.go @@ -19,6 +19,15 @@ type WorkerManagerUserData struct { WorkerConfig BootstrapConfig `json:"workerConfig"` } +type Provider uint + +const ( + NO_PROVIDER = iota + AWS_PROVIDER + AZURE_PROVIDER + GCP_PROVIDER +) + func (userData *WorkerManagerUserData) UpdateConfig(c *gwconfig.Config, providerType interface{}) error { wp := strings.SplitN(userData.WorkerPoolID, "/", -1) if len(wp) != 2 {