From 76270ab3abfa2d41985df1b8093ccfc0d93b892e Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 14 Oct 2022 10:11:09 +0200 Subject: [PATCH 1/7] [POC] Machine Provisioner CLI Signed-off-by: Waleed Malik --- cmd/machine-provisioner/create.go | 87 +++++++++++++ cmd/machine-provisioner/main.go | 83 +++++++++++++ go.mod | 5 +- go.sum | 3 + .../provider/anexia/instance_test.go | 2 +- pkg/cloudprovider/provider/aws/provider.go | 1 - pkg/cloudprovider/types/types.go | 7 +- pkg/provisioner/provisioner.go | 114 ++++++++++++++++++ 8 files changed, 296 insertions(+), 6 deletions(-) create mode 100644 cmd/machine-provisioner/create.go create mode 100644 cmd/machine-provisioner/main.go create mode 100644 pkg/provisioner/provisioner.go diff --git a/cmd/machine-provisioner/create.go b/cmd/machine-provisioner/create.go new file mode 100644 index 000000000..5ceefdd6f --- /dev/null +++ b/cmd/machine-provisioner/create.go @@ -0,0 +1,87 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "os" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/provisioner" + + "sigs.k8s.io/yaml" +) + +func newCreateCommand(rootFlags *pflag.FlagSet) *cobra.Command { + cmd := &cobra.Command{ + Use: "create", + Short: "Create a machine", + Long: "", + Args: cobra.ExactArgs(0), + SilenceErrors: true, + Example: `machine-provisioner create --machine-config ./machines.yaml`, + Run: func(_ *cobra.Command, _ []string) { + err := runCreateMachineCommand(opts.MachineConfig) + if err != nil { + logrus.Fatal(err) + } + }, + } + + return cmd +} + +func runCreateMachineCommand(machineConfigFile string) error { + logrus.Info("Running command to create machines") + + if len(machineConfigFile) == 0 { + return errors.New("machine configuration path is empty") + } + + machineConfig, err := os.ReadFile(machineConfigFile) + if err != nil { + return errors.New("failed to read machine configuration") + } + + machines, err := parseYAMLToObjects(machineConfig) + if err != nil { + return err + } + + _, err = provisioner.CreateMachines(context.Background(), machines) + if err != nil { + return err + } + + // fmt.Printf("Response %s", *resp) + + return nil +} + +func parseYAMLToObjects(machineByte []byte) ([]clusterv1alpha1.Machine, error) { + machine := []clusterv1alpha1.Machine{} + if err := yaml.Unmarshal(machineByte, &machine); err != nil { + return nil, err + } + + return machine, nil +} diff --git a/cmd/machine-provisioner/main.go b/cmd/machine-provisioner/main.go new file mode 100644 index 000000000..7be657985 --- /dev/null +++ b/cmd/machine-provisioner/main.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +type options struct { + LogFormat string + MachineConfig string +} + +var opts options + +func main() { + rootCmd := newRootCmd() + if err := rootCmd.Execute(); err != nil { + logrus.Fatalf("Error executing machine-provisioner: %v", err) + } +} + +func newRootCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: filepath.Base(os.Args[0]), + Short: "Tool to provision machines", + Long: "Tool to provision machines on various cloud providers.", + PersistentPreRun: runRootCmd, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + + // Options + cmd.PersistentFlags().StringVar(&opts.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") + cmd.PersistentFlags().StringVar(&opts.MachineConfig, "machine-config", "./machines.yaml", "Path to the YAML file for machines") + + cmd.AddCommand(newCreateCommand(cmd.PersistentFlags())) + + return cmd +} + +func runRootCmd(cmd *cobra.Command, args []string) { + err := configureLogging(opts.LogFormat) + if err != nil { + logrus.Warn(err) + } +} + +func configureLogging(logFormat string) error { + logrus.SetLevel(logrus.DebugLevel) + + switch logFormat { + case "json": + logrus.SetFormatter(&logrus.JSONFormatter{}) + default: + // just let the library use default on empty string. + if logFormat != "" { + return fmt.Errorf("unsupported logging formatter: %q", logFormat) + } + } + return nil +} diff --git a/go.mod b/go.mod index d1474b6c8..a279f5d56 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,9 @@ require ( github.com/prometheus/client_golang v1.12.2 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 github.com/sethvargo/go-password v0.2.0 + github.com/sirupsen/logrus v1.8.1 + github.com/spf13/cobra v1.4.0 + github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/go-vcloud-director/v2 v2.15.0 github.com/vmware/govmomi v0.28.0 @@ -113,6 +116,7 @@ require ( github.com/hashicorp/go-version v1.2.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -136,7 +140,6 @@ require ( github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect diff --git a/go.sum b/go.sum index 2325c16c0..8cda23297 100644 --- a/go.sum +++ b/go.sum @@ -535,6 +535,7 @@ github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= @@ -806,6 +807,7 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -824,6 +826,7 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= diff --git a/pkg/cloudprovider/provider/anexia/instance_test.go b/pkg/cloudprovider/provider/anexia/instance_test.go index 0d7641a5f..8340752a9 100644 --- a/pkg/cloudprovider/provider/anexia/instance_test.go +++ b/pkg/cloudprovider/provider/anexia/instance_test.go @@ -20,8 +20,8 @@ import ( "testing" "github.com/gophercloud/gophercloud/testhelper" - "go.anx.io/go-anxcloud/pkg/vsphere/info" + v1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index e72cdcae4..d39d15294 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -34,7 +34,6 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/aws/smithy-go" - gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index adc9010b1..d5517edeb 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -84,9 +84,10 @@ type MachineUpdater func(*clusterv1alpha1.Machine, ...MachineModifier) error // ProviderData is the struct the cloud providers get when creating or deleting an instance. type ProviderData struct { - Ctx context.Context - Update MachineUpdater - Client ctrlruntimeclient.Client + Ctx context.Context + Update MachineUpdater + Client ctrlruntimeclient.Client + ProvisionerMode bool } // GetMachineUpdater returns an MachineUpdater based on the passed in context and ctrlruntimeclient.Client. diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go new file mode 100644 index 000000000..04f3d55a6 --- /dev/null +++ b/pkg/provisioner/provisioner.go @@ -0,0 +1,114 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provisioner + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/sirupsen/logrus" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" +) + +const maxRetrieForMachines = 5 + +func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*string, error) { + providerData := &cloudprovidertypes.ProviderData{ + Ctx: ctx, + ProvisionerMode: true, + } + + // TODO: Dump all the errors in an array and do the max that is possible without early exit + for _, machine := range machines { + prov, err := getProvider(ctx, machine) + if err != nil { + return nil, err + } + + machineCreated := false + providerInstance, err := prov.Get(ctx, &machine, providerData) + if err != nil { + // case 1: instance was not found and we are going to create one + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { + // Create the instance + _, err := prov.Create(ctx, &machine, providerData, "") + if err != nil { + return nil, err + } + machineCreated = true + } else if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { + // case 2: terminal error was returned and manual interaction is required to recover + return nil, fmt.Errorf("failed to create machine at cloudprovider, due to %w", err) + } else { + // case 3: transient error was returned, requeue the request and try again in the future + return nil, fmt.Errorf("failed to get instance from provider: %w", err) + } + } + + if machineCreated { + for i := 0; i < maxRetrieForMachines; i++ { + providerInstance, err = prov.Get(ctx, &machine, providerData) + if err != nil { + return nil, err + } + + addresses := providerInstance.Addresses() + if len(addresses) > 0 { + break + } + logrus.Debug("Waiting 10 seconds for machine address assignment.") + time.Sleep(10 * time.Second) + } + } + + // Instance exists + addresses := providerInstance.Addresses() + if len(addresses) == 0 { + return nil, fmt.Errorf("machine %s has not been assigned an IP yet", providerInstance.Name()) + } + + if machineCreated { + logrus.Infof("Machine %q was successfully created.", providerInstance.Name()) + } else { + logrus.Infof("Machine %q already exists.", providerInstance.Name()) + } + } + + return nil, nil +} + +func getProvider(ctx context.Context, machine clusterv1alpha1.Machine) (cloudprovidertypes.Provider, error) { + providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, fmt.Errorf("failed to get provider config: %w", err) + } + skg := providerconfig.NewConfigVarResolver(ctx, nil) + prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) + if err != nil { + return nil, fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) + } + + return prov, nil +} From a77aea05cda76f2daa9af2aa270a51c782979c22 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 14 Oct 2022 11:26:48 +0200 Subject: [PATCH 2/7] Generate output for machine-provisioner Signed-off-by: Waleed Malik --- cmd/machine-provisioner/create.go | 14 +++++- pkg/provisioner/output.go | 78 +++++++++++++++++++++++++++++++ pkg/provisioner/provisioner.go | 9 +++- 3 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 pkg/provisioner/output.go diff --git a/cmd/machine-provisioner/create.go b/cmd/machine-provisioner/create.go index 5ceefdd6f..a7c3d6fd7 100644 --- a/cmd/machine-provisioner/create.go +++ b/cmd/machine-provisioner/create.go @@ -18,6 +18,7 @@ package main import ( "context" + "encoding/json" "errors" "os" @@ -67,13 +68,22 @@ func runCreateMachineCommand(machineConfigFile string) error { return err } - _, err = provisioner.CreateMachines(context.Background(), machines) + out, err := provisioner.CreateMachines(context.Background(), machines) if err != nil { return err } - // fmt.Printf("Response %s", *resp) + b, err := json.MarshalIndent(out, "", " ") + if err != nil { + return err + } + + err = os.WriteFile("machines.json", b, 0600) + if err != nil { + return err + } + logrus.Info("Create task ran successfully. Output is available in %q.", provisioner.OutputFileName) return nil } diff --git a/pkg/provisioner/output.go b/pkg/provisioner/output.go new file mode 100644 index 000000000..b313f965c --- /dev/null +++ b/pkg/provisioner/output.go @@ -0,0 +1,78 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provisioner + +import ( + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + + v1 "k8s.io/api/core/v1" +) + +const OutputFileName = "machines.json" + +type output struct { + Machines []machine `json:"machines"` +} + +type machine struct { + Name string `json:"name"` + ID string `json:"id"` + PublicAddress string `json:"public_address,omitempty"` + PrivateAddress string `json:"private_address,omitempty"` + InternalDNS string `json:"internal_dns,omitempty"` + ExternalDNS string `json:"external_dns,omitempty"` + Hostname string `json:"hostname,omitempty"` + SSHUser string `json:"ssh_user,omitempty"` + Bastion bool `json:"bastion,omitempty"` +} + +func getMachineProvisionerOutput(instances []instance.Instance) output { + var out output + + for _, instance := range instances { + machine := getMachineInfo(instance) + out.Machines = append(out.Machines, machine) + } + return out +} + +func getMachineInfo(inst instance.Instance) machine { + var publicAddress, privateAddress, hostname, internalDNS, externalDNS string + for address, addressType := range inst.Addresses() { + if addressType == v1.NodeExternalIP { + publicAddress = address + } else if addressType == v1.NodeInternalIP { + privateAddress = address + } else if addressType == v1.NodeHostName { + hostname = address + } else if addressType == v1.NodeInternalDNS { + internalDNS = address + } else if addressType == v1.NodeExternalDNS { + externalDNS = address + } + } + + return machine{ + Name: inst.Name(), + ID: inst.ProviderID(), + PublicAddress: publicAddress, + PrivateAddress: privateAddress, + Hostname: hostname, + InternalDNS: internalDNS, + ExternalDNS: externalDNS, + } +} diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go index 04f3d55a6..df22fca1b 100644 --- a/pkg/provisioner/provisioner.go +++ b/pkg/provisioner/provisioner.go @@ -27,6 +27,7 @@ import ( clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -34,12 +35,14 @@ import ( const maxRetrieForMachines = 5 -func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*string, error) { +func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*output, error) { providerData := &cloudprovidertypes.ProviderData{ Ctx: ctx, ProvisionerMode: true, } + var instances []instance.Instance + // TODO: Dump all the errors in an array and do the max that is possible without early exit for _, machine := range machines { prov, err := getProvider(ctx, machine) @@ -94,9 +97,11 @@ func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*s } else { logrus.Infof("Machine %q already exists.", providerInstance.Name()) } + instances = append(instances, providerInstance) } - return nil, nil + output := getMachineProvisionerOutput(instances) + return &output, nil } func getProvider(ctx context.Context, machine clusterv1alpha1.Machine) (cloudprovidertypes.Provider, error) { From 2aecfba3c4e25e2b083f0d8a56815b1a1fbe006f Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 14 Oct 2022 11:38:26 +0200 Subject: [PATCH 3/7] Wait till the public and private IP addresses are available Signed-off-by: Waleed Malik --- cmd/machine-provisioner/create.go | 2 +- pkg/provisioner/output.go | 13 +++++++++++++ pkg/provisioner/provisioner.go | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/cmd/machine-provisioner/create.go b/cmd/machine-provisioner/create.go index a7c3d6fd7..1d7a04754 100644 --- a/cmd/machine-provisioner/create.go +++ b/cmd/machine-provisioner/create.go @@ -83,7 +83,7 @@ func runCreateMachineCommand(machineConfigFile string) error { return err } - logrus.Info("Create task ran successfully. Output is available in %q.", provisioner.OutputFileName) + logrus.Infof("Create task ran successfully. Output is available in %q.", provisioner.OutputFileName) return nil } diff --git a/pkg/provisioner/output.go b/pkg/provisioner/output.go index b313f965c..9e6921a3d 100644 --- a/pkg/provisioner/output.go +++ b/pkg/provisioner/output.go @@ -76,3 +76,16 @@ func getMachineInfo(inst instance.Instance) machine { ExternalDNS: externalDNS, } } + +func publicAndPrivateIPExist(addresses map[string]v1.NodeAddressType) bool { + var publicIPExists, privateIPExists bool + for _, addressType := range addresses { + if addressType == v1.NodeExternalIP { + publicIPExists = true + } else if addressType == v1.NodeInternalIP { + privateIPExists = true + } + } + + return publicIPExists && privateIPExists +} diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go index df22fca1b..14f9a8162 100644 --- a/pkg/provisioner/provisioner.go +++ b/pkg/provisioner/provisioner.go @@ -78,10 +78,10 @@ func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*o } addresses := providerInstance.Addresses() - if len(addresses) > 0 { + if len(addresses) > 0 && publicAndPrivateIPExist(addresses) { break } - logrus.Debug("Waiting 10 seconds for machine address assignment.") + logrus.Info("Waiting 10 seconds for machine address assignment.") time.Sleep(10 * time.Second) } } From a495dc6bfc19e29f32544917dad8d0028a265d2a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 14 Oct 2022 11:55:57 +0200 Subject: [PATCH 4/7] Support to specify SSH username Signed-off-by: Waleed Malik --- pkg/provisioner/output.go | 13 ++++++------- pkg/provisioner/provisioner.go | 27 ++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/pkg/provisioner/output.go b/pkg/provisioner/output.go index 9e6921a3d..86cc35847 100644 --- a/pkg/provisioner/output.go +++ b/pkg/provisioner/output.go @@ -17,8 +17,6 @@ limitations under the License. package provisioner import ( - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - v1 "k8s.io/api/core/v1" ) @@ -40,7 +38,7 @@ type machine struct { Bastion bool `json:"bastion,omitempty"` } -func getMachineProvisionerOutput(instances []instance.Instance) output { +func getMachineProvisionerOutput(instances []MachineInstance) output { var out output for _, instance := range instances { @@ -50,9 +48,9 @@ func getMachineProvisionerOutput(instances []instance.Instance) output { return out } -func getMachineInfo(inst instance.Instance) machine { +func getMachineInfo(instance MachineInstance) machine { var publicAddress, privateAddress, hostname, internalDNS, externalDNS string - for address, addressType := range inst.Addresses() { + for address, addressType := range instance.inst.Addresses() { if addressType == v1.NodeExternalIP { publicAddress = address } else if addressType == v1.NodeInternalIP { @@ -67,13 +65,14 @@ func getMachineInfo(inst instance.Instance) machine { } return machine{ - Name: inst.Name(), - ID: inst.ProviderID(), + Name: instance.inst.Name(), + ID: instance.inst.ProviderID(), PublicAddress: publicAddress, PrivateAddress: privateAddress, Hostname: hostname, InternalDNS: internalDNS, ExternalDNS: externalDNS, + SSHUser: instance.sshUser, } } diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go index 14f9a8162..36cde9d5d 100644 --- a/pkg/provisioner/provisioner.go +++ b/pkg/provisioner/provisioner.go @@ -33,7 +33,15 @@ import ( providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) -const maxRetrieForMachines = 5 +const ( + maxRetrieForMachines = 5 + hostnameAnnotation = "ssh-username" +) + +type MachineInstance struct { + inst instance.Instance + sshUser string +} func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*output, error) { providerData := &cloudprovidertypes.ProviderData{ @@ -41,7 +49,7 @@ func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*o ProvisionerMode: true, } - var instances []instance.Instance + var instances []MachineInstance // TODO: Dump all the errors in an array and do the max that is possible without early exit for _, machine := range machines { @@ -97,7 +105,20 @@ func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*o } else { logrus.Infof("Machine %q already exists.", providerInstance.Name()) } - instances = append(instances, providerInstance) + + sshUser := "root" + if machine.Annotations != nil { + if user := machine.Annotations[hostnameAnnotation]; sshUser != "" { + sshUser = user + } + } + + machineInstance := MachineInstance{ + inst: providerInstance, + sshUser: sshUser, + } + + instances = append(instances, machineInstance) } output := getMachineProvisionerOutput(instances) From dafde83f243e789eef3a2deae5c7fa25764a3c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Fri, 14 Oct 2022 12:01:56 +0200 Subject: [PATCH 5/7] SSH keys userdata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/provisioner/provisioner.go | 46 +++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go index 36cde9d5d..a1a8730a3 100644 --- a/pkg/provisioner/provisioner.go +++ b/pkg/provisioner/provisioner.go @@ -20,6 +20,8 @@ import ( "context" "errors" "fmt" + "strings" + "text/template" "time" "github.com/sirupsen/logrus" @@ -31,13 +33,45 @@ import ( cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" ) const ( maxRetrieForMachines = 5 hostnameAnnotation = "ssh-username" + + userDataTemplate = `#cloud-config +ssh_pwauth: false + +{{- if .ProviderSpec.SSHPublicKeys }} +ssh_authorized_keys: +{{- range .ProviderSpec.SSHPublicKeys }} +- "{{ . }}" +{{- end }} +{{- end }} +` ) +func getUserData(pconfig *providerconfigtypes.Config) (string, error) { + data := struct { + ProviderSpec *providerconfigtypes.Config + }{ + ProviderSpec: pconfig, + } + + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse user-data template: %w", err) + } + + var buf strings.Builder + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("failed to execute user-data template: %w", err) + } + + return userdatahelper.CleanupTemplateOutput(buf.String()) +} + type MachineInstance struct { inst instance.Instance sshUser string @@ -63,8 +97,18 @@ func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*o if err != nil { // case 1: instance was not found and we are going to create one if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { + // Get userdata (needed to inject SSH keys to instances) + pconfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, fmt.Errorf("failed to get providerSpec: %w", err) + } + userdata, err := getUserData(pconfig) + if err != nil { + return nil, err + } + // Create the instance - _, err := prov.Create(ctx, &machine, providerData, "") + _, err = prov.Create(ctx, &machine, providerData, userdata) if err != nil { return nil, err } From 25de0de4ea669b70552055fb3bb9c452d46ddb67 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 14 Oct 2022 12:07:30 +0200 Subject: [PATCH 6/7] Reduce wait time Signed-off-by: Waleed Malik --- pkg/provisioner/provisioner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go index a1a8730a3..8b54c36f3 100644 --- a/pkg/provisioner/provisioner.go +++ b/pkg/provisioner/provisioner.go @@ -133,8 +133,8 @@ func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*o if len(addresses) > 0 && publicAndPrivateIPExist(addresses) { break } - logrus.Info("Waiting 10 seconds for machine address assignment.") - time.Sleep(10 * time.Second) + logrus.Info("Waiting 5 seconds for machine address assignment.") + time.Sleep(5 * time.Second) } } From ca2a7622da4420929e5a4c8a731ef946ef03f3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Fri, 14 Oct 2022 12:24:33 +0200 Subject: [PATCH 7/7] Remove unneeded fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/provisioner/output.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pkg/provisioner/output.go b/pkg/provisioner/output.go index 86cc35847..93b814575 100644 --- a/pkg/provisioner/output.go +++ b/pkg/provisioner/output.go @@ -27,12 +27,8 @@ type output struct { } type machine struct { - Name string `json:"name"` - ID string `json:"id"` PublicAddress string `json:"public_address,omitempty"` PrivateAddress string `json:"private_address,omitempty"` - InternalDNS string `json:"internal_dns,omitempty"` - ExternalDNS string `json:"external_dns,omitempty"` Hostname string `json:"hostname,omitempty"` SSHUser string `json:"ssh_user,omitempty"` Bastion bool `json:"bastion,omitempty"` @@ -49,7 +45,7 @@ func getMachineProvisionerOutput(instances []MachineInstance) output { } func getMachineInfo(instance MachineInstance) machine { - var publicAddress, privateAddress, hostname, internalDNS, externalDNS string + var publicAddress, privateAddress, hostname string for address, addressType := range instance.inst.Addresses() { if addressType == v1.NodeExternalIP { publicAddress = address @@ -58,20 +54,14 @@ func getMachineInfo(instance MachineInstance) machine { } else if addressType == v1.NodeHostName { hostname = address } else if addressType == v1.NodeInternalDNS { - internalDNS = address - } else if addressType == v1.NodeExternalDNS { - externalDNS = address + hostname = address } } return machine{ - Name: instance.inst.Name(), - ID: instance.inst.ProviderID(), PublicAddress: publicAddress, PrivateAddress: privateAddress, Hostname: hostname, - InternalDNS: internalDNS, - ExternalDNS: externalDNS, SSHUser: instance.sshUser, } }