diff --git a/cmd/machine-provisioner/create.go b/cmd/machine-provisioner/create.go new file mode 100644 index 000000000..1d7a04754 --- /dev/null +++ b/cmd/machine-provisioner/create.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/provisioner" + + "sigs.k8s.io/yaml" +) + +func newCreateCommand(rootFlags *pflag.FlagSet) *cobra.Command { + cmd := &cobra.Command{ + Use: "create", + Short: "Create a machine", + Long: "", + Args: cobra.ExactArgs(0), + SilenceErrors: true, + Example: `machine-provisioner create --machine-config ./machines.yaml`, + Run: func(_ *cobra.Command, _ []string) { + err := runCreateMachineCommand(opts.MachineConfig) + if err != nil { + logrus.Fatal(err) + } + }, + } + + return cmd +} + +func runCreateMachineCommand(machineConfigFile string) error { + logrus.Info("Running command to create machines") + + if len(machineConfigFile) == 0 { + return errors.New("machine configuration path is empty") + } + + machineConfig, err := os.ReadFile(machineConfigFile) + if err != nil { + return errors.New("failed to read machine configuration") + } + + machines, err := parseYAMLToObjects(machineConfig) + if err != nil { + return err + } + + out, err := provisioner.CreateMachines(context.Background(), machines) + if err != nil { + return err + } + + b, err := json.MarshalIndent(out, "", " ") + if err != nil { + return err + } + + err = os.WriteFile("machines.json", b, 0600) + if err != nil { + return err + } + + logrus.Infof("Create task ran successfully. Output is available in %q.", provisioner.OutputFileName) + return nil +} + +func parseYAMLToObjects(machineByte []byte) ([]clusterv1alpha1.Machine, error) { + machine := []clusterv1alpha1.Machine{} + if err := yaml.Unmarshal(machineByte, &machine); err != nil { + return nil, err + } + + return machine, nil +} diff --git a/cmd/machine-provisioner/main.go b/cmd/machine-provisioner/main.go new file mode 100644 index 000000000..7be657985 --- /dev/null +++ b/cmd/machine-provisioner/main.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +type options struct { + LogFormat string + MachineConfig string +} + +var opts options + +func main() { + rootCmd := newRootCmd() + if err := rootCmd.Execute(); err != nil { + logrus.Fatalf("Error executing machine-provisioner: %v", err) + } +} + +func newRootCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: filepath.Base(os.Args[0]), + Short: "Tool to provision machines", + Long: "Tool to provision machines on various cloud providers.", + PersistentPreRun: runRootCmd, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + + // Options + cmd.PersistentFlags().StringVar(&opts.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") + cmd.PersistentFlags().StringVar(&opts.MachineConfig, "machine-config", "./machines.yaml", "Path to the YAML file for machines") + + cmd.AddCommand(newCreateCommand(cmd.PersistentFlags())) + + return cmd +} + +func runRootCmd(cmd *cobra.Command, args []string) { + err := configureLogging(opts.LogFormat) + if err != nil { + logrus.Warn(err) + } +} + +func configureLogging(logFormat string) error { + logrus.SetLevel(logrus.DebugLevel) + + switch logFormat { + case "json": + logrus.SetFormatter(&logrus.JSONFormatter{}) + default: + // just let the library use default on empty string. + if logFormat != "" { + return fmt.Errorf("unsupported logging formatter: %q", logFormat) + } + } + return nil +} diff --git a/go.mod b/go.mod index d1474b6c8..a279f5d56 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,9 @@ require ( github.com/prometheus/client_golang v1.12.2 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 github.com/sethvargo/go-password v0.2.0 + github.com/sirupsen/logrus v1.8.1 + github.com/spf13/cobra v1.4.0 + github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/go-vcloud-director/v2 v2.15.0 github.com/vmware/govmomi v0.28.0 @@ -113,6 +116,7 @@ require ( github.com/hashicorp/go-version v1.2.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -136,7 +140,6 @@ require ( github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect diff --git a/go.sum b/go.sum index 2325c16c0..8cda23297 100644 --- a/go.sum +++ b/go.sum @@ -535,6 +535,7 @@ github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= @@ -806,6 +807,7 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -824,6 +826,7 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= diff --git a/pkg/cloudprovider/provider/anexia/instance_test.go b/pkg/cloudprovider/provider/anexia/instance_test.go index 0d7641a5f..8340752a9 100644 --- a/pkg/cloudprovider/provider/anexia/instance_test.go +++ b/pkg/cloudprovider/provider/anexia/instance_test.go @@ -20,8 +20,8 @@ import ( "testing" "github.com/gophercloud/gophercloud/testhelper" - "go.anx.io/go-anxcloud/pkg/vsphere/info" + v1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index e72cdcae4..d39d15294 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -34,7 +34,6 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/aws/smithy-go" - gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index adc9010b1..d5517edeb 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -84,9 +84,10 @@ type MachineUpdater func(*clusterv1alpha1.Machine, ...MachineModifier) error // ProviderData is the struct the cloud providers get when creating or deleting an instance. type ProviderData struct { - Ctx context.Context - Update MachineUpdater - Client ctrlruntimeclient.Client + Ctx context.Context + Update MachineUpdater + Client ctrlruntimeclient.Client + ProvisionerMode bool } // GetMachineUpdater returns an MachineUpdater based on the passed in context and ctrlruntimeclient.Client. diff --git a/pkg/provisioner/output.go b/pkg/provisioner/output.go new file mode 100644 index 000000000..93b814575 --- /dev/null +++ b/pkg/provisioner/output.go @@ -0,0 +1,80 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provisioner + +import ( + v1 "k8s.io/api/core/v1" +) + +const OutputFileName = "machines.json" + +type output struct { + Machines []machine `json:"machines"` +} + +type machine struct { + PublicAddress string `json:"public_address,omitempty"` + PrivateAddress string `json:"private_address,omitempty"` + Hostname string `json:"hostname,omitempty"` + SSHUser string `json:"ssh_user,omitempty"` + Bastion bool `json:"bastion,omitempty"` +} + +func getMachineProvisionerOutput(instances []MachineInstance) output { + var out output + + for _, instance := range instances { + machine := getMachineInfo(instance) + out.Machines = append(out.Machines, machine) + } + return out +} + +func getMachineInfo(instance MachineInstance) machine { + var publicAddress, privateAddress, hostname string + for address, addressType := range instance.inst.Addresses() { + if addressType == v1.NodeExternalIP { + publicAddress = address + } else if addressType == v1.NodeInternalIP { + privateAddress = address + } else if addressType == v1.NodeHostName { + hostname = address + } else if addressType == v1.NodeInternalDNS { + hostname = address + } + } + + return machine{ + PublicAddress: publicAddress, + PrivateAddress: privateAddress, + Hostname: hostname, + SSHUser: instance.sshUser, + } +} + +func publicAndPrivateIPExist(addresses map[string]v1.NodeAddressType) bool { + var publicIPExists, privateIPExists bool + for _, addressType := range addresses { + if addressType == v1.NodeExternalIP { + publicIPExists = true + } else if addressType == v1.NodeInternalIP { + privateIPExists = true + } + } + + return publicIPExists && privateIPExists +} diff --git a/pkg/provisioner/provisioner.go b/pkg/provisioner/provisioner.go new file mode 100644 index 000000000..8b54c36f3 --- /dev/null +++ b/pkg/provisioner/provisioner.go @@ -0,0 +1,184 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provisioner + +import ( + "context" + "errors" + "fmt" + "strings" + "text/template" + "time" + + "github.com/sirupsen/logrus" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" +) + +const ( + maxRetrieForMachines = 5 + hostnameAnnotation = "ssh-username" + + userDataTemplate = `#cloud-config +ssh_pwauth: false + +{{- if .ProviderSpec.SSHPublicKeys }} +ssh_authorized_keys: +{{- range .ProviderSpec.SSHPublicKeys }} +- "{{ . }}" +{{- end }} +{{- end }} +` +) + +func getUserData(pconfig *providerconfigtypes.Config) (string, error) { + data := struct { + ProviderSpec *providerconfigtypes.Config + }{ + ProviderSpec: pconfig, + } + + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse user-data template: %w", err) + } + + var buf strings.Builder + if err := tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("failed to execute user-data template: %w", err) + } + + return userdatahelper.CleanupTemplateOutput(buf.String()) +} + +type MachineInstance struct { + inst instance.Instance + sshUser string +} + +func CreateMachines(ctx context.Context, machines []clusterv1alpha1.Machine) (*output, error) { + providerData := &cloudprovidertypes.ProviderData{ + Ctx: ctx, + ProvisionerMode: true, + } + + var instances []MachineInstance + + // TODO: Dump all the errors in an array and do the max that is possible without early exit + for _, machine := range machines { + prov, err := getProvider(ctx, machine) + if err != nil { + return nil, err + } + + machineCreated := false + providerInstance, err := prov.Get(ctx, &machine, providerData) + if err != nil { + // case 1: instance was not found and we are going to create one + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { + // Get userdata (needed to inject SSH keys to instances) + pconfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, fmt.Errorf("failed to get providerSpec: %w", err) + } + userdata, err := getUserData(pconfig) + if err != nil { + return nil, err + } + + // Create the instance + _, err = prov.Create(ctx, &machine, providerData, userdata) + if err != nil { + return nil, err + } + machineCreated = true + } else if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { + // case 2: terminal error was returned and manual interaction is required to recover + return nil, fmt.Errorf("failed to create machine at cloudprovider, due to %w", err) + } else { + // case 3: transient error was returned, requeue the request and try again in the future + return nil, fmt.Errorf("failed to get instance from provider: %w", err) + } + } + + if machineCreated { + for i := 0; i < maxRetrieForMachines; i++ { + providerInstance, err = prov.Get(ctx, &machine, providerData) + if err != nil { + return nil, err + } + + addresses := providerInstance.Addresses() + if len(addresses) > 0 && publicAndPrivateIPExist(addresses) { + break + } + logrus.Info("Waiting 5 seconds for machine address assignment.") + time.Sleep(5 * time.Second) + } + } + + // Instance exists + addresses := providerInstance.Addresses() + if len(addresses) == 0 { + return nil, fmt.Errorf("machine %s has not been assigned an IP yet", providerInstance.Name()) + } + + if machineCreated { + logrus.Infof("Machine %q was successfully created.", providerInstance.Name()) + } else { + logrus.Infof("Machine %q already exists.", providerInstance.Name()) + } + + sshUser := "root" + if machine.Annotations != nil { + if user := machine.Annotations[hostnameAnnotation]; sshUser != "" { + sshUser = user + } + } + + machineInstance := MachineInstance{ + inst: providerInstance, + sshUser: sshUser, + } + + instances = append(instances, machineInstance) + } + + output := getMachineProvisionerOutput(instances) + return &output, nil +} + +func getProvider(ctx context.Context, machine clusterv1alpha1.Machine) (cloudprovidertypes.Provider, error) { + providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, fmt.Errorf("failed to get provider config: %w", err) + } + skg := providerconfig.NewConfigVarResolver(ctx, nil) + prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) + if err != nil { + return nil, fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) + } + + return prov, nil +}