Add UserData plugins (#473)

* Initial outline of userdata plugin handling

* Add head comment

* Start with plugin registration process

* Change discovery strategy of plugins

* More robust start of the plugin manager and change to net/rpc

* Add plugin counterpart for the userdata manager

* Minor fixes in manager and plugin, migrate CentOS plugin

* Fix comment

* Add test for CentOS userdata plugin

* Migrate CoreOS userdata provider to new plugin

* Add Ubuntu UserData plugin and do some harmonization between implementations

* Remove old providers

* Finish userdata and plugins, dependencies and Makefile still missing

* Extract userdata os configuration into own package

* More compatible OS configuration changes

* Fixes and adding plugins to Makefile

* Add check for already running plugins

* Add deploying of plugins

* Fix missing plugin copy in Dockerfile

* Test with explicit plugin names

* Fix linter troubles

* Some more handling of linter and deployment feedback

* Fix error in plugin find function

* Be more chatty when looking for plugin

* Test plugin deployment

* Fix wrong placement of test

* Change plugin loading to lazy mode

* Copy plugins before e2e tests

* Check /usr/local/bin

* Temporary change loading for test that locally works

* Change plugins from daemons to individual command calls

* Fix error formatting and move loading for debugging

* Enhance plugin search with ENV variable and add some debugging output

* Correctly propagate userdata plugin output

* Dont try to marshal an interface

* Update tests

* Fix review comments

* Add required ignition wrapping to coreos userdata provider

* Rename plugin API package

* Handle review comments

* Some more simplifications

* Code reorg after review

* Code reorg after review and discussion

* Handle format errors

* Change of import blocks to std, 3rd party, k8s, loodse

* Change indent (dunno why it changes before)

* Final fixes after review

.gitignore

@@ -1,5 +1,5 @@
 .idea
-machine-controller
+machine-controller*
 .terraform
 terraform.tfstate
 terraform.tfstate.backup
@@ -12,3 +12,6 @@ examples/*.pem
 examples/*.csr
 examples/*.srl
 /webhook
+# As long as we don't support modules.
+go.mod
+go.sum

Dockerfile

@@ -2,7 +2,6 @@ FROM alpine:3.7
 
 RUN apk add --no-cache ca-certificates cdrkit
 
-COPY machine-controller /usr/local/bin
-COPY webhook /usr/local/bin
+COPY machine-controller machine-controller-userdata-* webhook /usr/local/bin/
 
 USER nobody

Makefile

@@ -33,6 +33,18 @@ machine-controller: $(shell find cmd pkg -name '*.go') vendor
 		-ldflags '-s -w' \
 		-o machine-controller \
 		github.com/kubermatic/machine-controller/cmd/controller
+	go build -v \
+		-ldflags '-s -w' \
+		-o machine-controller-userdata-centos \
+		github.com/kubermatic/machine-controller/cmd/userdata/centos
+	go build -v \
+		-ldflags '-s -w' \
+		-o machine-controller-userdata-coreos \
+		github.com/kubermatic/machine-controller/cmd/userdata/coreos
+	go build -v \
+		-ldflags '-s -w' \
+		-o machine-controller-userdata-ubuntu \
+		github.com/kubermatic/machine-controller/cmd/userdata/ubuntu
 
 webhook: $(shell find cmd pkg -name '*.go') vendor
 	go build -v \

cmd/userdata/centos/main.go

@@ -0,0 +1,30 @@
+//
+// UserData plugin for CentOS.
+//
+
+package main
+
+import (
+	"flag"
+
+	"github.com/golang/glog"
+
+	"github.com/kubermatic/machine-controller/pkg/userdata/centos"
+	userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin"
+)
+
+func main() {
+	// Parse flags.
+	var debug bool
+
+	flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging")
+	flag.Parse()
+
+	// Instantiate provider and start plugin.
+	var provider = &centos.Provider{}
+	var p = userdataplugin.New(provider, debug)
+
+	if err := p.Run(); err != nil {
+		glog.Fatalf("error running CentOS plugin: %v", err)
+	}
+}

cmd/userdata/coreos/main.go

@@ -0,0 +1,31 @@
+//
+// UserData plugin for CoreOS.
+//
+
+package main
+
+import (
+	"flag"
+
+	"github.com/golang/glog"
+
+	"github.com/kubermatic/machine-controller/pkg/userdata/convert"
+	"github.com/kubermatic/machine-controller/pkg/userdata/coreos"
+	userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin"
+)
+
+func main() {
+	// Parse flags.
+	var debug bool
+
+	flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging")
+	flag.Parse()
+
+	// Instantiate provider and start plugin.
+	var provider = &coreos.Provider{}
+	var p = userdataplugin.New(convert.NewIgnition(provider), debug)
+
+	if err := p.Run(); err != nil {
+		glog.Fatalf("error running CoreOS plugin: %v", err)
+	}
+}

cmd/userdata/ubuntu/main.go

@@ -0,0 +1,30 @@
+//
+// UserData plugin for Ubuntu.
+//
+
+package main
+
+import (
+	"flag"
+
+	"github.com/golang/glog"
+
+	userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin"
+	"github.com/kubermatic/machine-controller/pkg/userdata/ubuntu"
+)
+
+func main() {
+	// Parse flags.
+	var debug bool
+
+	flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging")
+	flag.Parse()
+
+	// Instantiate provider and start plugin.
+	var provider = &ubuntu.Provider{}
+	var p = userdataplugin.New(provider, debug)
+
+	if err := p.Run(); err != nil {
+		glog.Fatalf("error running Ubuntu plugin: %v", err)
+	}
+}

cmd/webhook/main.go

@@ -4,11 +4,11 @@ import (
 	"flag"
 
 	"github.com/golang/glog"
-
-	"github.com/kubermatic/machine-controller/pkg/admission"
-
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/kubermatic/machine-controller/pkg/admission"
+	userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager"
 )
 
 var (
@@ -43,7 +43,12 @@ func main() {
 		glog.Fatalf("error building kubernetes clientset for kubeClient: %v", err)
 	}
 
-	s := admission.New(admissionListenAddress, kubeClient)
+	um, err := userdatamanager.New()
+	if err != nil {
+		glog.Fatalf("error initialising userdata plugins: %v", err)
+	}
+
+	s := admission.New(admissionListenAddress, kubeClient, um)
 	if err := s.ListenAndServeTLS(admissionTLSCertPath, admissionTLSKeyPath); err != nil {
 		glog.Fatalf("Failed to start server: %v", err)
 	}

hack/ci-e2e-test.sh

@@ -34,6 +34,11 @@ mv kubectl /usr/local/bin
 echo "Building machine-controller and webhook"
 make machine-controller webhook
 
+# Copy individual plugins with success control.
+echo "Copying machine-controller plugins"
+cp machine-controller-userdata-* /usr/local/bin
+ls -l /usr/local/bin
+
 # Generate ssh keypair
 echo "Generating ssh keypair"
 chmod 0700 $HOME/.ssh

pkg/admission/admission.go

@@ -16,17 +16,23 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
+
+	userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager"
 )
 
 type admissionData struct {
-	coreClient kubernetes.Interface
+	coreClient      kubernetes.Interface
+	userDataManager *userdatamanager.Manager
 }
 
 var jsonPatch = admissionv1beta1.PatchTypeJSONPatch
 
-func New(listenAddress string, coreClient kubernetes.Interface) *http.Server {
+func New(listenAddress string, coreClient kubernetes.Interface, um *userdatamanager.Manager) *http.Server {
 	m := http.NewServeMux()
-	ad := &admissionData{coreClient: coreClient}
+	ad := &admissionData{
+		coreClient:      coreClient,
+		userDataManager: um,
+	}
 	m.HandleFunc("/machinedeployments", handleFuncFactory(ad.mutateMachineDeployments))
 	m.HandleFunc("/machines", handleFuncFactory(ad.mutateMachines))
 	m.HandleFunc("/healthz", healthZHandler)

pkg/admission/machines.go

@@ -6,15 +6,13 @@ import (
 
 	"github.com/golang/glog"
 
-	clusterv1alpha1conversions "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/conversions"
-	"github.com/kubermatic/machine-controller/pkg/cloudprovider"
-	"github.com/kubermatic/machine-controller/pkg/providerconfig"
-	"github.com/kubermatic/machine-controller/pkg/userdata"
-
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
-
 	clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
+
+	clusterv1alpha1conversions "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/conversions"
+	"github.com/kubermatic/machine-controller/pkg/cloudprovider"
+	"github.com/kubermatic/machine-controller/pkg/providerconfig"
 )
 
 // BypassSpecNoModificationRequirementAnnotation is used to bypass the "no machine.spec modification" allowed
@@ -92,8 +90,8 @@ func (ad *admissionData) defaultAndValidateMachineSpec(spec *clusterv1alpha1.Mac
 		return fmt.Errorf("failed to get cloud provider %q: %v", providerConfig.CloudProvider, err)
 	}
 
-	// Verify operating system
-	if _, err := userdata.ForOS(providerConfig.OperatingSystem); err != nil {
+	// Verify operating system.
+	if _, err := ad.userDataManager.ForOS(providerConfig.OperatingSystem); err != nil {
 		return fmt.Errorf("failed to get OS '%s': %v", providerConfig.OperatingSystem, err)
 	}
 

pkg/apis/plugin/plugin.go

@@ -0,0 +1,43 @@
+//
+// Environment and serialisation types for UserData plugins.
+//
+
+package plugin
+
+import (
+	"net"
+
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
+)
+
+const (
+	// EnvUserDataRequest names the environment variable containing
+	// the user data request.
+	EnvUserDataRequest = "MACHINE_CONTROLLER_USER_DATA_REQUEST"
+
+	// EnvPluginDir names the environment variable containing
+	// a user defined location of the plugins.
+	EnvPluginDir = "MACHINE_CONTROLLER_USERDATA_PLUGIN_DIR"
+)
+
+// UserDataRequest requests user data with the given arguments.
+type UserDataRequest struct {
+	MachineSpec           clusterv1alpha1.MachineSpec
+	KubeConfig            *clientcmdapi.Config
+	CloudProviderName     string
+	CloudConfig           string
+	DNSIPs                []net.IP
+	ExternalCloudProvider bool
+}
+
+// UserDataResponse contains the responded user data.
+type UserDataResponse struct {
+	UserData string
+	Err      string
+}
+
+// ErrorResponse contains a single responded error.
+type ErrorResponse struct {
+	Err string
+}

pkg/controller/machine/machine.go

@@ -25,13 +25,6 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/heptiolabs/healthcheck"
-	"github.com/kubermatic/machine-controller/pkg/cloudprovider"
-	"github.com/kubermatic/machine-controller/pkg/cloudprovider/cloud"
-	cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors"
-	"github.com/kubermatic/machine-controller/pkg/cloudprovider/instance"
-	"github.com/kubermatic/machine-controller/pkg/node/eviction"
-	"github.com/kubermatic/machine-controller/pkg/providerconfig"
-	"github.com/kubermatic/machine-controller/pkg/userdata"
 	"github.com/prometheus/client_golang/prometheus"
 
 	corev1 "k8s.io/api/core/v1"
@@ -53,12 +46,20 @@ import (
 	"k8s.io/client-go/tools/reference"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/client-go/util/workqueue"
-
 	"sigs.k8s.io/cluster-api/pkg/apis/cluster/common"
 	clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	clusterv1alpha1clientset "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
 	machinescheme "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/scheme"
 	clusterlistersv1alpha1 "sigs.k8s.io/cluster-api/pkg/client/listers_generated/cluster/v1alpha1"
+
+	"github.com/kubermatic/machine-controller/pkg/cloudprovider"
+	"github.com/kubermatic/machine-controller/pkg/cloudprovider/cloud"
+	cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors"
+	"github.com/kubermatic/machine-controller/pkg/cloudprovider/instance"
+	"github.com/kubermatic/machine-controller/pkg/node/eviction"
+	"github.com/kubermatic/machine-controller/pkg/providerconfig"
+	userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager"
+	userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin"
 )
 
 const (
@@ -93,6 +94,7 @@ type Controller struct {
 	kubeconfigProvider KubeconfigProvider
 
 	machineCreateDeleteData *cloud.MachineCreateDeleteData
+	userDataManager         *userdatamanager.Manager
 
 	joinClusterTimeout *time.Duration
 
@@ -114,7 +116,7 @@ type MetricsCollection struct {
 	Errors  prometheus.Counter
 }
 
-// NewMachineController returns a new machine controller
+// NewMachineController returns a new machine controller.
 func NewMachineController(
 	kubeClient kubernetes.Interface,
 	machineClient clusterv1alpha1clientset.Interface,
@@ -173,6 +175,12 @@ func NewMachineController(
 		PVLister: pvLister,
 	}
 
+	m, err := userdatamanager.New()
+	if err != nil {
+		return nil, err
+	}
+	controller.userDataManager = m
+
 	machineInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: controller.enqueueMachine,
 		UpdateFunc: func(old, new interface{}) {
@@ -418,15 +426,15 @@ func (c *Controller) syncHandler(key string) error {
 		return c.deleteMachine(prov, machine)
 	}
 
-	// step 3: essentially creates an instance for the given machine
-	userdataProvider, err := userdata.ForOS(providerConfig.OperatingSystem)
+	// Step 3: Essentially creates an instance for the given machine.
+	userdataPlugin, err := c.userDataManager.ForOS(providerConfig.OperatingSystem)
 	if err != nil {
 		return fmt.Errorf("failed to userdata provider for '%s': %v", providerConfig.OperatingSystem, err)
 	}
 
 	// case 3.2: creates an instance if there is no node associated with the given machine
 	if machine.Status.NodeRef == nil {
-		return c.ensureInstanceExistsForMachine(prov, machine, userdataProvider, providerConfig)
+		return c.ensureInstanceExistsForMachine(prov, machine, userdataPlugin, providerConfig)
 	}
 
 	node, err := c.getNodeByNodeRef(machine.Status.NodeRef)
@@ -450,7 +458,7 @@ func (c *Controller) syncHandler(key string) error {
 		}
 	} else {
 		// Node is not ready anymore? Maybe it got deleted
-		return c.ensureInstanceExistsForMachine(prov, machine, userdataProvider, providerConfig)
+		return c.ensureInstanceExistsForMachine(prov, machine, userdataPlugin, providerConfig)
 	}
 
 	// case 3.3: if the node exists make sure if it has labels and taints attached to it.
@@ -570,7 +578,7 @@ func (c *Controller) deleteNodeForMachine(machine *clusterv1alpha1.Machine) erro
 	return err
 }
 
-func (c *Controller) ensureInstanceExistsForMachine(prov cloud.Provider, machine *clusterv1alpha1.Machine, userdataProvider userdata.Provider, providerConfig *providerconfig.Config) error {
+func (c *Controller) ensureInstanceExistsForMachine(prov cloud.Provider, machine *clusterv1alpha1.Machine, userdataPlugin userdataplugin.Provider, providerConfig *providerconfig.Config) error {
 	glog.V(6).Infof("Requesting instance for machine '%s' from cloudprovider because no associated node with status ready found...", machine.Name)
 
 	providerInstance, err := prov.Get(machine)
@@ -588,7 +596,11 @@ func (c *Controller) ensureInstanceExistsForMachine(prov cloud.Provider, machine
 				return fmt.Errorf("failed to create bootstrap kubeconfig: %v", err)
 			}
 
-			userdata, err := userdataProvider.UserData(machine.Spec, kubeconfig, prov, c.clusterDNSIPs, c.externalCloudProvider)
+			cloudConfig, cloudProviderName, err := prov.GetCloudConfig(machine.Spec)
+			if err != nil {
+				return fmt.Errorf("failed to render cloud config: %v", err)
+			}
+			userdata, err := userdataPlugin.UserData(machine.Spec, kubeconfig, cloudConfig, cloudProviderName, c.clusterDNSIPs, c.externalCloudProvider)
 			if err != nil {
 				c.recorder.Eventf(machine, corev1.EventTypeWarning, "UserdataRenderingFailed", "Userdata rendering failed: %v", err)
 				return fmt.Errorf("failed get userdata: %v", err)