Run against latest existinginfra with cluster driven repaving (#312)

* Work with new capei provider
* Fix wksctl os to correctly handle file and flux configuration
* switch to marking workload clusters in spec
* Set up machine info pool in wksctl to pass to controller in existinginfra
* Removes the requirement for a wks-controller.yaml
  The wks controller comes from the existinginfra provider.  We no longer
  need it in the repository when creating a cluster.
* inline files in cluster manifest for integration tests
* plumb inline ssh key through PEM file processing
* set version on cluster for apply_test
* Re-enables k8s 1.16 cluster support
* Plumb Context through 'plan' steps
  Doesn't do anything different as yet, but this will be useful for
  tracing, or for cancelling a command.
* point to new version of existinginfra and allow for the new context parameter in CreateSeedNodeSetupPlan
* store correct versions in local repo
* add explicit kubernetes version to multimaster test
* pin correct image in apply_test and multimaster_test

Co-authored-by: Jerry Jackson <jerry@weave.works>
Co-authored-by: Mark Emeis <mark.emeis@weave.works>
Co-authored-by: Simon Howe <simon@weave.works>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>

cmd/wksctl/apply/apply.go

@@ -1,22 +1,30 @@
 package apply
 
 import (
+	"bytes"
 	"context"
+	"io/ioutil"
 	"path/filepath"
 	"strings"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	existinginfrav1 "github.com/weaveworks/cluster-api-provider-existinginfra/apis/cluster.weave.works/v1alpha3"
 	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/apis/wksprovider/machine/config"
 	capeios "github.com/weaveworks/cluster-api-provider-existinginfra/pkg/apis/wksprovider/machine/os"
+	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/scheme"
+	capeispecs "github.com/weaveworks/cluster-api-provider-existinginfra/pkg/specs"
 	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/utilities/kubeadm"
+	"github.com/weaveworks/libgitops/pkg/serializer"
 	"github.com/weaveworks/wksctl/pkg/addons"
 	wksos "github.com/weaveworks/wksctl/pkg/apis/wksprovider/machine/os"
 	"github.com/weaveworks/wksctl/pkg/manifests"
 	"github.com/weaveworks/wksctl/pkg/plan/runners/ssh"
 	"github.com/weaveworks/wksctl/pkg/specs"
+	"github.com/weaveworks/wksctl/pkg/utilities"
 	"github.com/weaveworks/wksctl/pkg/utilities/manifest"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 )
 
 // Cmd represents the apply command
@@ -96,6 +104,22 @@ func (a *Applier) Apply(ctx context.Context) error {
 	return a.initiateCluster(ctx, clusterPath, machinesPath)
 }
 
+// parseCluster converts the manifest file into a Cluster
+func parseCluster(clusterManifest []byte) (c *clusterv1.Cluster, eic *existinginfrav1.ExistingInfraCluster, err error) {
+	return capeispecs.ParseCluster(ioutil.NopCloser(bytes.NewReader(clusterManifest)))
+}
+
+func unparseCluster(c *clusterv1.Cluster, eic *existinginfrav1.ExistingInfraCluster) ([]byte, error) {
+	var buf bytes.Buffer
+	s := serializer.NewSerializer(scheme.Scheme, nil)
+	fw := serializer.NewYAMLFrameWriter(&buf)
+	err := s.Encoder().Encode(fw, c, eic)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
 func (a *Applier) initiateCluster(ctx context.Context, clusterManifestPath, machinesManifestPath string) error {
 	sp := specs.NewFromPaths(clusterManifestPath, machinesManifestPath)
 	sshClient, err := ssh.NewClientForMachine(sp.MasterSpec, sp.ClusterSpec.User, a.Params.sshKeyPath, log.GetLevel() > log.InfoLevel)
@@ -156,31 +180,68 @@ func (a *Applier) initiateCluster(ctx context.Context, clusterManifestPath, mach
 		}
 	}
 
-	if err := wksos.SetupSeedNode(ctx, installer, wksos.SeedNodeParams{
+	clusterManifest, err := ioutil.ReadFile(clusterManifestPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to read cluster manifest: ")
+	}
+
+	// Read manifests and pass in the contents
+	cluster, eic, err := parseCluster(clusterManifest)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse cluster manifest: ")
+	}
+
+	eic.Spec.DeprecatedSSHKeyPath = a.Params.sshKeyPath
+	clusterManifest, err = unparseCluster(cluster, eic)
+	if err != nil {
+		return errors.Wrap(err, "failed to annotate cluster manifest: ")
+	}
+
+	machinesManifest, err := ioutil.ReadFile(machinesManifestPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to read machines manifest: ")
+	}
+
+	// Read sealed secret cert and key
+	var cert []byte
+	var key []byte
+	if utilities.FileExists(a.Params.sealedSecretCertPath) && utilities.FileExists(sealedSecretKeyPath) {
+		cert, err = ioutil.ReadFile(a.Params.sealedSecretCertPath)
+		if err != nil {
+			return errors.Wrap(err, "failed to read sealed secret certificate: ")
+		}
+
+		key, err = ioutil.ReadFile(sealedSecretKeyPath)
+		if err != nil {
+			return errors.Wrap(err, "failed to read sealed secret key: ")
+		}
+	}
+
+	if err := wksos.SetupSeedNode(installer, capeios.SeedNodeParams{
 		PublicIP:             sp.GetMasterPublicAddress(),
 		PrivateIP:            sp.GetMasterPrivateAddress(),
 		ServicesCIDRBlocks:   sp.Cluster.Spec.ClusterNetwork.Services.CIDRBlocks,
 		PodsCIDRBlocks:       sp.Cluster.Spec.ClusterNetwork.Pods.CIDRBlocks,
-		ClusterManifestPath:  clusterManifestPath,
-		MachinesManifestPath: machinesManifestPath,
-		SSHKeyPath:           a.Params.sshKeyPath,
+		ExistingInfraCluster: *eic,
+		ClusterManifest:      string(clusterManifest),
+		MachinesManifest:     string(machinesManifest),
 		BootstrapToken:       token,
 		KubeletConfig: config.KubeletConfig{
 			NodeIP:         sp.GetMasterPrivateAddress(),
 			CloudProvider:  sp.GetCloudProvider(),
 			ExtraArguments: sp.GetKubeletArguments(),
 		},
-		Controller: wksos.ControllerParams{
+		Controller: capeios.ControllerParams{
 			ImageOverride: controllerImage,
 		},
-		GitData: wksos.GitParams{
+		GitData: capeios.GitParams{
 			GitURL:           a.Params.gitURL,
 			GitBranch:        a.Params.gitBranch,
 			GitPath:          a.Params.gitPath,
 			GitDeployKeyPath: a.Params.gitDeployKeyPath,
 		},
-		SealedSecretKeyPath:  sealedSecretKeyPath,
-		SealedSecretCertPath: a.Params.sealedSecretCertPath,
+		SealedSecretKey:      string(key),
+		SealedSecretCert:     string(cert),
 		ConfigDirectory:      configDir,
 		ImageRepository:      sp.ClusterSpec.ImageRepository,
 		ControlPlaneEndpoint: sp.ClusterSpec.ControlPlaneEndpoint,

cmd/wksctl/applyaddons/applyaddons.go

@@ -9,6 +9,7 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	capeispecs "github.com/weaveworks/cluster-api-provider-existinginfra/pkg/specs"
 	"github.com/weaveworks/launcher/pkg/kubectl"
 	"github.com/weaveworks/wksctl/pkg/addons"
 	"github.com/weaveworks/wksctl/pkg/specs"
@@ -40,7 +41,7 @@ func init() {
 		&applyAddonsOptions.namespace, "namespace", manifest.DefaultNamespace, "namespace portion of kubeconfig path")
 }
 
-func applyAddonsUsingConfig(sp *specs.Specs, basePath, kubeconfig string) error {
+func applyAddonsUsingConfig(sp *capeispecs.Specs, basePath, kubeconfig string) error {
 	fmt.Println("==> Applying addons (2)")
 
 	for _, addonDesc := range sp.ClusterSpec.Addons {

cmd/wksctl/init/init.go

@@ -11,17 +11,15 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/pelletier/go-toml"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	wksos "github.com/weaveworks/wksctl/pkg/apis/wksprovider/machine/os"
+	capeios "github.com/weaveworks/cluster-api-provider-existinginfra/pkg/apis/wksprovider/machine/os"
 	"github.com/weaveworks/wksctl/pkg/specs"
 	"github.com/weaveworks/wksctl/pkg/utilities/manifest"
 	"github.com/weaveworks/wksctl/pkg/version"
 )
 
-// A command that initializes a user's cloned git repository with a correct image tag for wks-controller and
-// updated git information for flux manifests.
+// A command that initializes a user's cloned git repository with updated git information for flux manifests.
 
 type initOptionType struct {
 	dependencyPath       string
@@ -46,7 +44,7 @@ var (
 	Cmd = &cobra.Command{
 		Use:          "init",
 		Short:        "Update stored kubernetes manifests to match the local cluster environment",
-		Long:         "'wksctl init' configures existing kubernetes 'flux.yaml', 'wks-controller.yaml' and 'weave-net.yaml' manifests in a repository with information about the local GitOps repository, the preferred weave system namespace, and current container image tags. The files can be anywhere in the repository. If either file is absent, 'wksctl init' will return an error.",
+		Long:         "'wksctl init' configures existing kubernetes 'flux.yaml', and 'weave-net.yaml' manifests in a repository with information about the local GitOps repository, the preferred weave system namespace, and current container image tags. The files can be anywhere in the repository. If either file is absent, 'wksctl init' will return an error.",
 		Example:      "wksctl init --namespace=wksctl --git-url=git@github.com:haskellcurry/lambda.git --git-branch=development --git-path=src",
 		RunE:         initRun,
 		SilenceUsage: true,
@@ -56,18 +54,14 @@ var (
 
 	namespacePrefixPattern = "kind: Namespace\n  metadata:\n    name: "
 	namespaceNamePattern   = multiLineRegexp(namespacePrefixPattern + `\S+`)
-	controllerImageSegment = multiLineRegexp(`(image:\s*\S*[-]controller)(:\s*\S+)?`)
 	namespacePattern       = multiLineRegexp(`namespace:\s*\S+`)
 	gitURLPattern          = multiLineRegexp(`(--git-url)=\S+`)
 	gitBranchPattern       = multiLineRegexp(`(--git-branch)=\S+`)
 	gitPathPattern         = multiLineRegexp(`(--git-path)=\S+`)
 
 	updates = []manifestUpdate{
 		{name: "weave-net", selector: equal("weave-net.yaml"), updater: updateWeaveNetManifests},
-		{name: "wks-controller", selector: equal("wks-controller.yaml"), updater: updateControllerManifests},
 		{name: "flux", selector: and(prefix("flux"), extension("yaml")), updater: updateFluxManifests}}
-
-	dependencies = &toml.Tree{}
 )
 
 func multiLineRegexp(pattern string) *regexp.Regexp {
@@ -84,8 +78,6 @@ func init() {
 	Cmd.Flags().StringVar(&initOptions.gitPath, "git-path", ".", "Relative path to files in Git")
 	Cmd.Flags().StringVar(
 		&initOptions.namespace, "namespace", manifest.DefaultNamespace, "namespace portion of kubeconfig path")
-	Cmd.Flags().StringVar(
-		&initOptions.version, "controller-version", version.Version, "version of wks-controller to use")
 	Cmd.Flags().StringVar(&initOptions.clusterManifestPath, "cluster", "cluster.yaml", "Location of cluster manifest")
 	Cmd.Flags().StringVar(&initOptions.machinesManifestPath, "machines", "machines.yaml", "Location of machines manifest")
 	Cmd.Flags().StringVar(
@@ -126,20 +118,6 @@ func updatedArg(item string) []byte {
 	return []byte(fmt.Sprintf("$1=%s", item))
 }
 
-func updateControllerManifests(contents []byte, options initOptionType) ([]byte, error) {
-	controllerVersion, ok := dependencies.Get("controller.version").(string)
-	if !ok {
-		controllerVersion = options.version
-	}
-	var withVersion []byte
-	if strings.Contains(controllerVersion, "/") {
-		withVersion = controllerImageSegment.ReplaceAll(contents, []byte(`image: `+controllerVersion))
-	} else {
-		withVersion = controllerImageSegment.ReplaceAll(contents, []byte(`$1:`+controllerVersion))
-	}
-	return withVersion, nil
-}
-
 func updateWeaveNetManifests(contents []byte, options initOptionType) ([]byte, error) {
 	clusterManifestPath := (path.Join(options.localRepoDirectory, options.clusterManifestPath))
 	machinesManifestPath := (path.Join(options.localRepoDirectory, options.machinesManifestPath))
@@ -149,7 +127,7 @@ func updateWeaveNetManifests(contents []byte, options initOptionType) ([]byte, e
 	if len(podsCIDRBlocks) > 0 && podsCIDRBlocks[0] != "" {
 		// setting the pod CIDR block is currently only supported for the weave-net CNI
 		log.Debug("Updating weave-net manifest.")
-		manifests, err := wksos.SetWeaveNetPodCIDRBlock([][]byte{contents}, podsCIDRBlocks[0])
+		manifests, err := capeios.SetWeaveNetPodCIDRBlock([][]byte{contents}, podsCIDRBlocks[0])
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to inject ipalloc_range")
 		}
@@ -200,8 +178,8 @@ func updateManifests(options initOptionType) error {
 			}
 			return nil
 		})
-	if !found["flux"] || !found["wks-controller"] {
-		return errors.New("Both 'flux.yaml' and 'wks-controller.yaml' must be present in the repository")
+	if !found["flux"] {
+		return errors.New("'flux.yaml' must be present in the repository")
 	}
 	return err
 }
@@ -210,13 +188,5 @@ func initRun(cmd *cobra.Command, args []string) error {
 	if initOptions.version == "" {
 		initOptions.version = version.Version // from main command
 	}
-	bytes, err := ioutil.ReadFile(initOptions.dependencyPath)
-	if err != nil {
-		return err
-	}
-	dependencies, err = toml.Load(string(bytes))
-	if err != nil {
-		return err
-	}
 	return updateManifests(initOptions)
 }

cmd/wksctl/init/init_test.go

@@ -173,125 +173,3 @@ func TestFluxTranslate(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, string(res), fluxOutputs)
 }
-
-const controllerInputs = `
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: wks-controller
-  namespace: system
-  labels:
-    name: wks-controller
-    control-plane: wks-controller
-    controller-tools.k8s.io: "1.0"
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      name: wks-controller
-  template:
-    metadata:
-      labels:
-        name: wks-controller
-        control-plane: wks-controller
-        controller-tools.k8s.io: "1.0"
-    spec:
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      # Allow scheduling on master nodes. This is required because during
-      # bootstrapping of the cluster, we may initially have just one master,
-      # and would then need to deploy this controller there to set the entire
-      # cluster up.
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-        operator: Exists
-      # Mark this as a critical addon:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      # Only schedule on nodes which are ready and reachable:
-      - effect: NoExecute
-        key: node.alpha.kubernetes.io/notReady
-        operator: Exists
-      - effect: NoExecute
-        key: node.alpha.kubernetes.io/unreachable
-        operator: Exists
-      containers:
-      - name: controller
-        image: docker.io/weaveworks/wksctl-controller:version1.2.3
-        command:
-        - /bin/controller
-        - --verbose
-        resources:
-          limits:
-            cpu: 100m
-            memory: 30Mi
-          requests:
-            cpu: 100m
-            memory: 20Mi
-`
-const controllerOutputs = `
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: wks-controller
-  namespace: system
-  labels:
-    name: wks-controller
-    control-plane: wks-controller
-    controller-tools.k8s.io: "1.0"
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      name: wks-controller
-  template:
-    metadata:
-      labels:
-        name: wks-controller
-        control-plane: wks-controller
-        controller-tools.k8s.io: "1.0"
-    spec:
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      # Allow scheduling on master nodes. This is required because during
-      # bootstrapping of the cluster, we may initially have just one master,
-      # and would then need to deploy this controller there to set the entire
-      # cluster up.
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-        operator: Exists
-      # Mark this as a critical addon:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      # Only schedule on nodes which are ready and reachable:
-      - effect: NoExecute
-        key: node.alpha.kubernetes.io/notReady
-        operator: Exists
-      - effect: NoExecute
-        key: node.alpha.kubernetes.io/unreachable
-        operator: Exists
-      containers:
-      - name: controller
-        image: docker.io/weaveworks/wksctl-controller:version1.2.3
-        command:
-        - /bin/controller
-        - --verbose
-        resources:
-          limits:
-            cpu: 100m
-            memory: 30Mi
-          requests:
-            cpu: 100m
-            memory: 20Mi
-`
-
-func TestControllerTranslate(t *testing.T) {
-	res, err := updateControllerManifests([]byte(controllerInputs),
-		initOptionType{
-			version: "version1.2.3",
-		})
-	assert.NoError(t, err)
-	assert.Equal(t, string(res), controllerOutputs)
-}