.*: replace tectonic node controller with machine config operator

config.tf

@@ -59,17 +59,18 @@ variable "tectonic_container_images" {
   default = {
     addon_resizer                        = "gcr.io/google_containers/addon-resizer:2.1"
     bootkube                             = "quay.io/coreos/bootkube:v0.10.0"
-    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
+    machine_config_operator              = "openshift/origin-machine-config-operator:v4.0.0"
     etcd_cert_signer                     = "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6"
     etcd                                 = "quay.io/coreos/etcd:v3.2.14"
-    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
-    kube_core_operator                   = "quay.io/coreos/kube-core-operator-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
-    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
-    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
+    hyperkube                            = "openshift/origin-node:latest"
+    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer-dev:0b97bdb8c663deab99a75cc0d6854ca240670320"
+    kube_core_operator                   = "quay.io/coreos/kube-core-operator-dev:0b97bdb8c663deab99a75cc0d6854ca240670320"
+    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator-dev:0b97bdb8c663deab99a75cc0d6854ca240670320"
+    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator-dev:0b97bdb8c663deab99a75cc0d6854ca240670320"
     tectonic_alm_operator                = "quay.io/coreos/tectonic-alm-operator:v0.3.1"
-    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
-    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
-    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator-dev:0a24db2288f00b10ced358d9643debd601ffd0f1"
+    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator-dev:0b97bdb8c663deab99a75cc0d6854ca240670320"
+    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator-dev:0b97bdb8c663deab99a75cc0d6854ca240670320"
+    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator-dev:0b97bdb8c663deab99a75cc0d6854ca240670320"
   }
 }
 

installer/pkg/config-generator/fixtures/kube-system.yaml

@@ -69,20 +69,6 @@ data:
     kind: TectonicNetworkOperatorConfig
     networkProfile: canal
     podCIDR: 10.2.0.0/16
-  tnco-config: |
-    apiVersion: v1
-    controllerConfig:
-      additionalConfigs: []
-      apiVersion: v1
-      baseDomain: cluster.com
-      cloudProviderConfig: ""
-      clusterDNSIP: 10.3.0.10
-      clusterName: test
-      etcdInitialCount: 3
-      kind: TectonicNodeControllerConfig
-      nodePoolUpdateLimit: null
-      platform: aws
-    kind: TectonicNodeControllerOperatorConfig
 kind: ConfigMap
 metadata:
   name: cluster-config-v1

installer/pkg/config-generator/generator.go

@@ -13,7 +13,6 @@ import (
 	"github.com/coreos/tectonic-config/config/kube-addon"
 	"github.com/coreos/tectonic-config/config/kube-core"
 	"github.com/coreos/tectonic-config/config/tectonic-network"
-	tnco "github.com/coreos/tectonic-config/config/tectonic-node-controller"
 	"github.com/coreos/tectonic-config/config/tectonic-utility"
 	"github.com/ghodss/yaml"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -65,10 +64,6 @@ func New(cluster config.Cluster) ConfigGenerator {
 
 // KubeSystem returns, if successful, a yaml string for the kube-system.
 func (c *ConfigGenerator) KubeSystem() (string, error) {
-	tncoConfig, err := c.tncoConfig()
-	if err != nil {
-		return "", err
-	}
 	coreConfig, err := c.coreConfig()
 	if err != nil {
 		return "", err
@@ -81,7 +76,6 @@ func (c *ConfigGenerator) KubeSystem() (string, error) {
 	return configMap("kube-system", genericData{
 		"kco-config":     coreConfig,
 		"network-config": c.networkConfig(),
-		"tnco-config":    tncoConfig,
 		"install-config": installConfig,
 	})
 }
@@ -211,15 +205,6 @@ func (c *ConfigGenerator) CoreConfig() (string, error) {
 	return marshalYAML(coreConfig)
 }
 
-// TncoConfig returns, if successful, a yaml string for the on-disk tnco-config.
-func (c *ConfigGenerator) TncoConfig() (string, error) {
-	tncoConfig, err := c.tncoConfig()
-	if err != nil {
-		return "", err
-	}
-	return marshalYAML(tncoConfig)
-}
-
 func (c *ConfigGenerator) addonConfig() (*kubeaddon.OperatorConfig, error) {
 	addonConfig := kubeaddon.OperatorConfig{
 		TypeMeta: metav1.TypeMeta{
@@ -284,38 +269,6 @@ func (c *ConfigGenerator) networkConfig() *tectonicnetwork.OperatorConfig {
 	return &networkConfig
 }
 
-func (c *ConfigGenerator) tncoConfig() (*tnco.OperatorConfig, error) {
-	tncoConfig := tnco.OperatorConfig{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: tnco.TNCOConfigAPIVersion,
-			Kind:       tnco.TNCOConfigKind,
-		},
-	}
-
-	tncoConfig.ControllerConfig = tnco.ControllerConfig{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: tnco.TNCConfigAPIVersion,
-			Kind:       tnco.TNCConfigKind,
-		},
-	}
-
-	cidrhost, err := cidrhost(c.Cluster.Networking.ServiceCIDR, 10)
-	if err != nil {
-		return nil, err
-	}
-
-	tncoConfig.ControllerConfig.ClusterDNSIP = cidrhost
-	tncoConfig.ControllerConfig.Platform = tectonicCloudProvider(c.Platform)
-	tncoConfig.ControllerConfig.CloudProviderConfig = "" // TODO(yifan): Get CloudProviderConfig.
-	tncoConfig.ControllerConfig.ClusterName = c.Cluster.Name
-	tncoConfig.ControllerConfig.BaseDomain = c.Cluster.BaseDomain
-	tncoConfig.ControllerConfig.EtcdInitialCount = c.Cluster.NodeCount(c.Cluster.Master.NodePools)
-	tncoConfig.ControllerConfig.AdditionalConfigs = []string{} // TODO(yifan): Get additional configs.
-	tncoConfig.ControllerConfig.NodePoolUpdateLimit = nil      // TODO(yifan): Get the node pool update limit.
-
-	return &tncoConfig, nil
-}
-
 func (c *ConfigGenerator) utilityConfig() (*tectonicutility.OperatorConfig, error) {
 	utilityConfig := tectonicutility.OperatorConfig{
 		TypeMeta: metav1.TypeMeta{

installer/pkg/config-generator/tls.go

@@ -42,8 +42,8 @@ const (
 	rootCAKeyPath                = "generated/tls/root-ca.key"
 	serviceServingCACertPath     = "generated/tls/service-serving-ca.crt"
 	serviceServingCAKeyPath      = "generated/tls/service-serving-ca.key"
-	tncCertPath                  = "generated/tls/tnc.crt"
-	tncKeyPath                   = "generated/tls/tnc.key"
+	machineConfigServerCertPath  = "generated/tls/machine-config-server.crt"
+	machineConfigServerKeyPath   = "generated/tls/machine-config-server.key"
 	serviceAccountPubkeyPath     = "generated/tls/service-account.pub"
 	serviceAccountPrivateKeyPath = "generated/tls/service-account.key"
 )
@@ -239,18 +239,18 @@ func (c *ConfigGenerator) GenerateTLSConfig(clusterDir string) error {
 		return fmt.Errorf("failed to generate kubelet certificate: %v", err)
 	}
 
-	// TNC certs
-	tncDomain := fmt.Sprintf("%s-tnc.%s", c.Name, c.BaseDomain)
+	// MachineConfigServer certs
+	mcsDomain := fmt.Sprintf("%s-tnc.%s", c.Name, c.BaseDomain)
 	cfg = &tls.CertCfg{
 		ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		DNSNames:     []string{tncDomain},
-		Subject:      pkix.Name{CommonName: tncDomain},
+		DNSNames:     []string{mcsDomain},
+		Subject:      pkix.Name{CommonName: mcsDomain},
 		Validity:     tls.ValidityTenYears,
 		IsCA:         false,
 	}
 
-	if _, _, err := generateCert(clusterDir, caKey, caCert, tncKeyPath, tncCertPath, cfg, false); err != nil {
-		return fmt.Errorf("failed to generate tnc certificate: %v", err)
+	if _, _, err := generateCert(clusterDir, caKey, caCert, machineConfigServerKeyPath, machineConfigServerCertPath, cfg, false); err != nil {
+		return fmt.Errorf("failed to generate machine-config-server certificate: %v", err)
 	}
 
 	// Cluster API cert

installer/pkg/workflow/utils.go

@@ -68,16 +68,6 @@ func generateClusterConfigMaps(m *metadata) error {
 		return err
 	}
 
-	tncoConfig, err := configGenerator.TncoConfig()
-	if err != nil {
-		return err
-	}
-
-	tncoConfigFilePath := filepath.Join(clusterGeneratedPath, tncoConfigFileName)
-	if err := ioutil.WriteFile(tncoConfigFilePath, []byte(tncoConfig), 0666); err != nil {
-		return err
-	}
-
 	kubeSystem, err := configGenerator.KubeSystem()
 	if err != nil {
 		return err

modules/bootkube/assets.tf

@@ -49,12 +49,12 @@ data "template_file" "bootkube_sh" {
   template = "${file("${path.module}/resources/bootkube.sh")}"
 
   vars {
-    bootkube_image           = "${var.container_images["bootkube"]}"
-    kube_core_renderer_image = "${var.container_images["kube_core_renderer"]}"
-    tnc_operator_image       = "${var.container_images["tnc_operator"]}"
-    etcd_cert_signer_image   = "${var.container_images["etcd_cert_signer"]}"
-    etcdctl_image            = "${var.container_images["etcd"]}"
-    etcd_cluster             = "${join(",", data.template_file.initial_cluster.*.rendered)}"
+    bootkube_image                = "${var.container_images["bootkube"]}"
+    kube_core_renderer_image      = "${var.container_images["kube_core_renderer"]}"
+    machine_config_operator_image = "${var.container_images["machine_config_operator"]}"
+    etcd_cert_signer_image        = "${var.container_images["etcd_cert_signer"]}"
+    etcdctl_image                 = "${var.container_images["etcd"]}"
+    etcd_cluster                  = "${join(",", data.template_file.initial_cluster.*.rendered)}"
   }
 }
 

modules/bootkube/manifests.tf

@@ -3,19 +3,22 @@ variable "manifest_names" {
     "01-tectonic-namespace.yaml",
     "02-ingress-namespace.yaml",
     "03-openshift-web-console-namespace.yaml",
+    "04-openshift-machine-config-operator.yaml",        # https://github.com/openshift/machine-config-operator/tree/master/install/00_namespace.yaml
     "app-version-kind.yaml",
     "app-version-tectonic-network.yaml",
-    "app-version-tnc.yaml",
+    "cluster-apiserver-secret.yaml",
     "kube-apiserver-secret.yaml",
     "kube-cloud-config.yaml",
     "kube-controller-manager-secret.yaml",
-    "node-config-kind.yaml",
+    "machine-config-operator-00-config-crd.yaml",       # https://github.com/openshift/machine-config-operator/tree/master/install/01_mcoconfig.crd.yaml
+    "machine-config-operator-01-images-configmap.yaml", # https://github.com/openshift/machine-config-operator/tree/master/install/02_images.configmap.yaml
+    "machine-config-operator-02-rbac.yaml",             # https://github.com/openshift/machine-config-operator/tree/master/install/03_rbac.yaml
+    "machine-config-operator-03-deployment.yaml",       # https://github.com/openshift/machine-config-operator/tree/master/install/04_deployment.yaml
+    "machine-config-server-tls-secret.yaml",
     "openshift-apiserver-secret.yaml",
-    "cluster-apiserver-secret.yaml",
     "pull.json",
     "tectonic-network-operator.yaml",
-    "tectonic-node-controller-operator.yaml",
-    "tnc-tls-secret.yaml",
+    "operatorstatus-crd.yaml",
   ]
 }
 
@@ -26,7 +29,7 @@ data "template_file" "manifest_file_list" {
 
   vars {
     tectonic_network_operator_image = "${var.container_images["tectonic_network_operator"]}"
-    tnc_operator_image              = "${var.container_images["tnc_operator"]}"
+    machine_config_operator_image   = "${var.container_images["machine_config_operator"]}"
 
     cloud_provider_config = "${var.cloud_provider_config}"
 
@@ -57,8 +60,8 @@ data "template_file" "manifest_file_list" {
     etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}"
     etcd_client_key  = "${base64encode(var.etcd_client_key_pem)}"
 
-    tnc_tls_cert = "${base64encode(var.tnc_cert_pem)}"
-    tnc_tls_key  = "${base64encode(var.tnc_key_pem)}"
+    mcs_tls_cert = "${base64encode(var.mcs_cert_pem)}"
+    mcs_tls_key  = "${base64encode(var.mcs_key_pem)}"
   }
 }
 

modules/bootkube/resources/bootkube.sh

@@ -11,20 +11,32 @@ echo "Rendering Kubernetes core manifests..."
     --config=/assets/kco-config.yaml \
     --output=/assets
 
-echo "Rendering TNC manifests..."
+echo "Rendering MCO manifests..."
 
 # shellcheck disable=SC2154
 /usr/bin/podman run \
     --user 0 \
     --volume "$PWD:/assets:z" \
-    "${tnc_operator_image}" \
-    --config=/assets/tnco-config.yaml \
-    --render-bootstrap=true \
-    --render-output=/assets/tnc-bootstrap
+    "${machine_config_operator_image}" \
+    bootstrap \
+    --etcd-ca=/assets/tls/etcd-client-ca.crt \
+    --root-ca=/assets/tls/root-ca.crt \
+    --config-file=/assets/manifests/cluster-config.yaml \
+    --dest-dir=/assets/mco-bootstrap \
+    --images-json-configmap=/assets/manifests/machine-config-operator-01-images-configmap.yaml
 
 mkdir -p /etc/kubernetes/manifests/
-cp "$PWD/tnc-bootstrap/tectonic-node-controller-pod.yaml" /etc/kubernetes/manifests/
-cp "$PWD/tnc-bootstrap/tectonic-node-controller-config.yaml" /etc/kubernetes/tnc-config
+mkdir -p /etc/mcc/bootstrap/
+mkdir -p /etc/ssl/mcs/
+# Bootstrap MachineConfigController uses /etc/mcc/bootstrap/manifests/ dir to
+# 1. read the controller config rendered by MachineConfigOperator
+# 2. read the default MachineConfigPools rendered by MachineConfigOperator
+# 3. read any additional MachineConfigs that are needed for the default MachineConfigPools.
+cp -r "$PWD/mco-bootstrap/manifests" /etc/mcc/bootstrap/manifests
+# /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets.
+cp "$PWD/tls/machine-config-server.crt" /etc/ssl/mcs/tls.crt
+cp "$PWD/tls/machine-config-server.key" /etc/ssl/mcs/tls.key
+cp "$PWD/mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml" /etc/kubernetes/manifests/
 
 # We originally wanted to run the etcd cert signer as
 # a static pod, but kubelet could't remove static pod
@@ -87,7 +99,7 @@ done
 echo "etcd cluster up. Killing etcd certificate signer..."
 
 /usr/bin/podman kill "$SIGNER"
-rm /etc/kubernetes/manifests/tectonic-node-controller-pod.yaml
+rm /etc/kubernetes/manifests/machineconfigoperator-bootstrap-pod.yaml
 
 cp -r "$PWD/bootstrap-configs" /etc/kubernetes/bootstrap-configs
 

modules/bootkube/resources/manifests/04-openshift-machine-config-operator.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openshift-machine-config-operator
+  labels:
+    name: openshift-machine-config-operator
+    openshift.io/run-level: "1"

modules/bootkube/resources/manifests/machine-config-operator-00-config-crd.yaml

@@ -0,0 +1,24 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: mcoconfigs.machineconfiguration.openshift.io
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: machineconfiguration.openshift.io
+  # list of versions supported by this CustomResourceDefinition
+  versions:
+    - name: v1
+      # Each version can be enabled/disabled by Served flag.
+      served: true
+      # One and only one version must be marked as the storage version.
+      storage: true
+  # either Namespaced or Cluster
+  scope: Namespaced
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: mcoconfigs
+    # singular name to be used as an alias on the CLI and for display
+    singular: mcoconfig
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: MCOConfig

modules/bootkube/resources/manifests/machine-config-operator-01-images-configmap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: machine-config-operator-images
+  namespace: openshift-machine-config-operator
+data:
+  images.json: '{"machineConfigController": "openshift/origin-machine-config-controller:v4.0.0", "machineConfigDaemon": "openshift/origin-machine-config-daemon:v4.0.0", "machineConfigServer": "openshift/origin-machine-config-server:v4.0.0"}'

modules/bootkube/resources/manifests/machine-config-operator-02-rbac.yaml

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: default-account-openshift-machine-config-operator
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: openshift-machine-config-operator
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io