templates: Migrate from Container Linux to Fedora CoreOS config format

go.mod

@@ -10,14 +10,12 @@ require (
 	github.com/Masterminds/sprig v2.20.0+incompatible
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
-	github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/apparentlymart/go-cidr v1.0.0
 	github.com/ashcrow/osrelease v0.0.0-20180626175927-9b292693c55c
 	github.com/clarketm/json v1.14.1
 	github.com/containers/image v3.0.2+incompatible
 	github.com/containers/storage v1.13.5
-	github.com/coreos/container-linux-config-transpiler v0.9.0
+	github.com/coreos/fcct v0.5.0
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/coreos/ign-converter v0.0.0-20200228175238-237c8512310a
 	github.com/coreos/ignition v0.35.0
@@ -69,7 +67,6 @@ require (
 	github.com/xeipuuv/gojsonschema v1.1.0 // indirect
 	golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0
 	google.golang.org/appengine v1.6.1 // indirect
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	k8s.io/api v0.18.0
 	k8s.io/apiextensions-apiserver v0.18.0
 	k8s.io/apimachinery v0.18.0

go.sum

@@ -54,13 +54,9 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 h1:uwcvnXW76Y0rHM+qs7y8iHknWUWXYFNlD6FEVhc47TU=
 github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c=
-github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo=
-github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
@@ -101,10 +97,10 @@ github.com/containers/storage v1.13.5 h1:/SUzGeOP2HDijpF7Yur21Ch6WTZC1BNeZF917CW
 github.com/containers/storage v1.13.5/go.mod h1:HELz8Sn+UVbPaUZMI8RvIG9doD4y4z6Gtg4k7xdd2ZY=
 github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U=
-github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/fcct v0.5.0 h1:f/z+MCoR2vULes+MyoPEApQ6iluy/JbXoRi6dahPItQ=
+github.com/coreos/fcct v0.5.0/go.mod h1:cbE+j77YSQwFB2fozWVB3qsI2Pi3YiVEbDz/b6Yywdo=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -131,6 +127,8 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068 h1:y2aHj7QqyAJ6YBBONTAr17YxHHiogDkYnTsJvFNhxwY=
 github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE=
+github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5 h1:DjoHHi6+9J7DGYPvBdmszKZLY+ucx2bnA77jf8KIk9M=
+github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -908,6 +906,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v0.0.0-20190624233834-05ebafbffc79/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=

pkg/controller/common/helpers.go

@@ -1,11 +1,13 @@
 package common
 
 import (
+	"fmt"
 	"io/ioutil"
 	"reflect"
 	"sort"
 
 	"github.com/clarketm/json"
+	fcctbase "github.com/coreos/fcct/base/v0_1"
 	ignconverter "github.com/coreos/ign-converter"
 	ign2error "github.com/coreos/ignition/config/shared/errors"
 	ign "github.com/coreos/ignition/config/v2_2"
@@ -14,6 +16,7 @@ import (
 	ign3 "github.com/coreos/ignition/v2/config/v3_0"
 	ign3types "github.com/coreos/ignition/v2/config/v3_0/types"
 	validate3 "github.com/coreos/ignition/v2/config/validate"
+	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -218,3 +221,43 @@ func IgnParseWrapper(rawIgn []byte) (ignconfig interface{}, err error) {
 	}
 	return ign2types.Config{}, errors.Errorf("parsing Ignition config failed with error: %v\nReport: %v", err, rpt)
 }
+
+// TranspileCoreOSConfigToIgn transpiles Fedora CoreOS config to ignition
+// internally it transpiles to Ign spec v3 config and translates to spec v2
+func TranspileCoreOSConfigToIgn(files, units []string) (*ign2types.Config, error) {
+	var ctCfg fcctbase.Config
+	overwrite := true
+	// Convert data to Ignition resources
+	for _, d := range files {
+		f := new(fcctbase.File)
+		if err := yaml.Unmarshal([]byte(d), f); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal file into struct: %v", err)
+		}
+		f.Overwrite = &overwrite
+
+		// Add the file to the config
+		ctCfg.Storage.Files = append(ctCfg.Storage.Files, *f)
+	}
+
+	for _, d := range units {
+		u := new(fcctbase.Unit)
+		if err := yaml.Unmarshal([]byte(d), u); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal systemd unit into struct: %v", err)
+		}
+
+		// Add the unit to the config
+		ctCfg.Systemd.Units = append(ctCfg.Systemd.Units, *u)
+	}
+
+	ign3Cfg, tSet, err := ctCfg.ToIgn3_0()
+	if err != nil {
+		return nil, fmt.Errorf("failed to transpile config to Ignition config %s\nTranslation set: %v", err, tSet)
+	}
+
+	convertedIgnCfgV2, errV3 := convertIgnition3to2(ign3Cfg)
+	if errV3 != nil {
+		return nil, errors.Errorf("converting Ignition spec v3 config to v2 failed with error: %v", errV3)
+	}
+
+	return &convertedIgnCfgV2, nil
+}

pkg/controller/template/render.go

@@ -13,10 +13,6 @@ import (
 
 	"github.com/Masterminds/sprig"
 	"github.com/clarketm/json"
-	ctconfig "github.com/coreos/container-linux-config-transpiler/config"
-	cttypes "github.com/coreos/container-linux-config-transpiler/config/types"
-	igntypes "github.com/coreos/ignition/config/v2_2/types"
-	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
@@ -266,9 +262,9 @@ func generateMachineConfigForName(config *RenderConfig, role, name, templateDir,
 		return vs
 	}
 
-	ignCfg, err := transpileToIgn(keySortVals(files), keySortVals(units))
+	ignCfg, err := ctrlcommon.TranspileCoreOSConfigToIgn(keySortVals(files), keySortVals(units))
 	if err != nil {
-		return nil, fmt.Errorf("error transpiling ct config to Ignition config: %v", err)
+		return nil, fmt.Errorf("error transpiling CoreOS config to Ignition config: %v", err)
 	}
 	mcfg, err := MachineConfigFromIgnConfig(role, name, ignCfg)
 	if err != nil {
@@ -303,38 +299,6 @@ func MachineConfigFromIgnConfig(role, name string, ignCfg interface{}) (*mcfgv1.
 	}, nil
 }
 
-func transpileToIgn(files, units []string) (*igntypes.Config, error) {
-	var ctCfg cttypes.Config
-
-	// Convert data to Ignition resources
-	for _, d := range files {
-		f := new(cttypes.File)
-		if err := yaml.Unmarshal([]byte(d), f); err != nil {
-			return nil, fmt.Errorf("failed to unmarshal file into struct: %v", err)
-		}
-
-		// Add the file to the config
-		ctCfg.Storage.Files = append(ctCfg.Storage.Files, *f)
-	}
-
-	for _, d := range units {
-		u := new(cttypes.SystemdUnit)
-		if err := yaml.Unmarshal([]byte(d), u); err != nil {
-			return nil, fmt.Errorf("failed to unmarshal systemd unit into struct: %v", err)
-		}
-
-		// Add the unit to the config
-		ctCfg.Systemd.Units = append(ctCfg.Systemd.Units, *u)
-	}
-
-	ignCfg, rep := ctconfig.Convert(ctCfg, "", nil)
-	if rep.IsFatal() {
-		return nil, fmt.Errorf("failed to convert config to Ignition config %s", rep)
-	}
-
-	return &ignCfg, nil
-}
-
 // renderTemplate renders a template file with values from a RenderConfig
 // returns the rendered file data
 func renderTemplate(config RenderConfig, path string, b []byte) ([]byte, error) {

pkg/controller/template/render_test.go

@@ -240,10 +240,7 @@ func TestSkipMissing(t *testing.T) {
 	}
 }
 
-const (
-	templateDir = "../../../templates"
-	resultDir   = "./test_data/templates"
-)
+const templateDir = "../../../templates"
 
 var (
 	configs = map[string]string{
@@ -340,13 +337,13 @@ func TestGenerateMachineConfigs(t *testing.T) {
 			t.Errorf("Failed to find pull secret for master")
 		}
 		if !foundKubeletUnitMaster {
-			t.Errorf("Failed to find kubelet unit")
+			t.Errorf("Failed to find kubelet unit for master")
 		}
 		if !foundPullSecretWorker {
-			t.Errorf("Failed to find pull secret")
+			t.Errorf("Failed to find pull secret for worker")
 		}
 		if !foundKubeletUnitWorker {
-			t.Errorf("Failed to find kubelet unit")
+			t.Errorf("Failed to find kubelet unit for worker")
 		}
 	}
 }

templates/common/_base/files/additional-trust-bundle.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0600
 path: "/etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt"
 contents:

templates/common/_base/files/cleanup-cni-conf.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/tmpfiles.d/cleanup-cni.conf"
 contents:

templates/common/_base/files/cloud-provider-ca.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem"
 contents:

templates/common/_base/files/container-storage.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/containers/storage.conf"
 contents:

templates/common/_base/files/etc-networkmanager-dispatcher.d-90-long-hostname.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0755
 path: "/etc/NetworkManager/dispatcher.d/90-long-hostname"
 contents:

templates/common/_base/files/kubelet-ca.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/kubernetes/kubelet-ca.crt"
 contents:

templates/common/_base/files/nm-ignore-sdn.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/NetworkManager/conf.d/sdn.conf"
 contents:

templates/common/_base/files/pull-secret.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0600
 path: "/var/lib/kubelet/config.json"
 contents:

templates/common/_base/files/root-ca.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/kubernetes/ca.crt"
 contents:

templates/common/_base/files/sysctl-forward-conf.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/sysctl.d/forward.conf"
 contents:

templates/common/_base/files/sysctl-inotify.conf.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/sysctl.d/inotify.conf"
 contents:

templates/common/_base/files/volume-plugins.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0755
 path: "/etc/kubernetes/kubelet-plugins/volume/exec/.dummy"
 contents:

templates/common/_base/units/machine-config-daemon-firstboot-v42.service → templates/common/_base/units/machine-config-daemon-firstboot-v42.service.yaml

@@ -1,4 +1,4 @@
-name: "machine-config-daemon-firstboot-v42.service"
+name: machine-config-daemon-firstboot-v42.service
 enabled: true
 contents: |
   [Unit]

templates/common/_base/units/machine-config-daemon-firstboot.service → templates/common/_base/units/machine-config-daemon-firstboot.service.yaml

@@ -1,4 +1,4 @@
-name: "machine-config-daemon-firstboot.service"
+name: machine-config-daemon-firstboot.service
 enabled: true
 contents: |
   [Unit]

templates/common/_base/units/machine-config-daemon-host.service → templates/common/_base/units/machine-config-daemon-host.service.yaml

@@ -1,4 +1,4 @@
-name: "machine-config-daemon-host.service"
+name: machine-config-daemon-host.service
 enabled: false
 contents: |
   [Unit]
@@ -20,3 +20,20 @@ contents: |
 
   [Install]
   WantedBy=multi-user.target
+dropins:
+  - name: 10-mco-default-env.conf
+    contents: |
+      [Unit]
+      Description=MCO environment configuration
+      {{if .Proxy -}}
+      [Service]
+      {{if .Proxy.HTTPProxy -}}
+      Environment=HTTP_PROXY={{.Proxy.HTTPProxy}}
+      {{end -}}
+      {{if .Proxy.HTTPSProxy -}}
+      Environment=HTTPS_PROXY={{.Proxy.HTTPSProxy}}
+      {{end -}}
+      {{if .Proxy.NoProxy -}}
+      Environment=NO_PROXY={{.Proxy.NoProxy}}
+      {{end -}}
+      {{end -}}

templates/common/baremetal/files/NetworkManager-mdns-hostname.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0755
 path: "/etc/NetworkManager/dispatcher.d/40-mdns-hostname"
 contents:

templates/common/baremetal/files/NetworkManager-resolv-prepender.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0755
 path: "/etc/NetworkManager/dispatcher.d/30-resolv-prepender"
 contents:

templates/common/baremetal/files/baremetal-NetworkManager-kni-conf.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/NetworkManager/conf.d/99-kni.conf"
 contents:

templates/common/baremetal/files/baremetal-coredns-corefile.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/kubernetes/static-pod-resources/coredns/Corefile.tmpl"
 contents:

templates/common/baremetal/files/baremetal-coredns.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/kubernetes/manifests/coredns.yaml"
 contents:

templates/common/baremetal/files/baremetal-keepalived.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/kubernetes/manifests/keepalived.yaml"
 contents:

templates/common/baremetal/files/baremetal-mdns-publisher.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/kubernetes/manifests/mdns-publisher.yaml"
 contents:

templates/common/baremetal/files/etc-systemd-system-crio-stream-address.conf.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/systemd/system/crio.service.d/20-stream-address.conf"
 contents:

templates/common/baremetal/units/nodeip-configuration.service → templates/common/baremetal/units/nodeip-configuration.service.yaml

@@ -1,4 +1,4 @@
-name: "nodeip-configuration.service"
+name: nodeip-configuration.service
 enabled: true
 contents: |
   [Unit]

templates/common/openstack/files/NetworkManager-mdns-hostname.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0755
 path: "/etc/NetworkManager/dispatcher.d/40-mdns-hostname"
 contents:

templates/common/openstack/files/openstack-NetworkManager-conf.yaml

@@ -1,4 +1,3 @@
-filesystem: "root"
 mode: 0644
 path: "/etc/NetworkManager/conf.d/99-openstack.conf"
 contents: