pkg/cmd/render: add support for s390 and s390x arch

Signed-off-by: Sam Batschelet <sbatsche@redhat.com>

bindata/bootkube/bootstrap-manifests/etcd-member-pod.yaml

@@ -69,6 +69,7 @@ spec:
       set -euo pipefail
 
       exec etcd \
+        --initial-cluster="etcd-bootstrap=https://{{ .EtcdAddress.EscapedBootstrapIP }}:2380" \
         --initial-advertise-peer-urls=https://{{ .EtcdAddress.EscapedBootstrapIP }}:2380 \
         --cert-file=/etc/ssl/etcd/system:etcd-server:{{ .Hostname }}.crt \
         --key-file=/etc/ssl/etcd/system:etcd-server:{{ .Hostname }}.key \
@@ -100,14 +101,7 @@ spec:
       mountPath: /etc/etcd/
 
     env:
-    - name: ETCD_DATA_DIR
-      value: "/var/lib/etcd"
-    - name: ETCD_NAME
-      value: "etcd-bootstrap"
-    - name: ETCD_HEARTBEAT_INTERVAL
-      value: {{ .EtcdHeartbeat }}
-    - name: ETCD_ELECTION_TIMEOUT
-      value: {{ .EtcdElectionTimeout }}
+{{.ComputedEnvVars }}
     ports:
     - name: peer
       containerPort: 2380

pkg/cmd/render/env.go

@@ -0,0 +1,89 @@
+package render
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/openshift/cluster-etcd-operator/pkg/etcdenvvar"
+)
+
+var envVarFns = []envVarFunc{
+	getFixedEtcdEnvVars,
+	getHeartbeatInterval,
+	getElectionTimeout,
+	getUnsupportedArch,
+	getEtcdName,
+}
+
+type envVarFunc func(platform, arch string) (map[string]string, error)
+
+func getEtcdEnv(platform, arch string) (map[string]string, error) {
+	ret := map[string]string{}
+	for _, envVarFn := range envVarFns {
+		newEnvVars, err := envVarFn(platform, arch)
+		if err != nil {
+			return nil, err
+		}
+		if newEnvVars == nil {
+			continue
+		}
+		for k, v := range newEnvVars {
+			if currV, ok := ret[k]; ok {
+				return nil, fmt.Errorf("key %q already set to %q", k, currV)
+			}
+			ret[k] = v
+		}
+	}
+	return ret, nil
+}
+
+func getFixedEtcdEnvVars(platform, arch string) (map[string]string, error) {
+	return etcdenvvar.FixedEtcdEnvVars, nil
+}
+
+func getEtcdName(platform, arch string) (map[string]string, error) {
+	return map[string]string{
+		"ETCD_NAME": "etcd-bootstrap",
+	}, nil
+}
+
+func getHeartbeatInterval(platform, arch string) (map[string]string, error) {
+	var heartbeat string
+
+	switch platform {
+	case "Azure":
+		heartbeat = "500"
+	default:
+		heartbeat = "100"
+	}
+
+	return map[string]string{
+		"ETCD_HEARTBEAT_INTERVAL": heartbeat,
+	}, nil
+}
+
+func getElectionTimeout(platform, arch string) (map[string]string, error) {
+	var timeout string
+
+	switch platform {
+	case "Azure":
+		timeout = "2500"
+	default:
+		timeout = "1000"
+	}
+
+	return map[string]string{
+		"ETCD_ELECTION_TIMEOUT": timeout,
+	}, nil
+
+}
+
+func getUnsupportedArch(platform, arch string) (map[string]string, error) {
+	if !strings.HasPrefix(arch, "s390") {
+		// dont set unless it is defined.
+		return nil, nil
+	}
+	return map[string]string{
+		"ETCD_UNSUPPORTED_ARCH": arch,
+	}, nil
+}

pkg/cmd/render/env_test.go

@@ -0,0 +1,21 @@
+package render
+
+import "testing"
+
+func TestEtcdEnv(t *testing.T) {
+	for _, test := range []struct {
+		platform, arch, wantKey, wantValue string
+	}{
+		{"AWS", "amd64", "ETCD_HEARTBEAT_INTERVAL", "100"},
+		{"AWS", "amd64", "ETCD_ELECTION_TIMEOUT", "1000"},
+		{"Azure", "ppc64le", "ETCD_HEARTBEAT_INTERVAL", "500"},
+		{"Azure", "amd64", "ETCD_ELECTION_TIMEOUT", "2500"},
+		{"None", "ppc64le", "ETCD_HEARTBEAT_INTERVAL", "100"},
+		{"Azure", "s390x", "ETCD_UNSUPPORTED_ARCH", "s390x"},
+	} {
+		env, _ := getEtcdEnv(test.platform, test.arch)
+		if env[test.wantKey] != test.wantValue {
+			t.Errorf("getEtcdEnv(%q, %q) =  want %q = %q got %q = %q", test.platform, test.arch, test.wantKey, test.wantValue, test.wantKey, env[test.wantKey])
+		}
+	}
+}

pkg/cmd/render/render.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/openshift/cluster-etcd-operator/pkg/operator/etcd_assets"
@@ -20,7 +21,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
+	apiruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog"
 )
 
@@ -139,8 +141,6 @@ type TemplateData struct {
 	options.ManifestConfig
 	options.FileConfig
 
-	// EtcdDiscoveryDomain is the domain used for SRV discovery.
-	EtcdDiscoveryDomain    string
 	EtcdServerCertDNSNames string
 	EtcdPeerCertDNSNames   string
 
@@ -153,16 +153,17 @@ type TemplateData struct {
 	// SingleStackIPv6 is true if the stack is IPv6 only.
 	SingleStackIPv6 bool
 
-	// Hostname as reported by the kernel
+	// Hostname as reported by the kernel.
 	Hostname string
 
-	// BootstrapIP is address of the bootstrap node
+	// BootstrapIP is address of the bootstrap node.
 	BootstrapIP string
 
 	// Platform is the underlying provider the cluster is run on.
-	Platform            string
-	EtcdHeartbeat       string
-	EtcdElectionTimeout string
+	Platform string
+
+	// ComputedEnvVars name/value pairs to populate env: for static pod.
+	ComputedEnvVars string
 }
 
 type StaticFile struct {
@@ -178,21 +179,16 @@ func newTemplateData(opts *renderOpts) (*TemplateData, error) {
 		ManifestConfig: options.ManifestConfig{
 			Images: options.Images{
 				Etcd:            opts.etcdImage,
-				SetupEtcdEnv:    opts.setupEtcdEnvImage,
 				KubeClientAgent: opts.kubeClientAgentImage,
 			},
 		},
-		EtcdDiscoveryDomain: opts.etcdDiscoveryDomain,
 		EtcdServerCertDNSNames: strings.Join([]string{
 			"localhost",
 			"etcd.kube-system.svc",
 			"etcd.kube-system.svc.cluster.local",
 			"etcd.openshift-etcd.svc",
 			"etcd.openshift-etcd.svc.cluster.local",
 		}, ","),
-		EtcdPeerCertDNSNames: strings.Join([]string{
-			opts.etcdDiscoveryDomain,
-		}, ","),
 		BootstrapIP: opts.bootstrapIP,
 	}
 
@@ -209,8 +205,6 @@ func newTemplateData(opts *renderOpts) (*TemplateData, error) {
 		return nil, err
 	}
 
-	// TODO installer should feed value by flag this should be removed
-	// derive bootstrapIP from local interfaces if not passed by flag
 	if templateData.BootstrapIP == "" {
 		if err := templateData.setBootstrapIP(); err != nil {
 			return nil, err
@@ -223,13 +217,8 @@ func newTemplateData(opts *renderOpts) (*TemplateData, error) {
 		return nil, err
 	}
 
-	switch templateData.Platform {
-	case "Azure":
-		templateData.EtcdHeartbeat = `"500"`
-		templateData.EtcdElectionTimeout = `"2500"`
-	default:
-		templateData.EtcdHeartbeat = `"100"`
-		templateData.EtcdElectionTimeout = `"1000"`
+	if err := templateData.setComputedEnvVars(); err != nil {
+		return nil, err
 	}
 
 	return &templateData, nil
@@ -323,7 +312,7 @@ func (t *TemplateData) getClusterConfigFromFile(clusterConfigFile string) (*unst
 	if err != nil {
 		return nil, err
 	}
-	clusterConfigObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, configJson)
+	clusterConfigObj, err := apiruntime.Decode(unstructured.UnstructuredJSONScheme, configJson)
 	if err != nil {
 		return nil, err
 	}
@@ -358,6 +347,7 @@ func (t *TemplateData) setHostname() error {
 	t.Hostname = hostname
 	return nil
 }
+
 func (t *TemplateData) setPlatform(infraConfigFilePath string) error {
 	infrastructure := &configv1.Infrastructure{}
 	infraConfigFileData, err := ioutil.ReadFile(infraConfigFilePath)
@@ -412,6 +402,25 @@ func (t *TemplateData) setSingleStackIPv6() error {
 	return nil
 }
 
+func (t *TemplateData) setComputedEnvVars() error {
+	envVarMap, err := getEtcdEnv(t.Platform, runtime.GOARCH)
+	if err != nil {
+		return err
+	}
+	if len(envVarMap) == 0 {
+		return fmt.Errorf("missing env var values")
+	}
+
+	envVarLines := []string{}
+	for _, k := range sets.StringKeySet(envVarMap).List() {
+		v := envVarMap[k]
+		envVarLines = append(envVarLines, fmt.Sprintf("    - name: %q", k))
+		envVarLines = append(envVarLines, fmt.Sprintf("      value: %q", v))
+	}
+	t.ComputedEnvVars = strings.Join(envVarLines, "\n")
+	return nil
+}
+
 // TODO: add to util
 func EscapeIpv6Address(addr string) (string, error) {
 	if ip := net.ParseIP(addr); ip == nil {

pkg/etcdenvvar/etcd_env.go

@@ -2,6 +2,7 @@ package etcdenvvar
 
 import (
 	"fmt"
+	"runtime"
 	"strings"
 
 	"github.com/openshift/cluster-etcd-operator/pkg/dnshelpers"
@@ -24,6 +25,12 @@ type envVarContext struct {
 	targetImagePullSpec  string
 }
 
+var FixedEtcdEnvVars = map[string]string{
+	"ETCD_DATA_DIR":              "/var/lib/etcd",
+	"ETCD_QUOTA_BACKEND_BYTES":   "7516192768", // 7 gig
+	"ETCD_INITIAL_CLUSTER_STATE": "existing",
+}
+
 type envVarFunc func(envVarContext envVarContext) (map[string]string, error)
 
 var envVarFns = []envVarFunc{
@@ -35,6 +42,7 @@ var envVarFns = []envVarFunc{
 	getEtcdctlEnvVars,
 	getHeartbeatInterval,
 	getElectionTimeout,
+	getUnsupportedArch,
 }
 
 // getEtcdEnvVars returns the env vars that need to be set on the etcd static pods that will be rendered.
@@ -45,6 +53,7 @@ var envVarFns = []envVarFunc{
 //   ETCD_HEARTBEAT_INTERVAL
 //   ETCD_ELECTION_TIMEOUT
 //   ETCD_INITIAL_CLUSTER_STATE
+//   ETCD_UNSUPPORTED_ARCH
 //   NODE_%s_IP
 //   NODE_%s_ETCD_URL_HOST
 //   NODE_%s_ETCD_NAME
@@ -70,6 +79,9 @@ func getEtcdEnvVars(envVarContext envVarContext) (map[string]string, error) {
 		if err != nil {
 			return nil, err
 		}
+		if newEnvVars == nil {
+			continue
+		}
 		for k, v := range newEnvVars {
 			if currV, ok := ret[k]; ok {
 				return nil, fmt.Errorf("key %q already set to %q", k, currV)
@@ -82,11 +94,7 @@ func getEtcdEnvVars(envVarContext envVarContext) (map[string]string, error) {
 }
 
 func getFixedEtcdEnvVars(envVarContext envVarContext) (map[string]string, error) {
-	return map[string]string{
-		"ETCD_DATA_DIR":              "/var/lib/etcd",
-		"ETCD_QUOTA_BACKEND_BYTES":   "7516192768", // 7 gig
-		"ETCD_INITIAL_CLUSTER_STATE": "existing",
-	}, nil
+	return FixedEtcdEnvVars, nil
 }
 
 func getEtcdctlEnvVars(envVarContext envVarContext) (map[string]string, error) {
@@ -249,3 +257,14 @@ func getElectionTimeout(envVarContext envVarContext) (map[string]string, error)
 func envVarSafe(nodeName string) string {
 	return strings.ReplaceAll(strings.ReplaceAll(nodeName, "-", "_"), ".", "_")
 }
+
+func getUnsupportedArch(envVarContext envVarContext) (map[string]string, error) {
+	arch := runtime.GOARCH
+	if !strings.HasPrefix(arch, "s390") {
+		// dont set unless it is defined.
+		return nil, nil
+	}
+	return map[string]string{
+		"ETCD_UNSUPPORTED_ARCH": arch,
+	}, nil
+}