Skip to content

Commit

Permalink
adding etcd settings to protokube
Browse files Browse the repository at this point in the history
  • Loading branch information
mschurenko committed Oct 20, 2017
1 parent 90c7ccb commit 91e390b
Show file tree
Hide file tree
Showing 8 changed files with 205 additions and 54 deletions.
6 changes: 6 additions & 0 deletions nodeup/pkg/model/convenience.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"

"github.com/golang/glog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// s is a helper that builds a *string from a string value
Expand Down Expand Up @@ -171,3 +172,8 @@ func addHostPathMapping(pod *v1.Pod, container *v1.Container, name, path string)

return &container.VolumeMounts[len(container.VolumeMounts)-1]
}

// convEtcdSettingsToMs converts etcd settings to a string rep of int milliseconds
func convEtcdSettingsToMs(dur *metav1.Duration) string {
return strconv.FormatInt(dur.Nanoseconds()/1000000, 10)
}
57 changes: 36 additions & 21 deletions nodeup/pkg/model/protokube.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,22 +192,24 @@ type ProtokubeFlags struct {
Channels []string `json:"channels,omitempty" flag:"channels"`
Cloud *string `json:"cloud,omitempty" flag:"cloud"`
// ClusterID flag is required only for vSphere cloud type, to pass cluster id information to protokube. AWS and GCE workflows ignore this flag.
ClusterID *string `json:"cluster-id,omitempty" flag:"cluster-id"`
Containerized *bool `json:"containerized,omitempty" flag:"containerized"`
DNSInternalSuffix *string `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"`
DNSProvider *string `json:"dnsProvider,omitempty" flag:"dns"`
DNSServer *string `json:"dns-server,omitempty" flag:"dns-server"`
EtcdImage *string `json:"etcd-image,omitempty" flag:"etcd-image"`
InitializeRBAC *bool `json:"initializeRBAC,omitempty" flag:"initialize-rbac"`
LogLevel *int32 `json:"logLevel,omitempty" flag:"v"`
Master *bool `json:"master,omitempty" flag:"master"`
PeerTLSCaFile *string `json:"peer-ca,omitempty" flag:"peer-ca"`
PeerTLSCertFile *string `json:"peer-cert,omitempty" flag:"peer-cert"`
PeerTLSKeyFile *string `json:"peer-key,omitempty" flag:"peer-key"`
TLSCAFile *string `json:"tls-ca,omitempty" flag:"tls-ca"`
TLSCertFile *string `json:"tls-cert,omitempty" flag:"tls-cert"`
TLSKeyFile *string `json:"tls-key,omitempty" flag:"tls-key"`
Zone []string `json:"zone,omitempty" flag:"zone"`
ClusterID *string `json:"cluster-id,omitempty" flag:"cluster-id"`
Containerized *bool `json:"containerized,omitempty" flag:"containerized"`
DNSInternalSuffix *string `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"`
DNSProvider *string `json:"dnsProvider,omitempty" flag:"dns"`
DNSServer *string `json:"dns-server,omitempty" flag:"dns-server"`
EtcdImage *string `json:"etcd-image,omitempty" flag:"etcd-image"`
EtcdLeaderElectionTimeout *string `json:"etcd-election-timeout,omitempty" flag:"etcd-election-timeout"`
EtcdHearbeatInterval *string `json:"etcd-heartbeat-interval,omitempty" flag:"etcd-heartbeat-interval"`
InitializeRBAC *bool `json:"initializeRBAC,omitempty" flag:"initialize-rbac"`
LogLevel *int32 `json:"logLevel,omitempty" flag:"v"`
Master *bool `json:"master,omitempty" flag:"master"`
PeerTLSCaFile *string `json:"peer-ca,omitempty" flag:"peer-ca"`
PeerTLSCertFile *string `json:"peer-cert,omitempty" flag:"peer-cert"`
PeerTLSKeyFile *string `json:"peer-key,omitempty" flag:"peer-key"`
TLSCAFile *string `json:"tls-ca,omitempty" flag:"tls-ca"`
TLSCertFile *string `json:"tls-cert,omitempty" flag:"tls-cert"`
TLSKeyFile *string `json:"tls-key,omitempty" flag:"tls-key"`
Zone []string `json:"zone,omitempty" flag:"zone"`
}

// ProtokubeFlags is responsible for building the command line flags for protokube
Expand All @@ -216,12 +218,25 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) *ProtokubeF
// lets keep that for another PR and allow the version change
imageVersion := t.Cluster.Spec.EtcdClusters[0].Version

var leaderElectionTimeout string
var heartbeatInterval string

if v := t.Cluster.Spec.EtcdClusters[0].LeaderElectionTimeout; v != nil {
leaderElectionTimeout = convEtcdSettingsToMs(v)
}

if v := t.Cluster.Spec.EtcdClusters[0].HeartbeatInterval; v != nil {
heartbeatInterval = convEtcdSettingsToMs(v)
}

f := &ProtokubeFlags{
Channels: t.NodeupConfig.Channels,
Containerized: fi.Bool(true),
EtcdImage: s(fmt.Sprintf("gcr.io/google_containers/etcd:%s", imageVersion)),
LogLevel: fi.Int32(4),
Master: b(t.IsMaster),
Channels: t.NodeupConfig.Channels,
Containerized: fi.Bool(true),
EtcdImage: s(fmt.Sprintf("gcr.io/google_containers/etcd:%s", imageVersion)),
EtcdLeaderElectionTimeout: s(leaderElectionTimeout),
EtcdHearbeatInterval: s(heartbeatInterval),
LogLevel: fi.Int32(4),
Master: b(t.IsMaster),
}

// initialize rbac on Kubernetes >= 1.6 and master
Expand Down
39 changes: 22 additions & 17 deletions protokube/cmd/protokube/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ func run() error {
var zones []string
var applyTaints, initializeRBAC, containerized, master bool
var cloud, clusterID, dnsServer, dnsProviderID, dnsInternalSuffix, gossipSecret, gossipListen string
var flagChannels, tlsCert, tlsKey, tlsCA, peerCert, peerKey, peerCA, etcdImageSource string
var flagChannels, tlsCert, tlsKey, tlsCA, peerCert, peerKey, peerCA string
var etcdImageSource, etcdElectionTimeout, etcdHeartbeatInterval string

flag.BoolVar(&applyTaints, "apply-taints", applyTaints, "Apply taints to nodes based on the role")
flag.BoolVar(&containerized, "containerized", containerized, "Set if we are running containerized.")
Expand All @@ -82,6 +83,8 @@ func run() error {
flags.StringSliceVarP(&zones, "zone", "z", []string{}, "Configure permitted zones and their mappings")
flags.StringVar(&dnsProviderID, "dns", "aws-route53", "DNS provider we should use (aws-route53, google-clouddns, coredns)")
flags.StringVar(&etcdImageSource, "etcd-image", "gcr.io/google_containers/etcd:2.2.1", "Etcd Source Container Registry")
flags.StringVar(&etcdElectionTimeout, "etcd-election-timeout", etcdElectionTimeout, "time in ms for an election to timeout")
flags.StringVar(&etcdHeartbeatInterval, "etcd-heartbeat-interval", etcdHeartbeatInterval, "time in ms of a heartbeat interval")
flags.StringVar(&gossipSecret, "gossip-secret", gossipSecret, "Secret to use to secure gossip")

// Trick to avoid 'logging before flag.Parse' warning
Expand Down Expand Up @@ -282,22 +285,24 @@ func run() error {
}

k := &protokube.KubeBoot{
ApplyTaints: applyTaints,
Channels: channels,
DNS: dnsProvider,
EtcdImageSource: etcdImageSource,
InitializeRBAC: initializeRBAC,
InternalDNSSuffix: dnsInternalSuffix,
InternalIP: internalIP,
Kubernetes: protokube.NewKubernetesContext(),
Master: master,
ModelDir: modelDir,
PeerCA: peerCA,
PeerCert: peerCert,
PeerKey: peerKey,
TLSCA: tlsCA,
TLSCert: tlsCert,
TLSKey: tlsKey,
ApplyTaints: applyTaints,
Channels: channels,
DNS: dnsProvider,
EtcdImageSource: etcdImageSource,
EtcdElectionTimeout: etcdElectionTimeout,
EtcdHeartbeatInterval: etcdHeartbeatInterval,
InitializeRBAC: initializeRBAC,
InternalDNSSuffix: dnsInternalSuffix,
InternalIP: internalIP,
Kubernetes: protokube.NewKubernetesContext(),
Master: master,
ModelDir: modelDir,
PeerCA: peerCA,
PeerCert: peerCert,
PeerKey: peerKey,
TLSCA: tlsCA,
TLSCert: tlsCert,
TLSKey: tlsKey,
}

k.Init(volumes)
Expand Down
36 changes: 21 additions & 15 deletions protokube/pkg/protokube/etcd_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ type EtcdCluster struct {
PeerCert string
// PeerKey is the path to a peer ca for etcd
PeerKey string
// ElectionTimeout is the leader election timeout
ElectionTimeout string
// HeartbeatInterval is the heartbeat interval
HeartbeatInterval string
}

// EtcdNode is a definition for the etcd node
Expand All @@ -97,21 +101,23 @@ func newEtcdController(kubeBoot *KubeBoot, v *Volume, spec *etcd.EtcdClusterSpec

cluster := &EtcdCluster{
// @TODO we need to deprecate this port and use 2379, but that would be a breaking change
ClientPort: 4001,
ClusterName: "etcd-" + spec.ClusterKey,
CPURequest: resource.MustParse("200m"),
DataDirName: "data-" + spec.ClusterKey,
ImageSource: kubeBoot.EtcdImageSource,
TLSCA: kubeBoot.TLSCA,
TLSCert: kubeBoot.TLSCert,
TLSKey: kubeBoot.TLSKey,
PeerCA: kubeBoot.PeerCA,
PeerCert: kubeBoot.PeerCert,
PeerKey: kubeBoot.PeerKey,
PeerPort: 2380,
PodName: "etcd-server-" + spec.ClusterKey,
Spec: spec,
VolumeMountPath: v.Mountpoint,
ClientPort: 4001,
ClusterName: "etcd-" + spec.ClusterKey,
CPURequest: resource.MustParse("200m"),
DataDirName: "data-" + spec.ClusterKey,
ImageSource: kubeBoot.EtcdImageSource,
TLSCA: kubeBoot.TLSCA,
TLSCert: kubeBoot.TLSCert,
TLSKey: kubeBoot.TLSKey,
PeerCA: kubeBoot.PeerCA,
PeerCert: kubeBoot.PeerCert,
PeerKey: kubeBoot.PeerKey,
PeerPort: 2380,
PodName: "etcd-server-" + spec.ClusterKey,
Spec: spec,
VolumeMountPath: v.Mountpoint,
ElectionTimeout: kubeBoot.EtcdElectionTimeout,
HeartbeatInterval: kubeBoot.EtcdHeartbeatInterval,
}

// We used to build this through text files ... it turns out to just be more complicated than code!
Expand Down
10 changes: 9 additions & 1 deletion protokube/pkg/protokube/etcd_manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {
"/bin/sh", "-c", "/usr/local/bin/etcd 2>&1 | /bin/tee -a /var/log/etcd.log",
},
}
// build the the environment variables for etcd service
// build the environment variables for etcd service
container.Env = buildEtcdEnvironmentOptions(c)

container.LivenessProbe = &v1.Probe{
Expand Down Expand Up @@ -170,6 +170,14 @@ func buildEtcdEnvironmentOptions(c *EtcdCluster) []v1.EnvVar {
{Name: "ETCD_INITIAL_CLUSTER_STATE", Value: "new"},
{Name: "ETCD_INITIAL_CLUSTER_TOKEN", Value: c.ClusterToken}}...)

// add timeout/hearbeat settings
if notEmpty(c.ElectionTimeout) {
options = append(options, v1.EnvVar{Name: "ETCD_ELECTION_TIMEOUT", Value: c.ElectionTimeout})
}
if notEmpty(c.HeartbeatInterval) {
options = append(options, v1.EnvVar{Name: "ETCD_HEARTBEAT_INTERVAL", Value: c.HeartbeatInterval})
}

// @check if we are using peer certificates
if notEmpty(c.PeerCA) {
options = append(options, []v1.EnvVar{
Expand Down
4 changes: 4 additions & 0 deletions protokube/pkg/protokube/kube_boot.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ type KubeBoot struct {
ModelDir string
// Etcd container registry location.
EtcdImageSource string
// Etcd election timeout
EtcdElectionTimeout string
// Etcd heartbeat interval
EtcdHeartbeatInterval string
// TLSCA is the path to a client ca for etcd
TLSCA string
// TLSCert is the path to a tls certificate for etcd
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func TestBuildEtcdManifest(t *testing.T) {
}{
{TestFile: "non_tls.yaml"},
{TestFile: "tls.yaml"},
{TestFile: "etcd_env_vars.yaml"},
}
for i, x := range cs {
cluster, expected := loadTestIntegration(t, path.Join("main", x.TestFile))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
clientPort: 4001
clusterName: etcd-main
clusterToken: token-main
cpuRequest: "200m"
dataDirName: data-main
imageSource: gcr.io/google_containers/etcd:2.2.1
logFile: /var/log/etcd.log
peerPort: 2380
podName: etcd-server-main
volumeMountPath: /mnt/main
electionTimeout: "1000"
heartbeatInterval: "100"
me:
name: node0
internalName: node0.internal
nodes:
- name: node0
internalName: node0.internal
- name: node1
internalName: node1.internal
- name: node2
internalName: node2.internal
spec: {}
---
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
labels:
k8s-app: etcd-server-main
name: etcd-server-main
namespace: kube-system
spec:
containers:
- command:
- /bin/sh
- -c
- /usr/local/bin/etcd 2>&1 | /bin/tee -a /var/log/etcd.log
env:
- name: ETCD_NAME
value: node0
- name: ETCD_DATA_DIR
value: /var/etcd/data-main
- name: ETCD_LISTEN_PEER_URLS
value: http://0.0.0.0:2380
- name: ETCD_LISTEN_CLIENT_URLS
value: http://0.0.0.0:4001
- name: ETCD_ADVERTISE_CLIENT_URLS
value: http://node0.internal:4001
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: http://node0.internal:2380
- name: ETCD_INITIAL_CLUSTER_STATE
value: new
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: token-main
- name: ETCD_ELECTION_TIMEOUT
value: "1000"
- name: ETCD_HEARTBEAT_INTERVAL
value: "100"
- name: ETCD_INITIAL_CLUSTER
value: node0=http://node0.internal:2380,node1=http://node1.internal:2380,node2=http://node2.internal:2380
image: gcr.io/google_containers/etcd:2.2.1
livenessProbe:
httpGet:
host: 127.0.0.1
path: /health
port: 4001
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: etcd-container
ports:
- containerPort: 2380
hostPort: 2380
name: serverport
- containerPort: 4001
hostPort: 4001
name: clientport
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /var/etcd/data-main
name: varetcdata
- mountPath: /var/log/etcd.log
name: varlogetcd
- mountPath: /etc/hosts
name: hosts
readOnly: true
hostNetwork: true
tolerations:
- key: CriticalAddonsOnly
operator: Exists
volumes:
- hostPath:
path: /mnt/main/var/etcd/data-main
name: varetcdata
- hostPath:
path: /var/log/etcd.log
name: varlogetcd
- hostPath:
path: /etc/hosts
name: hosts
status: {}

0 comments on commit 91e390b

Please sign in to comment.