From efe9dce15023edf8a39a41beb45c547fdd56fdfe Mon Sep 17 00:00:00 2001 From: Paul Michali Date: Mon, 22 Oct 2018 12:59:44 +0000 Subject: [PATCH] Support insecure mode, where authentation token is not required, implying init step not needed and config YAML doesn't have to be updated on minons. --- README.md | 15 +- cmd/lazyjack.go | 2 +- config.go | 4 + prepare.go | 6 +- prepare_test.go | 437 +++++++++++++++++++++++++++++++++++++++++++++++ up.go | 12 +- up_test.go | 10 ++ validate.go | 11 +- validate_test.go | 9 + 9 files changed, 494 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8db6338..cbbd0e5 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ general: work-area: "/tmp/lazyjack" mode: "ipv6" kubernetes-version: "v1.12.0" + insecure: true topology: my-master: interface: "enp10s0" @@ -174,6 +175,12 @@ If omitted, the version of KubeAdm will be used to specify the Kubernetes versio NOTE: If you are using an un-released version, it may be beneficial to set this to `latest`. +### Insecure mode (insecure) +This optional boolean flag can be set to allow KubeAdm to run without specifying +an auth token. This means that the `init` step is not needed, and the config YAML +file does not need to be copied over to the minions, after the `prepare` step, thus +simplifying startup for a non-production environment. + ### Topology (topology) This is where you specify each of the systems to be provisioned. Each entry is referred to by the hostname, and contains three items. @@ -328,7 +335,7 @@ as root: ``` The commands do the following: -* **init** - Sets up tokens and certificates needed by Kuberentes. Must be run on the master node, **before** copying the config file to minion nodes. Only needed once. +* **init** - Sets up tokens and certificates needed by Kuberentes. Must be run on the master node, **before** copying the config file to minion nodes. Only needed once. Not needed, if running in insecure mode. * **prepare** - Prepares the node so that cluster can be brought up. Do on each node, before proceeded to next step. * **up** - Brings up Kubernetes cluster on the node. Do master first, and then minions. * **down** - Tears down the cluster on the node. Do minions first, and then master. @@ -378,7 +385,7 @@ For each command, there are a series of actions performed... ### For the `init` command * Creates CA certificate and key for KubeAdm. * Creates token and CA certificate hash. -* Updates the configuration YAML file (needed for `up` command on minions). +* Updates the configuration YAML file (needed for `up` command on minions, unless running in insecure mode). ### For the `prepare` command * (IPv6) Creates support network with IPv6 and IPv4. @@ -461,8 +468,8 @@ I also installed `ipset` and `ipvsadm`. * Some newer versions of docker break the enabling of IPv6 in the containers used for DNS64 and NAT64. * CNI v0.7.1+ is needed for full IPv6 support by plugins. * Relies on the tayga and bind6 containers (as provided by other developers), for IPv6 only mode. -* The `init` command modifies the specified configuration YAML file. As a result, `init` must be done before copying the config YAML to other nodes. -* Because the config YAML file is modified by the root user, permissions is set to 777, so that the non-root user can still modify the file. +* The `init` command modifies the specified configuration YAML file. As a result, `init` must be done before copying the config YAML to other nodes, unless you are running in insecure mode where the `init` step is not needed and the config YAML is not updated. +* In normal mode, because the config YAML file is modified by the root user, permissions is set to 777, so that the non-root user can still modify the file. ## Troubleshooting diff --git a/cmd/lazyjack.go b/cmd/lazyjack.go index 55b448a..28501d6 100644 --- a/cmd/lazyjack.go +++ b/cmd/lazyjack.go @@ -13,7 +13,7 @@ import ( ) const ( - Version = "1.3.1" + Version = "1.3.2" ) func init() { diff --git a/config.go b/config.go index 31119c5..fb47c6d 100644 --- a/config.go +++ b/config.go @@ -86,6 +86,7 @@ type GeneralSettings struct { KubeAdmVersion string // Internal FullKubeAdmVersion string // Internal K8sVersion string `yaml:"kubernetes-version"` + Insecure bool `yaml:"insecure"` } // Config defines the top level configuration read from YAML file. @@ -155,6 +156,9 @@ const ( // KubeAdmConfFile name of the configuration file used by KubeAdm KubeAdmConfFile = "kubeadm.conf" + // DefaultToken used when in insecure mode + DefaultToken = "abcdef.abcdefghijklmnop" + // MinimumPodMTU is the smallest MTU for IPv6 MinimumPodMTU = 1280 // DefaultPodMTU is the default MTU to use, when not specified diff --git a/prepare.go b/prepare.go index 88efaa7..c00dd94 100644 --- a/prepare.go +++ b/prepare.go @@ -48,7 +48,11 @@ func CollectKubeAdmConfigInfo(n *Node, c *Config) KubeAdmConfigInfo { info.AdvertiseAddress = fmt.Sprintf("%s%d", c.Mgmt.Prefix, n.ID) - info.AuthToken = c.General.Token + if c.General.Insecure { + info.AuthToken = DefaultToken + } else { + info.AuthToken = c.General.Token + } serviceNetMode := "::" devicePart := "a" diff --git a/prepare_test.go b/prepare_test.go index 35ff557..605b5d1 100644 --- a/prepare_test.go +++ b/prepare_test.go @@ -1692,6 +1692,443 @@ volumeStatsAggPeriod: 1m0s } } +func TestKubeAdmConfigContentsForInsecureKubeAdm_1_11(t *testing.T) { + c := &lazyjack.Config{ + General: lazyjack.GeneralSettings{ + Insecure: true, + Token: "56cdce.7b18ad347f3de81c", + KubeAdmVersion: "1.11", + K8sVersion: "v1.11.1", + }, + Pod: lazyjack.PodNetwork{ + CIDR: "fd00:40::/72", + }, + Service: lazyjack.ServiceNetwork{ + CIDR: "fd00:30::/110", + Mode: "ipv6", + Prefix: "fd00:30::", + }, + Mgmt: lazyjack.ManagementNetwork{ + Prefix: "fd00:100::", + }, + } + n := &lazyjack.Node{ + Name: "my-master", + ID: 10, + } + + expected := `# V1.11 based config +api: + advertiseAddress: "fd00:100::10" + bindPort: 6443 + controlPlaneEndpoint: "" +apiServerExtraArgs: + insecure-bind-address: "::" + insecure-port: "8080" +apiVersion: kubeadm.k8s.io/v1alpha2 +auditPolicy: + logDir: /var/log/kubernetes/audit + logMaxAge: 2 + path: "" +bootstrapTokens: +- groups: + - system:bootstrappers:kubeadm:default-node-token + token: abcdef.abcdefghijklmnop + ttl: 0s + usages: + - signing + - authentication +certificatesDir: /etc/kubernetes/pki +# clusterName: kubernetes +etcd: + local: + dataDir: /var/lib/etcd + image: "" +featureGates: {CoreDNS: false} +kind: MasterConfiguration +kubeProxy: + config: + bindAddress: "::" + clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + qps: 5 + # clusterCIDR: "" + configSyncPeriod: 15m0s + # conntrack: + # max: null + # maxPerCore: 32768 + # min: 131072 + # tcpCloseWaitTimeout: 1h0m0s + # tcpEstablishedTimeout: 24h0m0s + enableProfiling: false + healthzBindAddress: 0.0.0.0:10256 + hostnameOverride: "" + iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s + ipvs: + excludeCIDRs: null + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s + metricsBindAddress: 127.0.0.1:10249 + mode: "" + nodePortAddresses: null + oomScoreAdj: -999 + portRange: "" + resourceContainer: /kube-proxy + udpIdleTimeout: 250ms +kubeletConfiguration: + baseConfig: + address: 0.0.0.0 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s + cgroupDriver: cgroupfs + cgroupsPerQOS: true + clusterDNS: + - "fd00:30::a" + clusterDomain: cluster.local + containerLogMaxFiles: 5 + containerLogMaxSize: 10Mi + contentType: application/vnd.kubernetes.protobuf + cpuCFSQuota: true + cpuManagerPolicy: none + cpuManagerReconcilePeriod: 10s + enableControllerAttachDetach: true + enableDebuggingHandlers: true + enforceNodeAllocatable: + - pods + eventBurst: 10 + eventRecordQPS: 5 + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 5m0s + failSwapOn: true + fileCheckFrequency: 20s + hairpinMode: promiscuous-bridge + healthzBindAddress: 127.0.0.1 + healthzPort: 10248 + httpCheckFrequency: 20s + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + imageMinimumGCAge: 2m0s + iptablesDropBit: 15 + iptablesMasqueradeBit: 14 + kubeAPIBurst: 10 + kubeAPIQPS: 5 + makeIPTablesUtilChains: true + maxOpenFiles: 1000000 + maxPods: 110 + nodeStatusUpdateFrequency: 10s + oomScoreAdj: -999 + podPidsLimit: -1 + # port: 10250 + registryBurst: 10 + registryPullQPS: 5 + resolvConf: /etc/resolv.conf + rotateCertificates: true + runtimeRequestTimeout: 2m0s + serializeImagePulls: true + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 4h0m0s + syncFrequency: 1m0s + volumeStatsAggPeriod: 1m0s +kubernetesVersion: "v1.11.1" +networking: + # podSubnet: "fd00:40::/72" + serviceSubnet: "fd00:30::/110" +nodeRegistration: + name: my-master +unifiedControlPlaneImage: "" +` + actual := string(lazyjack.CreateKubeAdmConfigContents(n, c)) + if actual != expected { + t.Fatalf("FAILED: kubeadm.conf contents wrong\nExpected: %s\n Actual: %s\n", expected, actual) + } +} + +func TestKubeAdmConfigContentsForInsecureKubeAdm_1_12(t *testing.T) { + c := &lazyjack.Config{ + General: lazyjack.GeneralSettings{ + Insecure: true, + Token: "64rxu8.yvrzfofegfmyy1no", + KubeAdmVersion: "1.12", + K8sVersion: "v1.12.0", + }, + Pod: lazyjack.PodNetwork{ + CIDR: "fd00:40::/72", + }, + Service: lazyjack.ServiceNetwork{ + CIDR: "fd00:30::/110", + Mode: "ipv6", + Prefix: "fd00:30::", + }, + Mgmt: lazyjack.ManagementNetwork{ + Prefix: "fd00:100::", + }, + } + n := &lazyjack.Node{ + Name: "my-master", + ID: 10, + } + + expected := `# V1.12 based config +apiEndpoint: + advertiseAddress: "fd00:100::10" + bindPort: 6443 +apiVersion: kubeadm.k8s.io/v1alpha3 +bootstrapTokens: +- groups: + - system:bootstrappers:kubeadm:default-node-token + token: abcdef.abcdefghijklmnop + ttl: 0s + usages: + - signing + - authentication +kind: InitConfiguration +nodeRegistration: + criSocket: /var/run/dockershim.sock + name: my-master + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/master +--- +apiServerExtraArgs: + insecure-bind-address: "::" + insecure-port: "8080" +apiVersion: kubeadm.k8s.io/v1alpha3 +auditPolicy: + logDir: /var/log/kubernetes/audit + logMaxAge: 2 + path: "" +certificatesDir: /etc/kubernetes/pki +controlPlaneEndpoint: "" +etcd: + local: + dataDir: /var/lib/etcd + image: "" +featureGates: {CoreDNS: false} +imageRepository: k8s.gcr.io +kind: ClusterConfiguration +kubernetesVersion: "v1.12.0" +networking: + # podSubnet: "fd00:40::/72" + serviceSubnet: "fd00:30::/110" +unifiedControlPlaneImage: "" +` + actual := string(lazyjack.CreateKubeAdmConfigContents(n, c)) + if actual != expected { + t.Fatalf("FAILED: kubeadm.conf contents wrong\nExpected: %s\n Actual: %s\n", expected, actual) + } +} + +func TestKubeAdmConfigContentsForInsecureKubeAdm_1_13(t *testing.T) { + c := &lazyjack.Config{ + General: lazyjack.GeneralSettings{ + Insecure: true, + Token: "56cdce.7b18ad347f3de81c", + KubeAdmVersion: "1.13", + K8sVersion: "v1.13.0", + }, + Pod: lazyjack.PodNetwork{ + CIDR: "fd00:40::/72", + }, + Service: lazyjack.ServiceNetwork{ + CIDR: "fd00:30::/110", + Mode: "ipv6", + Prefix: "fd00:30::", + }, + Mgmt: lazyjack.ManagementNetwork{ + Prefix: "fd00:100::", + }, + } + n := &lazyjack.Node{ + Name: "my-master", + ID: 10, + } + + expected := `# V1.13 based config +apiEndpoint: + advertiseAddress: "fd00:100::10" + bindPort: 6443 +apiVersion: kubeadm.k8s.io/v1beta1 +bootstrapTokens: +- groups: + - system:bootstrappers:kubeadm:default-node-token + token: abcdef.abcdefghijklmnop + ttl: 24h0m0s + usages: + - signing + - authentication +kind: InitConfiguration +nodeRegistration: + criSocket: /var/run/dockershim.sock + name: my-master + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/master +--- +apiServerExtraArgs: + insecure-bind-address: "::" + insecure-port: "8080" +apiVersion: kubeadm.k8s.io/v1beta1 +auditPolicy: + logDir: /var/log/kubernetes/audit + logMaxAge: 2 + path: "" +certificatesDir: /etc/kubernetes/pki +# clusterName: kubernetes +controlPlaneEndpoint: "" +etcd: + local: + dataDir: /var/lib/etcd + image: "" +featureGates: {CoreDNS: false} +imageRepository: k8s.gcr.io +kind: ClusterConfiguration +kubernetesVersion: "v1.13.0" +networking: + dnsDomain: cluster.local + # podSubnet: "fd00:40::/72" + serviceSubnet: "fd00:30::/110" +unifiedControlPlaneImage: "" +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +bindAddress: "::" +clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + qps: 5 +# clusterCIDR: "" +configSyncPeriod: 15m0s +# conntrack: +# max: null +# maxPerCore: 32768 +# min: 131072 +# tcpCloseWaitTimeout: 1h0m0s +# tcpEstablishedTimeout: 24h0m0s +enableProfiling: false +healthzBindAddress: 0.0.0.0:10256 +hostnameOverride: "" +iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s +ipvs: + excludeCIDRs: null + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s +kind: KubeProxyConfiguration +metricsBindAddress: 127.0.0.1:10249 +mode: "" +nodePortAddresses: null +oomScoreAdj: -999 +portRange: "" +resourceContainer: /kube-proxy +udpIdleTimeout: 250ms +--- +address: 0.0.0.0 +apiVersion: kubelet.config.k8s.io/v1beta1 +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s +cgroupDriver: cgroupfs +cgroupsPerQOS: true +clusterDNS: +- "fd00:30::a" +clusterDomain: cluster.local +configMapAndSecretChangeDetectionStrategy: Watch +containerLogMaxFiles: 5 +containerLogMaxSize: 10Mi +contentType: application/vnd.kubernetes.protobuf +cpuCFSQuota: true +cpuCFSQuotaPeriod: 100ms +cpuManagerPolicy: none +cpuManagerReconcilePeriod: 10s +enableControllerAttachDetach: true +enableDebuggingHandlers: true +enforceNodeAllocatable: +- pods +eventBurst: 10 +eventRecordQPS: 5 +evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% +evictionPressureTransitionPeriod: 5m0s +failSwapOn: true +fileCheckFrequency: 20s +hairpinMode: promiscuous-bridge +healthzBindAddress: 127.0.0.1 +healthzPort: 10248 +httpCheckFrequency: 20s +imageGCHighThresholdPercent: 85 +imageGCLowThresholdPercent: 80 +imageMinimumGCAge: 2m0s +iptablesDropBit: 15 +iptablesMasqueradeBit: 14 +kind: KubeletConfiguration +kubeAPIBurst: 10 +kubeAPIQPS: 5 +makeIPTablesUtilChains: true +maxOpenFiles: 1000000 +maxPods: 110 +nodeLeaseDurationSeconds: 40 +nodeStatusUpdateFrequency: 10s +oomScoreAdj: -999 +podPidsLimit: -1 +# port: 10250 +registryBurst: 10 +registryPullQPS: 5 +resolvConf: /etc/resolv.conf +rotateCertificates: true +runtimeRequestTimeout: 2m0s +serializeImagePulls: true +staticPodPath: /etc/kubernetes/manifests +streamingConnectionIdleTimeout: 4h0m0s +syncFrequency: 1m0s +volumeStatsAggPeriod: 1m0s +` + actual := string(lazyjack.CreateKubeAdmConfigContents(n, c)) + if actual != expected { + t.Fatalf("FAILED: kubeadm.conf contents wrong\nExpected: %s\n Actual: %s\n", expected, actual) + } +} + func TestCreateKubeAdmConfFile(t *testing.T) { basePath := TempFileName(os.TempDir(), "-area") HelperSetupArea(basePath, t) diff --git a/up.go b/up.go index 78f4b13..aae5d40 100644 --- a/up.go +++ b/up.go @@ -89,9 +89,17 @@ func BuildKubeAdmCommand(n, master *Node, c *Config) []string { args = []string{"init", fmt.Sprintf("--config=%s", file)} } else { token := c.General.Token + if c.General.Insecure { + token = DefaultToken + } args = []string{"join", "--token", token} - args = append(args, "--discovery-token-ca-cert-hash", - fmt.Sprintf("sha256:%s", c.General.TokenCertHash)) + if c.General.Insecure { + args = append(args, "--discovery-token-unsafe-skip-ca-verification=true", + "--ignore-preflight-errors=all") + } else { + args = append(args, "--discovery-token-ca-cert-hash", + fmt.Sprintf("sha256:%s", c.General.TokenCertHash)) + } args = append(args, fmt.Sprintf("[%s%d]:6443", c.Mgmt.Prefix, master.ID)) } return args diff --git a/up_test.go b/up_test.go index 16b013b..8f5fc8f 100644 --- a/up_test.go +++ b/up_test.go @@ -57,6 +57,16 @@ func TestBuildKubeAdmCommand(t *testing.T) { if !SlicesEqual(actual, expected) { t.Errorf("KubeAdm init args incorrect for minion node. Expected %q, got %q", strings.Join(expected, " "), strings.Join(actual, " ")) } + + c.General.Insecure = true + actual = lazyjack.BuildKubeAdmCommand(minionNode, masterNode, c) + expected = []string{"join", "--token", lazyjack.DefaultToken, + "--discovery-token-unsafe-skip-ca-verification=true", + "--ignore-preflight-errors=all", "[fd00:100::10]:6443"} + + if !SlicesEqual(actual, expected) { + t.Errorf("KubeAdm init args incorrect for insecure minion node. Expected %q, got %q", strings.Join(expected, " "), strings.Join(actual, " ")) + } } func TestEnsureCNIAreaExists(t *testing.T) { diff --git a/validate.go b/validate.go index 0a306a9..fd1b7de 100644 --- a/validate.go +++ b/validate.go @@ -501,10 +501,10 @@ func SetupHandles(c *Config) error { // ValidateConfigContents checks contents of the config file. // Token and certificate hash validation is ignored during init -// phase, which will generate these values. Side effect is that -// base paths are set up based on defaults (unless overriden by -// config file). The netlink library handle is set (allowing UTs -// to override and mock that library). +// phase, which will generate these values, or if running in +// insecure mode. Side effect is that base paths are set up based on +// defaults (unless overriden by config file). The netlink library +// handle is set (allowing UTs to override and mock that library). // TODO: Validate support net v4 subnet > NAT64 subnet func ValidateConfigContents(c *Config, ignoreMissing bool) error { var err error @@ -521,6 +521,9 @@ func ValidateConfigContents(c *Config, ignoreMissing bool) error { return err } + if c.General.Insecure { + ignoreMissing = true // force on + } err = ValidateToken(c.General.Token, ignoreMissing) if err != nil { return err diff --git a/validate_test.go b/validate_test.go index 2d5f04c..94baf4f 100644 --- a/validate_test.go +++ b/validate_test.go @@ -1654,6 +1654,15 @@ func TestValidateSoftwareVersions(t *testing.T) { k8sVersion: "latest", expectedStr: "", }, + { + name: "unsupported verison (ok)", + execCmd: func(string, []string) (string, error) { + return "v1.9.1", nil + }, + version: "1.9", + k8sVersion: "v1.9.3", + expectedStr: "", + }, { name: "k8s mismatch", execCmd: func(string, []string) (string, error) {