diff --git a/.gitignore b/.gitignore index 9106b2a34..3360c3f7b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ /pkg/ /spec/reports/ /tmp/ +/examples/vagrant/.vagrant/ + diff --git a/README.md b/README.md index 1be634189..d72966d64 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,25 @@ You can view full sample of cluster.yml [here](./cluster.example.yml). - `pod_network_cidr` - IP address range for the pod network. (default "10.32.0.0/12") - `trusted_subnets` - array of trusted subnets where overlay network can be used without IPSEC. +## Using external etcd + +Kupo can spin up Kubernetes using an externally managed etcd. In this case you need to define the external etcd details in your `cluster.yml` file: + +```yaml +etcd: + endpoints: + - https://etcd-1.example.com:2379 + - https://etcd-2.example.com:2379 + - https://etcd-3.example.com:2379 + certificate: ./etcd_certs/client.pem + key: ./etcd_certs/client-key.pem + ca_certificate: ./etcd_certs/ca.pem +``` + +You need to specify all etcd peer endpoints in the list. + +Certificate and corresponding key is used to authenticate the access to etcd. The paths used are relative to the path where the `cluster.yml` file was loaded from. + ## Addons Kupo includes common functionality as addons. Addons can be enabled by introducing and enabling them in `cluster.yml`. diff --git a/examples/vagrant/README.md b/examples/vagrant/README.md index 1ebf91e8f..799c1e83d 100644 --- a/examples/vagrant/README.md +++ b/examples/vagrant/README.md @@ -13,10 +13,37 @@ $ kubectl get nodes ## Teardown +Complete teardown: ```sh $ vagrant destroy ``` +"Soft" teardown, stops and removes all kube related configs/pods etc.: +```sh +ssh -i ~/.vagrant.d/insecure_private_key vagrant@192.168.100.100 sudo kubeadm reset +ssh -i ~/.vagrant.d/insecure_private_key vagrant@192.168.100.101 sudo kubeadm reset +ssh -i ~/.vagrant.d/insecure_private_key vagrant@192.168.100.102 sudo kubeadm reset +``` + +### Testing etcd with certs + +`etcd_certs` dir has suitable certs for local testing, assuming etcd running on the same host as kube master components. + +Setup etcd on host-00: +```sh +$ vagrant up +$ ssh -i ~/.vagrant.d/insecure_private_key vagrant@192.168.100.100 + +vagrant@host-00:~$ sudo docker run -d -v /vagrant/etcd_certs:/certs -p 2379:2379 -p 2380:2380 -v /tmp/etcd-data.tmp:/etcd-data --name etcd gcr.io/etcd-development/etcd:v3.3.2 /usr/local/bin/etcd --name s1 --data-dir /etcd-data --listen-client-urls https://0.0.0.0:2379 --advertise-client-urls https://127.0.0.1:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://0.0.0.0:2380 --initial-cluster s1=http://0.0.0.0:2380 --initial-cluster-token tkn --cert-file=/certs/server.pem --key-file=/certs/server-key.pem --client-cert-auth --trusted-ca-file=/certs/ca.pem +``` + +``` +$ kupo up -c cluster-external-etcd.yml +$ export KUBECONFIG=~/.kupo/192.168.100.100 +$ kubectl get nodes +``` + + ## License Copyright (c) 2018 Kontena, Inc. diff --git a/examples/vagrant/cluster-external-etcd.yml b/examples/vagrant/cluster-external-etcd.yml new file mode 100644 index 000000000..597e73ff2 --- /dev/null +++ b/examples/vagrant/cluster-external-etcd.yml @@ -0,0 +1,37 @@ +hosts: + - address: 192.168.100.100 + private_address: 192.168.100.100 # just to advertise correct ip with vagrant + user: vagrant + role: master + ssh_key_path: ~/.vagrant.d/insecure_private_key + container_runtime: docker + - address: 192.168.100.101 + user: vagrant + role: worker + ssh_key_path: ~/.vagrant.d/insecure_private_key + container_runtime: docker + - address: 192.168.100.102 + user: vagrant + role: worker + ssh_key_path: ~/.vagrant.d/insecure_private_key + container_runtime: docker +network: + pod_network_cidr: 10.32.0.0/16 + trusted_subnets: + - 192.168.100.0/24 +etcd: + endpoints: + - https://127.0.0.1:2379 + certificate: ./etcd_certs/client.pem + key: ./etcd_certs/client-key.pem + ca_certificate: ./etcd_certs/ca.pem +addons: + ingress-nginx: + enabled: false + configmap: + map-hash-bucket-size: "128" + kured: + enabled: false + host-upgrades: + enabled: false + interval: 7d diff --git a/examples/vagrant/etcd_certs/ca-config.json b/examples/vagrant/etcd_certs/ca-config.json new file mode 100644 index 000000000..f9e254bc9 --- /dev/null +++ b/examples/vagrant/etcd_certs/ca-config.json @@ -0,0 +1,35 @@ +{ + "signing": { + "default": { + "expiry": "43800h" + }, + "profiles": { + "server": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ] + }, + "client": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "client auth" + ] + }, + "peer": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ] + } + } + } +} diff --git a/examples/vagrant/etcd_certs/ca-csr.json b/examples/vagrant/etcd_certs/ca-csr.json new file mode 100644 index 000000000..3a2e41c6c --- /dev/null +++ b/examples/vagrant/etcd_certs/ca-csr.json @@ -0,0 +1,7 @@ +{ + "CN": "etcd", + "key": { + "algo": "rsa", + "size": 2048 + } +} diff --git a/examples/vagrant/etcd_certs/ca-key.pem b/examples/vagrant/etcd_certs/ca-key.pem new file mode 100644 index 000000000..60bbbb588 --- /dev/null +++ b/examples/vagrant/etcd_certs/ca-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEApThaS0tQBtlLoPcAXDie2RSjm53TWyNbRMmXWvAfzDLhoIRd +cUy41+NX5LUilspYivlUASbpriJ7YPkqQJFlnqsRV9mtaAH/q7vhE5hv/YcjNoso +bwtwFEjMkkzRIN6fLFlQveKUOyxyPAEXX//d7u0L5dCMBufCN+/C7MYpjb+0k4Cj +Jjh0NkG785NhwHjEI5yiFw6yxXQ1zWXeILb2VDZLF1Sj4qzekG7I8sJlpj2jrUP3 +kt0kSkS8JQdJfXD5RvYTP7aD1Wzl9K4aPBtQkXk86dVESZAP/tSrz1JqFTcOZekO +JaLig0T+nHKxjbZgqGRHJR6v+rutI6lCL2K/NQIDAQABAoIBAF2rJC3ZxcYMPX9F +abPe+deyhsr74E6kLeNCsweAaVaYZihdwqgwSf9DSZpFQxXgI/CuR/zbNdJehDpH +KLgwdj9NVujKZTA5Kd0QCBvW6W7/xWvv2v2Rq3okh65N1KZg8DbxcAAnS8h8e0sf +h0QKyKTjSUKCSFDF5etWh9k6w2Yzpqhw9KyPPkShfv7LvCSJoTwcS/5U4E9WHlrp +9JPJBfrdOO6qDNZuiyq0OIYaocLqKISL6RNXMECZo4XXPCXQe8rBvu9/Hqm8ymAj +whQebB2ab9UjfUDJ5i5soP5pZM92HslS14NUYMtsm5pkYnBKwI9u7zkp3exxbJh+ +UlC1IYECgYEAwXsXcvvFlum5gwjCJAImeTLydv+nrN/r4e9ah5jp2qCCPUp7LGO3 +z3TQg4tyJdLqNxREMLfIvUJ7/M67i3m7lOXRCzTCcSSJDbeVEA8GnFr6k89w1Vx8 +o2CplDKavs8fae7cCEst51GoY22eSWJpNJmZlUK1zKFGXdANEsasLaECgYEA2puC +uM5Ldh8J6NbKMW3VWZxePTxJKfw5tRJw5qNEiP/rc4viHgLvAyblBEF/8XYMgj+M +qs/AKRqi2Cid0RCtna6yBlUXC7umL08LpNvSsFAbshj4gvztyLxT0QQyT3xIejnX +mTd3cKusVgN/jNcaicEd5QdTdZwBw2yyhilZYRUCgYBTHggS02oszLdvPUH5qhrR +EjvrNyTXNVLmOqcPfXdo/m802VxU03nRW4OAH1WoPhV0F/a7XxiThY4yKrWS03bs +HlZRlBa7+FAQXn9g6LOUU9k1ynXUkujQXuQ60Ap+UghSv9Qyk+liaEgIfrzzFZ3Q +hPPflUr4X4gVIR2cpVCuQQKBgF+CW/2UHAISk4jH9vTbkMAjAkVsxmZyjV/gG7WY +lYPplBwafIMyMuzEnIBcgaKkOdjaHQIv3DvZKFaagEDsMX5X6e28VWJJ7NR0i5jO +deGUTlVkdYb9LIJsTY5fb/+sRBoqlVialDOEvbmis6J8BFs1JXN/3OXgcCDKp8DX +5fX1AoGAGI6DGYq1+bQ2Hj9aUtoPQeb+s4aKU3KdPOjhRzT41Z7LjhkBsVkqwCRN +c5bVo6AZg6xcHAvhdeAQwTF06f3EpFIYbQDwaE2VIKqbcaynfIcrJC/PqhYpssDu +n64Tc1qa27JJge4PC6K0ImkSS1Uw9x8m0Pr5tYX7ChzMibdohlc= +-----END RSA PRIVATE KEY----- diff --git a/examples/vagrant/etcd_certs/ca.csr b/examples/vagrant/etcd_certs/ca.csr new file mode 100644 index 000000000..f86238655 --- /dev/null +++ b/examples/vagrant/etcd_certs/ca.csr @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICVDCCATwCAQAwDzENMAsGA1UEAxMEZXRjZDCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBAKU4WktLUAbZS6D3AFw4ntkUo5ud01sjW0TJl1rwH8wy4aCE +XXFMuNfjV+S1IpbKWIr5VAEm6a4ie2D5KkCRZZ6rEVfZrWgB/6u74ROYb/2HIzaL +KG8LcBRIzJJM0SDenyxZUL3ilDsscjwBF1//3e7tC+XQjAbnwjfvwuzGKY2/tJOA +oyY4dDZBu/OTYcB4xCOcohcOssV0Nc1l3iC29lQ2SxdUo+Ks3pBuyPLCZaY9o61D +95LdJEpEvCUHSX1w+Ub2Ez+2g9Vs5fSuGjwbUJF5POnVREmQD/7Uq89SahU3DmXp +DiWi4oNE/pxysY22YKhkRyUer/q7rSOpQi9ivzUCAwEAAaAAMA0GCSqGSIb3DQEB +CwUAA4IBAQBbje7Bb5yXGi1UzYgtVucLl6EL676IJ+NTahTU4YEKc81HCi8zQ0B/ +EFFXDafdpwmDExPKlnK4yKGFoobIyFOyZ3j05OVUd6Cpuwoxz9UCL25+TOyWR+OO +6b4tzf2hB6JizgZJhkfnLWg+uOAdyp50kkpaTiOUl6dLlmVo/9RtrLFyvivp2M+B +SLEoT853/Jj9JXiWxo+mAU3Xa4cx7VZR4WP7yK+WPTsM3cij8MaDIz0HRjZCOK8L +3hC+c4dzP1QYjokNTQQHw44Tqnr95+0jm+i41gLRGJY2+80pr8BnwIPfKzoTHvOI +yhrbXbgGEpWeT7y3bdVntrsyc6zmVEnq +-----END CERTIFICATE REQUEST----- diff --git a/examples/vagrant/etcd_certs/ca.pem b/examples/vagrant/etcd_certs/ca.pem new file mode 100644 index 000000000..be62d2358 --- /dev/null +++ b/examples/vagrant/etcd_certs/ca.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC7jCCAdagAwIBAgIUDjOJYLq55ckKIGxWdT3AbAi8NWUwDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAxMEZXRjZDAeFw0xODAzMTUwNzM2MDBaFw0yMzAzMTQwNzM2 +MDBaMA8xDTALBgNVBAMTBGV0Y2QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQClOFpLS1AG2Uug9wBcOJ7ZFKObndNbI1tEyZda8B/MMuGghF1xTLjX41fk +tSKWyliK+VQBJumuIntg+SpAkWWeqxFX2a1oAf+ru+ETmG/9hyM2iyhvC3AUSMyS +TNEg3p8sWVC94pQ7LHI8ARdf/93u7Qvl0IwG58I378LsximNv7STgKMmOHQ2Qbvz +k2HAeMQjnKIXDrLFdDXNZd4gtvZUNksXVKPirN6QbsjywmWmPaOtQ/eS3SRKRLwl +B0l9cPlG9hM/toPVbOX0rho8G1CReTzp1URJkA/+1KvPUmoVNw5l6Q4louKDRP6c +crGNtmCoZEclHq/6u60jqUIvYr81AgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP +BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBT8f31uN+FecEg3rrtaECc82LmLvzAN +BgkqhkiG9w0BAQsFAAOCAQEAKHJkNwLkY8d6ux/VWQqNWusEUv92pGeKrqSeRvVZ +x8UjxI1Tg5oaeft2K1P7+uyOQcTulOYF37tlrEcdUAP9GMLzgjLyUQoUujY+CdGn +Gt9yy0k9lVA536bY+sLcy2reyMgrM0YO8Hvm4oVsLBQdKGHCE9f2qa7Rh1fMTez5 +dH30twNeDuV35ABS1svw0n5Lb1rTxZEw3z/jsJRL9nLWiKJhU60uCGZUrmfH1cqV +3IH6OV75GIRkRvCmNaWr9LekxwaO5M+SxgO1B02jgP3vs+cyJJ02isgp1a0qXrpH +y0k1AjC6vE+iXdCcgkzfgaaz0qNK72xXtW/WIrOLbpx1JQ== +-----END CERTIFICATE----- diff --git a/examples/vagrant/etcd_certs/client-key.pem b/examples/vagrant/etcd_certs/client-key.pem new file mode 100644 index 000000000..dcbbbb1a0 --- /dev/null +++ b/examples/vagrant/etcd_certs/client-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIFSxEpZ1fota8RayFb6F9EPNH812RswALBD/+6ObGWAcoAoGCCqGSM49 +AwEHoUQDQgAEsbRt+xsQI6vhpePlK0kqxfxhPBy4FztvL9Dkv0L9N/XIfKz5D09J ++rY+ITzk4/cZ/B1v8ulDqvLw8O9IySjnyw== +-----END EC PRIVATE KEY----- diff --git a/examples/vagrant/etcd_certs/client.csr b/examples/vagrant/etcd_certs/client.csr new file mode 100644 index 000000000..6ab0a6c85 --- /dev/null +++ b/examples/vagrant/etcd_certs/client.csr @@ -0,0 +1,7 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHLMHMCAQAwETEPMA0GA1UEAxMGY2xpZW50MFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAEsbRt+xsQI6vhpePlK0kqxfxhPBy4FztvL9Dkv0L9N/XIfKz5D09J+rY+ +ITzk4/cZ/B1v8ulDqvLw8O9IySjny6AAMAoGCCqGSM49BAMCA0gAMEUCIQCxm7EJ +6xTgw6XVj8BmwLP6iU4JSELyllIDXfwyZ6Go6QIgS++vFMoy77zYIlzlhcOfyyrG +b5JKN4Ot2qARVoY5IyY= +-----END CERTIFICATE REQUEST----- diff --git a/examples/vagrant/etcd_certs/client.json b/examples/vagrant/etcd_certs/client.json new file mode 100644 index 000000000..e852fa937 --- /dev/null +++ b/examples/vagrant/etcd_certs/client.json @@ -0,0 +1,7 @@ +{ + "CN": "client", + "key": { + "algo": "ecdsa", + "size": 256 + } +} diff --git a/examples/vagrant/etcd_certs/client.pem b/examples/vagrant/etcd_certs/client.pem new file mode 100644 index 000000000..8bad4f717 --- /dev/null +++ b/examples/vagrant/etcd_certs/client.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICWDCCAUCgAwIBAgIUCC5owiVgzjD9lpqFdDFpO94j+C4wDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAxMEZXRjZDAeFw0xODAzMTUwNzM3MDBaFw0yMzAzMTQwNzM3 +MDBaMBExDzANBgNVBAMTBmNsaWVudDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BLG0bfsbECOr4aXj5StJKsX8YTwcuBc7by/Q5L9C/Tf1yHys+Q9PSfq2PiE85OP3 +Gfwdb/LpQ6ry8PDvSMko58ujdTBzMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAK +BggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBQv1vpTiEJ68g6PBBrT +oaoICEPMmzAfBgNVHSMEGDAWgBT8f31uN+FecEg3rrtaECc82LmLvzANBgkqhkiG +9w0BAQsFAAOCAQEAIuYXD9kNpUe084mx6sp79EwY/BfFdszrpRWXmZw6kwAc3KXM +D0YPaRWxe1V7DKffU00ByrjFNqdG5WQpNhRjipKQMfvBDXFgHmjhPDwcN3IASNlS +Q9eLlA9i6njpFGBKeb/LcyjxvD7TH5rTtINFr+nykE1oo5816hgJMPdcWfHtvMcF +2HIenxLjgYJOILlvkZzxbUr7qq3pJLSud+WtM80l4EleCrR8HM0ieLaVfIPpX1nK +Z7Zcxg1bP/LF5HlFR6MFK7VFc3SLJN0BYdXaYBKp1wDnI2+i6V0PCu0X7thNAHOs +FGxVoXTQU/MzGuwuRn3ViEt2ObkCCvRffAfrXw== +-----END CERTIFICATE----- diff --git a/examples/vagrant/etcd_certs/config.json b/examples/vagrant/etcd_certs/config.json new file mode 100644 index 000000000..08a09c33c --- /dev/null +++ b/examples/vagrant/etcd_certs/config.json @@ -0,0 +1,13 @@ +{ + "CN": "127.0.0.1", + "hosts": [ + "127.0.0.1" + ], + "key": { + "algo": "ecdsa", + "size": 256 + }, + "names": [ + ] +} + diff --git a/examples/vagrant/etcd_certs/server-key.pem b/examples/vagrant/etcd_certs/server-key.pem new file mode 100644 index 000000000..6b5e31151 --- /dev/null +++ b/examples/vagrant/etcd_certs/server-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIN/o4L15HcxcnQ2R5paUMLnvAZ77Z5AkBSren0GcGyanoAoGCCqGSM49 +AwEHoUQDQgAE3KxCzMvETrNSoaqvWUmo8McZrrMngbRlo3Kjacs2Inl4n3Ikq3Pg +CID9lN9Rn+pr7Y34KwbuUYj3KMJ5YVjbyg== +-----END EC PRIVATE KEY----- diff --git a/examples/vagrant/etcd_certs/server.csr b/examples/vagrant/etcd_certs/server.csr new file mode 100644 index 000000000..09f41a275 --- /dev/null +++ b/examples/vagrant/etcd_certs/server.csr @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHyMIGYAgEAMBQxEjAQBgNVBAMTCTEyNy4wLjAuMTBZMBMGByqGSM49AgEGCCqG +SM49AwEHA0IABNysQszLxE6zUqGqr1lJqPDHGa6zJ4G0ZaNyo2nLNiJ5eJ9yJKtz +4AiA/ZTfUZ/qa+2N+CsG7lGI9yjCeWFY28qgIjAgBgkqhkiG9w0BCQ4xEzARMA8G +A1UdEQQIMAaHBH8AAAEwCgYIKoZIzj0EAwIDSQAwRgIhAKNIsF2gZGIwlKLqqiyE +XJiCxaRcjNglvVy6mfsjDJwrAiEAqYvyLFN9UYuUK1yI/xqlXxITHXtsKBFzzlxh +y71FdUk= +-----END CERTIFICATE REQUEST----- diff --git a/examples/vagrant/etcd_certs/server.pem b/examples/vagrant/etcd_certs/server.pem new file mode 100644 index 000000000..d1ff7774e --- /dev/null +++ b/examples/vagrant/etcd_certs/server.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICeDCCAWCgAwIBAgIUE2Il1y7ZmXAYvC6v4n4s4/dBbeowDQYJKoZIhvcNAQEL +BQAwDzENMAsGA1UEAxMEZXRjZDAeFw0xODAzMTUwNzM5MDBaFw0yMzAzMTQwNzM5 +MDBaMBQxEjAQBgNVBAMTCTEyNy4wLjAuMTBZMBMGByqGSM49AgEGCCqGSM49AwEH +A0IABNysQszLxE6zUqGqr1lJqPDHGa6zJ4G0ZaNyo2nLNiJ5eJ9yJKtz4AiA/ZTf +UZ/qa+2N+CsG7lGI9yjCeWFY28qjgZEwgY4wDgYDVR0PAQH/BAQDAgWgMB0GA1Ud +JQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQW +BBQJlHeMI9ro4h5Sj39VvqKJEwpWlzAfBgNVHSMEGDAWgBT8f31uN+FecEg3rrta +ECc82LmLvzAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQB2w+uV +MUndF3KalF5b0xe5SfPsMx7ZzU3ycZ0VuYZ81rlaE5yR26KVUXcRLYmRIzuwzdR3 +BektDmXKb9vVH8y+Rh2wLMCZ+UC5TLPWyuI3cmm7MumFDT82cWq8EMlJVDxzbgzD +vJwR9Ae+TmsT2+8KHDhFLMh1YIKQKcFGhZvd9/kVPdlP0GaBq/8v/c4dGmCJuBkI +qsS2rBrev72yZZzxFdjYwyqG4Z8Xr0PRMQD/UXb+G8GA8XABByiuyz/0XaDUmpcr +/WAcYGK5ZFRUNS1ZtH2ndmKBNoN9Mxhh5LvWVBJMWrjUhkbioGbEj23WsrGfjEYR +2LPO6FgNtVlZ5BO+ +-----END CERTIFICATE----- diff --git a/lib/kupo/config.rb b/lib/kupo/config.rb index 8123b4620..842e510e6 100644 --- a/lib/kupo/config.rb +++ b/lib/kupo/config.rb @@ -2,14 +2,18 @@ require_relative 'types' require_relative 'configuration/host' require_relative 'configuration/network' +require_relative 'configuration/etcd' module Kupo class Config < Dry::Struct HOSTS_PER_DNS_REPLICA = 10 + constructor_type :schema + attribute :hosts, Types::Coercible::Array.of(Kupo::Configuration::Host) attribute :network, Kupo::Configuration::Network attribute :addons, Kupo::Types::Hash + attribute :etcd, Kupo::Configuration::Etcd # @return [Integer] def dns_replicas diff --git a/lib/kupo/config_schema.rb b/lib/kupo/config_schema.rb index b0d95d8a5..910887043 100644 --- a/lib/kupo/config_schema.rb +++ b/lib/kupo/config_schema.rb @@ -34,6 +34,12 @@ def self.messages optional(:pod_network_cidr).filled(:str?) optional(:trusted_subnets).each(type?: String) end + optional(:etcd).maybe.schema do + required(:endpoints).each(type?: String) + optional(:certificate).filled(:str?) + optional(:ca_certificate).filled(:str?) + optional(:key).filled(:str?) + end optional(:addons).value(type?: Hash) validate(network_dns_replicas: [:network, :hosts]) do |network, hosts| diff --git a/lib/kupo/configuration/etcd.rb b/lib/kupo/configuration/etcd.rb new file mode 100644 index 000000000..e01e36267 --- /dev/null +++ b/lib/kupo/configuration/etcd.rb @@ -0,0 +1,11 @@ +module Kupo::Configuration + class Etcd < Dry::Struct + constructor_type :schema + + attribute :endpoints, Kupo::Types::Array.member(Kupo::Types::String) + attribute :version, Kupo::Types::String + attribute :certificate, Kupo::Types::String + attribute :key, Kupo::Types::String + attribute :ca_certificate, Kupo::Types::String + end +end diff --git a/lib/kupo/phases/base.rb b/lib/kupo/phases/base.rb index 0553a6fec..47e046bd6 100644 --- a/lib/kupo/phases/base.rb +++ b/lib/kupo/phases/base.rb @@ -42,5 +42,13 @@ def ssh_exec_file(ssh, file) raise Kupo::ScriptExecError, "Script execution failed: #{file}" end end + + def exec_script(script, vars = {}) + file = File.realpath(File.join(__dir__, '..', 'scripts', script)) + parsed_file = Kupo::Erb.new(File.read(file)).render(vars) + ssh_exec_file(@ssh, StringIO.new(parsed_file)) + rescue Kupo::ScriptExecError + raise Kupo::ScriptExecError, "Failed to execute #{script}" + end end end \ No newline at end of file diff --git a/lib/kupo/phases/configure_host.rb b/lib/kupo/phases/configure_host.rb index 23a218679..59431931c 100644 --- a/lib/kupo/phases/configure_host.rb +++ b/lib/kupo/phases/configure_host.rb @@ -51,7 +51,9 @@ def call logger.info { "Configuring Kubernetes packages ..." } exec_script('configure-kube.sh', { - kube_version: KUBE_VERSION + kube_version: KUBE_VERSION, + kubeadm_version: ENV['KUBEADM_VERSION'] || KUBE_VERSION, + arch: @host.cpu_arch.name }) end @@ -63,14 +65,6 @@ def configure_repos # @param script [String] # @param vars [Hash] - def exec_script(script, vars = {}) - file = File.realpath(File.join(__dir__, '..', 'scripts', script)) - parsed_file = Kupo::Erb.new(File.read(file)).render(vars) - ssh_exec_file(@ssh, StringIO.new(parsed_file)) - rescue Kupo::ScriptExecError - raise Kupo::ScriptExecError, "Failed to execute #{script}" - end - def crio? @host.container_runtime == 'cri-o' end diff --git a/lib/kupo/phases/configure_master.rb b/lib/kupo/phases/configure_master.rb index a5799a5cb..9286f1d8e 100644 --- a/lib/kupo/phases/configure_master.rb +++ b/lib/kupo/phases/configure_master.rb @@ -38,22 +38,21 @@ def upgrade? end def install - sans = [@master.address, @master.private_address].compact.uniq - options = [ - "--apiserver-cert-extra-sans #{sans.join(',')}", - "--service-cidr #{@config.service_cidr}", - "--pod-network-cidr #{@config.pod_network_cidr}" - ] - if @master.container_runtime == 'cri-o' - options << "--cri-socket /var/run/crio/crio.sock" - end - if @master.private_address - options << "--apiserver-advertise-address #{@master.private_address}" - else - options << "--apiserver-advertise-address #{@master.address}" - end + cfg = generate_config + tmp_file = File.join('/tmp', 'kubeadm.cfg.' + SecureRandom.hex(16)) + @ssh.upload(StringIO.new(cfg.to_yaml), tmp_file) + + # Copy etcd certs over if needed + exec_script('configure-etcd-certs.sh', { + ca_certificate: File.read(@config.etcd.ca_certificate), + certificate: File.read(@config.etcd.certificate), + certificate_key: File.read(@config.etcd.key) + + }) if @config.etcd && @config.etcd.certificate + logger.info(@master.address) { "Initializing control plane ..." } - code = @ssh.exec("sudo kubeadm init #{options.join(' ')}") do |type, data| + + code = @ssh.exec("sudo kubeadm init --config #{tmp_file}") do |type, data| remote_output(type, data) end if code == 0 @@ -61,11 +60,48 @@ def install else raise Kupo::Error, "Initialization of control plane failed!" end - + @ssh.exec("rm #{tmp_file}") @ssh.exec('mkdir -p ~/.kube') @ssh.exec('sudo cat /etc/kubernetes/admin.conf > ~/.kube/config') end + def generate_config + + config = { + 'apiVersion' => 'kubeadm.k8s.io/v1alpha1', + 'kind' => 'MasterConfiguration', + 'kubernetesVersion' => '1.9.3', + 'apiServerCertSANs' => [@master.address, @master.private_address].compact.uniq, + 'networking' => { + 'serviceSubnet' => @config.network.service_cidr, + 'podSubnet' => @config.network.pod_network_cidr + } + } + + if @master.private_address + config['api'] = {'advertiseAddress' => @master.private_address} + else + config['api'] = {'advertiseAddress' => @master.address} + end + + if @master.container_runtime == 'cri-o' + config['criSocket'] = '/var/run/crio/crio.sock' + end + + # Only configure etcd if the external endpoints are given + if @config.etcd && @config.etcd.endpoints + config['etcd'] = { + 'endpoints' => @config.etcd.endpoints + } + + config['etcd']['certFile'] = '/etc/kupo/etcd/certificate.pem' if @config.etcd.certificate + config['etcd']['caFile'] = '/etc/kupo/etcd/ca-certificate.pem' if @config.etcd.ca_certificate + config['etcd']['keyFile'] = '/etc/kupo/etcd/certificate-key.pem' if @config.etcd.key + end + config + end + + def upgrade code = @ssh.exec("sudo kubeadm upgrade apply #{kube_component.version} -y") do |type, data| remote_output(type, data) diff --git a/lib/kupo/scripts/configure-etcd-certs.sh b/lib/kupo/scripts/configure-etcd-certs.sh new file mode 100644 index 000000000..e6863eefb --- /dev/null +++ b/lib/kupo/scripts/configure-etcd-certs.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +mkdir -p /etc/kupo/etcd +cat </etc/kupo/etcd/ca-certificate.pem +<%= ca_certificate %> +EOF +cat </etc/kupo/etcd/certificate.pem +<%= certificate %> +EOF +cat </etc/kupo/etcd/certificate-key.pem +<%= certificate_key %> +EOF diff --git a/lib/kupo/scripts/configure-kube.sh b/lib/kupo/scripts/configure-kube.sh index 2fd128603..59ac3c3e5 100755 --- a/lib/kupo/scripts/configure-kube.sh +++ b/lib/kupo/scripts/configure-kube.sh @@ -2,10 +2,15 @@ set -ex + if [ "$(kubelet --version)" = "Kubernetes v<%= kube_version %>" ]; then exit 0 fi -apt-mark unhold kubelet kubeadm kubectl -apt-get install -y kubelet=<%= kube_version %>-00 kubeadm=<%= kube_version %>-00 kubectl=<%= kube_version %>-00 -apt-mark hold kubelet kubelet kubeadm kubectl \ No newline at end of file +apt-mark unhold kubelet kubectl +apt-get install -y kubelet=<%= kube_version %>-00 kubectl=<%= kube_version %>-00 +apt-mark hold kubelet kubelet kubectl + +# Get kubeadm binary directly +curl -o /usr/bin/kubeadm https://storage.googleapis.com/kubernetes-release/release/v<%= kubeadm_version %>/bin/linux/<%= arch %>/kubeadm +chmod +x /usr/bin/kubeadm \ No newline at end of file diff --git a/lib/kupo/up_command.rb b/lib/kupo/up_command.rb index d0df914fc..b77381e42 100644 --- a/lib/kupo/up_command.rb +++ b/lib/kupo/up_command.rb @@ -25,7 +25,6 @@ def execute puts pastel.green("==> Reading instructions ...") configure(load_config(config_content)) end - def configure(config) master_hosts = master_hosts(config) signal_usage_error 'No master hosts defined' if master_hosts.size == 0 @@ -37,10 +36,13 @@ def configure(config) load_phases puts pastel.green("==> Starting to craft cluster ...") validate_hosts(config.hosts) - - handle_masters(master_hosts[0], config) - handle_workers(master_hosts[0], worker_hosts(config)) - handle_addons(master_hosts[0], config.addons) + # set workdir to the same dir where config was loaded from + # so that the certs etc. can be referenced more easily + Dir.chdir(File.dirname(config_file)) do + handle_masters(master_hosts[0], config) + handle_workers(master_hosts[0], worker_hosts(config)) + handle_addons(master_hosts[0], config.addons) + end craft_time = Time.now - start_time puts pastel.green("==> Cluster has been crafted, kupo! (took #{humanize_duration(craft_time.to_i)})") ensure @@ -132,7 +134,7 @@ def handle_masters(master, config) log_host_header(master) Phases::ConfigureHost.new(master).call Phases::ConfigureKubelet.new(master).call - Phases::ConfigureMaster.new(master, config.network).call + Phases::ConfigureMaster.new(master, config).call Phases::ConfigureClient.new(master).call Phases::ConfigureDNS.new(master, config).call Phases::ConfigureNetwork.new(master, config.network).call diff --git a/spec/kupo/phases/configure_dns_spec.rb b/spec/kupo/phases/configure_dns_spec.rb index e52d06587..760edb1e2 100644 --- a/spec/kupo/phases/configure_dns_spec.rb +++ b/spec/kupo/phases/configure_dns_spec.rb @@ -11,6 +11,7 @@ dns_replicas: config_dns_replicas, }, addons: {}, + etcd: {} ) } subject { described_class.new(master, config) } diff --git a/spec/kupo/phases/configure_master_spec.rb b/spec/kupo/phases/configure_master_spec.rb new file mode 100644 index 000000000..90a124d77 --- /dev/null +++ b/spec/kupo/phases/configure_master_spec.rb @@ -0,0 +1,109 @@ +require "kupo/config" +require "kupo/phases/configure_master" + +describe Kupo::Phases::ConfigureMaster do + let(:master) { Kupo::Configuration::Host.new(address: 'test', private_address: 'private') } + let(:config_hosts_count) { 1 } + + let(:config) { Kupo::Config.new( + hosts: (1..config_hosts_count).map { |i| Kupo::Configuration::Host.new() }, + network: { + service_cidr: '1.2.3.4/16', + pod_network_cidr: '10.0.0.0/16' + }, + addons: {}, + etcd: {} + ) } + subject { described_class.new(master, config) } + + before :each do + allow(Kupo::SSH::Client).to receive(:for_host) + end + + describe '#config_yaml' do + context 'with network configuration' do + let(:config) { Kupo::Config.new( + hosts: (1..config_hosts_count).map { |i| Kupo::Configuration::Host.new() }, + network: { + service_cidr: '1.2.3.4/16', + pod_network_cidr: '10.0.0.0/16' + }, + addons: {}, + etcd: {} + ) } + + it 'comes with correct subnets' do + config = subject.generate_config + expect(config.dig('networking', 'serviceSubnet')).to eq('1.2.3.4/16') + expect(config.dig('networking', 'podSubnet')).to eq('10.0.0.0/16') + end + + end + + it 'comes with correct master addresses' do + config = subject.generate_config + expect(config.dig('apiServerCertSANs')).to eq(['test', 'private']) + expect(config.dig('api', 'advertiseAddress')).to eq('private') + end + + it 'comes with no etcd config' do + config = subject.generate_config + expect(config.dig('etcd')).to be_nil + expect(config.dig('etcd', 'endpoints')).to be_nil + expect(config.dig('etcd', 'version')).to be_nil + end + + context 'with etcd endpoint configuration' do + let(:config) { Kupo::Config.new( + hosts: (1..config_hosts_count).map { |i| Kupo::Configuration::Host.new() }, + network: {}, + addons: {}, + etcd: { + endpoints: ['ep1', 'ep2'] + } + ) } + + it 'comes with proper etcd endpoint config' do + config = subject.generate_config + expect(config.dig('etcd', 'endpoints')).to eq(['ep1', 'ep2']) + end + end + + context 'with etcd certificate configuration' do + + let(:config) { Kupo::Config.new( + hosts: (1..config_hosts_count).map { |i| Kupo::Configuration::Host.new() }, + network: {}, + addons: {}, + etcd: { + endpoints: ['ep1', 'ep2'], + ca_certificate: 'ca-certificate.pem', + certificate: 'certificate.pem', + key: 'key.pem' + } + ) } + + it 'comes with proper etcd certificate config' do + config = subject.generate_config + expect(config.dig('etcd', 'caFile')).to eq('/etc/kupo/etcd/ca-certificate.pem') + expect(config.dig('etcd', 'certFile')).to eq('/etc/kupo/etcd/certificate.pem') + expect(config.dig('etcd', 'keyFile')).to eq('/etc/kupo/etcd/certificate-key.pem') + end + end + + context 'with cri-o configuration' do + let(:master) { Kupo::Configuration::Host.new(address: 'test', container_runtime: 'cri-o') } + let(:config) { Kupo::Config.new( + hosts: (1..config_hosts_count).map { |i| Kupo::Configuration::Host.new() }, + network: {}, + addons: {}, + etcd: {} + ) } + + it 'comes with proper etcd endpoint config' do + config = subject.generate_config + expect(config.dig('criSocket')).to eq('/var/run/crio/crio.sock') + end + end + end +end