From 924bb3b41a4126a4c89e4be8779ac014f0d41151 Mon Sep 17 00:00:00 2001 From: Michael Schubert Date: Mon, 22 Jan 2018 13:52:37 +0100 Subject: [PATCH] Refactor kube-spawn --- .gitignore | 1 - Makefile | 4 +- README.md | 2 +- cmd/kube-spawn-runc/README.md | 27 - cmd/kube-spawn-runc/kube-spawn-runc.go | 85 --- cmd/kube-spawn/create.go | 222 ++------ cmd/kube-spawn/destroy.go | 43 +- cmd/kube-spawn/kube-spawn.go | 60 ++- cmd/kube-spawn/list.go | 57 +- cmd/kube-spawn/restart.go | 58 --- cmd/kube-spawn/start.go | 161 +----- cmd/kube-spawn/stop.go | 131 +---- cmd/kube-spawn/up.go | 69 +-- doc/devel/release.md | 2 +- doc/troubleshooting.md | 5 - pkg/bootstrap/cninet.go | 2 - pkg/bootstrap/download.go | 27 +- pkg/bootstrap/node.go | 108 +--- pkg/bootstrap/scripts.go | 109 ---- pkg/cache/cache.go | 15 + pkg/cluster/cluster.go | 660 ++++++++++++++++++++++++ pkg/cluster/clusterfiles.go | 131 +++++ pkg/config/defaults.go | 253 --------- pkg/config/types.go | 106 ---- pkg/config/utils.go | 82 --- pkg/distribution/registry.go | 130 ----- pkg/machinectl/machinectl.go | 173 +++++++ pkg/machinetool/machinetool.go | 96 ---- pkg/multiprint/multiprint.go | 85 +++ pkg/nspawntool/binds.go | 43 -- pkg/nspawntool/kubeadm.go | 109 ---- pkg/nspawntool/run.go | 77 +-- pkg/script/docker-daemon-config.go | 13 - pkg/script/docker-kubeadm-extra-args.go | 7 - pkg/script/kubeadm-bootstrap.go | 37 -- pkg/script/kubeadm-config.go | 29 -- pkg/script/kubeadm-extra-args.go | 43 -- pkg/script/kubeadm-extra-runtime.go | 17 - pkg/script/kubelet-tmpfiles.go | 5 - pkg/script/rktlet-service.go | 17 - pkg/script/script.go | 16 - pkg/script/weave-networkd.go | 10 - pkg/utils/fs/fs.go | 4 +- pkg/utils/kubernetes.go | 131 ----- pkg/utils/terminal.go | 31 ++ pkg/utils/version.go | 13 - 46 files changed, 1337 insertions(+), 2169 deletions(-) delete mode 100644 cmd/kube-spawn-runc/README.md delete mode 100644 cmd/kube-spawn-runc/kube-spawn-runc.go delete mode 100644 cmd/kube-spawn/restart.go delete mode 100644 pkg/bootstrap/scripts.go create mode 100644 pkg/cache/cache.go create mode 100644 pkg/cluster/cluster.go create mode 100644 pkg/cluster/clusterfiles.go delete mode 100644 pkg/config/defaults.go delete mode 100644 pkg/config/types.go delete mode 100644 pkg/config/utils.go delete mode 100644 pkg/distribution/registry.go create mode 100644 pkg/machinectl/machinectl.go delete mode 100644 pkg/machinetool/machinetool.go create mode 100644 pkg/multiprint/multiprint.go delete mode 100644 pkg/nspawntool/binds.go delete mode 100644 pkg/nspawntool/kubeadm.go delete mode 100644 pkg/script/docker-daemon-config.go delete mode 100644 pkg/script/docker-kubeadm-extra-args.go delete mode 100644 pkg/script/kubeadm-bootstrap.go delete mode 100644 pkg/script/kubeadm-config.go delete mode 100644 pkg/script/kubeadm-extra-args.go delete mode 100644 pkg/script/kubeadm-extra-runtime.go delete mode 100644 pkg/script/kubelet-tmpfiles.go delete mode 100644 pkg/script/rktlet-service.go delete mode 100644 pkg/script/script.go delete mode 100644 pkg/script/weave-networkd.go delete mode 100644 pkg/utils/kubernetes.go create mode 100644 pkg/utils/terminal.go delete mode 100644 pkg/utils/version.go diff --git a/.gitignore b/.gitignore index 1da55f27..3e18bec3 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,3 @@ _testmain.go /cni-noop /cnispawn /kube-spawn -/kube-spawn-runc diff --git a/Makefile b/Makefile index a36a8cfc..16bc8bdb 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,6 @@ all: kube-spawn-build else all: - go build -o kube-spawn-runc ./cmd/kube-spawn-runc go build -o kube-spawn \ -ldflags "-X main.version=$(VERSION)" \ ./cmd/kube-spawn @@ -32,7 +31,6 @@ dep: clean: rm -f \ kube-spawn \ - kube-spawn-runc install: - install kube-spawn kube-spawn-runc "$(BINDIR)" + install kube-spawn "$(BINDIR)" diff --git a/README.md b/README.md index 9bd1bb24..37338ae3 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ All nodes can be seen with `machinectl list`, `machinectl shell` can be used to sudo machinectl shell root@kubespawn0 ``` -The password is `k8s`. +The password is `root`. ## Command Usage diff --git a/cmd/kube-spawn-runc/README.md b/cmd/kube-spawn-runc/README.md deleted file mode 100644 index ae1aa065..00000000 --- a/cmd/kube-spawn-runc/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# kube-spawn-runc - -`kube-spawn-runc` is a wrapper around `runc` to add the `--no-new-keyring` flag on `run` and `create` commands. -We have to do this as the keyring syscalls as used by Docker are forbidden by -system-nspawn. - -To use the wrapper create a custom runtime in `/etc/docker/daemon.json` and activate it as the default-runtime. -For how to do this refer to [this example](https://github.com/kinvolk/kube-spawn/blob/master/etc/daemon.json#L3-L6). - -## Debugging - -You can set the following environment variables on this wrapper: - -- `KUBE_SPAWN_RUNC_BINARY_PATH` : path to runc binary. By default we find `docker-runc` in PATH -- `KUBE_SPAWN_RUNC_LOG_PATH` : path to the log file. By default there are no logs - -To run with an environment variable: - -`systemctl edit containerd.service` - -now add: - -``` -[Service] -Environment=KUBE_SPAWN_RUNC_LOG_PATH=... -Environment=KUBE_SPAWN_RUNC_BINARY_PATH=... -``` diff --git a/cmd/kube-spawn-runc/kube-spawn-runc.go b/cmd/kube-spawn-runc/kube-spawn-runc.go deleted file mode 100644 index b80cddbd..00000000 --- a/cmd/kube-spawn-runc/kube-spawn-runc.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - - "github.com/kinvolk/kube-spawn/pkg/utils" -) - -var ( - runcPath string = os.Getenv("KUBE_SPAWN_RUNC_BINARY_PATH") - logPath string = os.Getenv("KUBE_SPAWN_RUNC_LOG_PATH") -) - -func main() { - var newArgs []string - - if logPath != "" { - fd, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0660) - if err != nil { - log.Printf("error opening logs, skipping: %s", err) - } - log.SetOutput(fd) - defer fd.Close() - } - - printToLogpath(fmt.Sprintf("old args: %#v", os.Args[1:])) - - for _, a := range os.Args[1:] { - newArgs = append(newArgs, a) - if a == "create" || a == "run" { - newArgs = append(newArgs, "--no-new-keyring") - } - } - - printToLogpath(fmt.Sprintf("new args: %#v", newArgs)) - - if runcPath == "" { - var err error - runcPath, err = exec.LookPath("docker-runc") - if err != nil { - // unable to find default - log.Fatal(err) - } - } - cmd := exec.Command(runcPath, newArgs...) - - // Selectively pass Stdout/Stderr, by determining if they are terminal or not. - // If we always pass them, interactive mode of connection to containers - // will fail with error messages like "container not started". - // If we never pass them, then "kubectl logs" won't be able to print any logs. - if !utils.IsTerminal(os.Stdout.Fd()) && !utils.IsTerminal(os.Stderr.Fd()) { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - - if err := cmd.Run(); err != nil { - log.Fatal(err) - } -} - -// print string only when logPath is explicitly set -func printToLogpath(fmtStr string) { - if logPath != "" { - log.Printf("%s", fmtStr) - } -} diff --git a/cmd/kube-spawn/create.go b/cmd/kube-spawn/create.go index bfc344de..a3036164 100644 --- a/cmd/kube-spawn/create.go +++ b/cmd/kube-spawn/create.go @@ -18,38 +18,27 @@ package main import ( "log" - "os/exec" "path" - "sync" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" - "golang.org/x/sys/unix" "github.com/kinvolk/kube-spawn/pkg/bootstrap" - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/kinvolk/kube-spawn/pkg/utils" + "github.com/kinvolk/kube-spawn/pkg/cache" + "github.com/kinvolk/kube-spawn/pkg/cluster" "github.com/kinvolk/kube-spawn/pkg/utils/fs" ) var ( createCmd = &cobra.Command{ Use: "create", - Short: "Generate the environment for a cluster", - Long: `Generate the environment for a cluster. -If you change 'kspawn.toml' this needs to be run again.`, + Short: "Create a new cluster environment", Example: ` -# Create an environment to run a 3 node cluster initialized with components from $GOPATH/k8s.io/kubernetes -$ sudo -E kube-spawn create --nodes 3 --dev -t mytag - -# Create a cluster environment using rkt as the container runtime -# You can specify paths to the binaries necessary using environment variables (in case they are not in your PATH) -$ sudo -E \ - KUBE_SPAWN_RKT_BIN=/opt/bin/rkt \ - KUBE_SPAWN_RKTLET_BIN=/opt/bin/rktlet \ - KUBE_SPAWN_RKT_STAGE1_IMAGE=/opt/bin/stage1-coreos.aci \ - kube-spawn create --container-runtime rkt`, +# Create an environment to run a 3 node cluster from a hyperkube image +$ sudo -E kube-spawn create --nodes 3 --hyperkube-image 10.22.0.1:5000/my-hyperkube-amd64-image:my-test + +# Create a cluster using rkt as the container runtime +$ sudo kube-spawn create --container-runtime rkt --rktlet-binary-path $GOPATH/src/github.com/kubernetes-incubator/rktlet/bin/rktlet`, Run: runCreate, } ) @@ -57,179 +46,62 @@ $ sudo -E \ func init() { kubespawnCmd.AddCommand(createCmd) - // do not set defaults here - // intead use: - // pkg/config/defaults.go - // and call from the if uninitialized {} block below - // - createCmd.Flags().StringP("container-runtime", "r", "", "runtime to use for the spawned cluster (docker or rkt)") - createCmd.Flags().String("kubernetes-version", "", `version kubernetes used to initialize the cluster. Irrelevant if used with --dev. Only accepts semantic version strings like "v1.8.5"`) - createCmd.Flags().StringP("hyperkube-tag", "t", "latest", `Docker tag of the hyperkube image to use. Only with --dev`) - createCmd.Flags().Bool("dev", false, "create a cluster from a local build of Kubernetes") - createCmd.Flags().IntP("nodes", "n", 0, "number of nodes to spawn") - createCmd.Flags().StringP("image", "i", "", "base image for nodes") - createCmd.Flags().String("cni-plugin-dir", "/opt/cni/bin", "path to directory with CNI plugins") - viper.BindPFlags(createCmd.Flags()) - - viper.BindEnv("runtime-config.rkt.rkt-bin", "KUBE_SPAWN_RKT_BIN") - viper.BindEnv("runtime-config.rkt.stage1-image", "KUBE_SPAWN_RKT_STAGE1_IMAGE") - viper.BindEnv("runtime-config.rkt.rktlet-bin", "KUBE_SPAWN_RKTLET_BIN") - - viper.BindEnv("runtime-config.crio.crio-bin", "KUBE_SPAWN_CRIO_BIN") - viper.BindEnv("runtime-config.crio.runc-bin", "KUBE_SPAWN_RUNC_BIN") - viper.BindEnv("runtime-config.crio.conmon-bin", "KUBE_SPAWN_CONMON_BIN") + createCmd.Flags().String("container-runtime", "docker", "Runtime to use for the cluster (can be docker or rkt)") + createCmd.Flags().String("machinectl-image", "coreos", "Name of the machinectl image to use for the kube-spawn containers") + createCmd.Flags().String("kubernetes-version", "v1.9.3", "Kubernetes version to install") + createCmd.Flags().String("kubernetes-source-dir", "", "Path to directory with Kubernetes sources") + createCmd.Flags().String("hyperkube-image", "", "Kubernetes hyperkube image to use (if unset, upstream k8s is installed)") + createCmd.Flags().String("cni-plugin-dir", "/opt/cni/bin", "Path to directory with CNI plugins") + createCmd.Flags().String("rkt-binary-path", "/usr/local/bin/rkt", "Path to rkt binary") + createCmd.Flags().String("rkt-stage1-image-path", "/usr/local/bin/stage1-coreos.aci", "Path to rkt stage1-coreos.aci image") + createCmd.Flags().String("rktlet-binary-path", "/usr/local/bin/rktlet", "Path to rktlet binary") - config.SetDefaults_Viper(viper.GetViper()) + viper.BindPFlags(createCmd.Flags()) } func runCreate(cmd *cobra.Command, args []string) { - if unix.Geteuid() != 0 { - log.Fatalf("non-root user cannot create clusters. abort.") - } - if len(args) > 0 { - log.Fatalf("too many arguments: %v", args) + log.Fatalf("Command create doesn't take arguments, got: %v", args) } - doCreate() -} - -func doCreate() { - cfg, err := config.LoadConfig() - if err != nil { - // ignore if config not found - // it means we started from scratch and need to generate one - if !config.IsNotFound(err) { - log.Fatal(errors.Wrap(err, "unable to load config")) - } - } - log.Printf("creating cluster environment %q", cfg.Name) - if cfg.DevCluster { - log.Printf("spawning from local kubernetes build") - } else { - log.Printf("spawning kubernetes version %q", cfg.KubernetesVersion) - } - if cfg.ContainerRuntime != config.RuntimeDocker { - log.Printf("spawning with container runtime %q", cfg.ContainerRuntime) + kubespawnDir := viper.GetString("dir") + clusterName := viper.GetString("cluster-name") + clusterDir := path.Join(kubespawnDir, "clusters", clusterName) + if exists, err := fs.PathExists(clusterDir); err != nil { + log.Fatalf("Failed to stat directory %q: %s\n", err) + } else if exists { + log.Fatalf("Cluster directory exists already at %q", clusterDir) } - if utils.CheckVersionConstraint(cfg.KubernetesVersion, "<1.7.5") { - log.Fatal("minimum supported version is 'v1.7.5'") - } - - // download files into cache - if !cfg.DevCluster { - if err := bootstrap.DownloadK8sBins(cfg); err != nil { - log.Fatal(err) - } - } - if err := bootstrap.DownloadSocatBin(cfg); err != nil { - log.Fatal(err) + // TODO + if err := bootstrap.PathSupportsOverlay(kubespawnDir); err != nil { + log.Fatalf("Unable to use overlayfs on underlying filesystem of %q: %v", kubespawnDir, err) } - if err := config.SetDefaults_Kubernetes(cfg); err != nil { - log.Fatal(errors.Wrap(err, "error settting kubernetes defaults")) - } - - if err := config.SetDefaults_BindmountConfiguration(cfg); err != nil { - log.Fatal(errors.Wrap(err, "error setting bindmount defaults")) - } - - if err := config.SetDefaults_RuntimeConfiguration(cfg); err != nil { - log.Fatal(errors.Wrap(err, "error setting container runtime defaults")) - } - - // TODO: the docker-runc wrapper ensures `--no-new-keyring` is - // set, otherwise Docker will attempt to use keyring syscalls - // which are not allowed in systemd-nspawn containers. It can - // be removed once we require systemd v235 or later. We then - // will be able to whitelist the required syscalls; see: - // https://github.com/systemd/systemd/pull/6798 - kubeSpawnRuncPath := "kube-spawn-runc" - if !utils.IsExecBinary(kubeSpawnRuncPath) { - if lp, err := exec.LookPath(kubeSpawnRuncPath); err != nil { - log.Fatal(errors.Wrap(err, "kube-spawn-runc binary not found but required")) - } else { - kubeSpawnRuncPath = lp - } - } - cfg.Copymap = append(cfg.Copymap, config.Pathmap{ - Dst: "/usr/bin/kube-spawn-runc", - Src: kubeSpawnRuncPath, - }) - - if cfg.Image == config.DefaultBaseImage { - if err := bootstrap.PrepareCoreosImage(); err != nil { - log.Fatal(errors.Wrap(err, "error setting up default base image")) - } - } - - // TODO: check config + env - // - check version - // - check version of k8s binaries - // - cni bridge works - // - base image exists - // - ??? coreos version correct - // - overlayfs works - // - conntrack hashsize - // - iptables setup correct - // - selinux setup correct - // if err := checks.RunCreateChecks(cfg); err != nil { - // log.Fatal(errors.Wrap(err, "check failed")) - // } - - log.Print("ensuring environment") - if err := bootstrap.EnsureRequirements(cfg); err != nil { - log.Fatal(err) - } - - if err := bootstrap.PathSupportsOverlay(cfg.KubeSpawnDir); err != nil { - log.Fatalf("unable to use overlayfs on %q: %v. Try to pass a directory with a different filesystem (like ext4 or XFS) to --dir.", cfg.KubeSpawnDir, err) + kluster, err := cluster.New(clusterDir, clusterName) + if err != nil { + log.Fatalf("Failed to create cluster object: %v", err) } - log.Print("generating scripts") - if err := bootstrap.GenerateScripts(cfg); err != nil { - log.Fatal(errors.Wrap(err, "error generating files")) + clusterSettings := &cluster.ClusterSettings{ + KubernetesVersion: viper.GetString("kubernetes-version"), + KubernetesSourceDir: viper.GetString("kubernetes-source-dir"), + CNIPluginDir: viper.GetString("cni-plugin-dir"), + ContainerRuntime: viper.GetString("container-runtime"), + RktBinaryPath: viper.GetString("rkt-binary-path"), + RktStage1ImagePath: viper.GetString("rkt-stage1-image-path"), + RktletBinaryPath: viper.GetString("rktlet-binary-path"), + HyperkubeImage: viper.GetString("hyperkube-image"), } - log.Print("copying files into environment") - - copyFailed := false - copyErrChan := make(chan error) - go func() { - for { - select { - case err, ok := <-copyErrChan: - if !ok { - return - } - copyFailed = true - log.Printf("%v", err) - } - } - }() - - var wg sync.WaitGroup - wg.Add(len(cfg.Copymap)) - - for _, pm := range cfg.Copymap { - go func(dst, src string) { - defer wg.Done() - // dst path is relative to the machine rootfs - dst = path.Join(cfg.KubeSpawnDir, cfg.Name, "rootfs", dst) - if copyErr := fs.CopyFile(src, dst); copyErr != nil { - copyErrChan <- errors.Wrapf(err, "failed to copy file %q -> %q", src, dst) - } - }(pm.Dst, pm.Src) + clusterCache, err := cache.New(path.Join(kubespawnDir, "cache")) + if err != nil { + log.Fatalf("Failed to create cache object: %v", err) } - wg.Wait() - close(copyErrChan) - - if copyFailed { - log.Fatalf("Copying necessary files didn't succeed, aborting") + if err := kluster.Create(clusterSettings, clusterCache); err != nil { + log.Fatalf("Failed to create cluster: %v", err) } - saveConfig(cfg) - log.Println("created cluster config") + log.Printf("Cluster %s created", clusterName) } diff --git a/cmd/kube-spawn/destroy.go b/cmd/kube-spawn/destroy.go index 9c879411..bd8ecd58 100644 --- a/cmd/kube-spawn/destroy.go +++ b/cmd/kube-spawn/destroy.go @@ -18,15 +18,12 @@ package main import ( "log" - "os" "path" - "github.com/pkg/errors" "github.com/spf13/cobra" - "golang.org/x/sys/unix" + "github.com/spf13/viper" - "github.com/kinvolk/kube-spawn/pkg/bootstrap" - "github.com/kinvolk/kube-spawn/pkg/config" + "github.com/kinvolk/kube-spawn/pkg/cluster" ) var ( @@ -44,36 +41,24 @@ func init() { } func runDestroy(cmd *cobra.Command, args []string) { - if unix.Geteuid() != 0 { - log.Fatalf("non-root user cannot destroy clusters. abort.") - } - if len(args) > 0 { - log.Fatalf("too many arguments: %v", args) + log.Fatalf("Command destroy doesn't take arguments, got: %v", args) } - cfg := loadConfig() - doDestroy(cfg) -} + kubespawnDir := viper.GetString("dir") + clusterName := viper.GetString("cluster-name") + clusterDir := path.Join(kubespawnDir, "clusters", clusterName) -func doDestroy(cfg *config.ClusterConfiguration) { - log.Printf("destroying cluster %q", cfg.Name) + kluster, err := cluster.New(clusterDir, clusterName) + if err != nil { + log.Fatalf("Failed to create cluster object: %v", err) + } - doStop(cfg, true) + log.Printf("Destroying cluster %s ...", clusterName) - cDir := path.Join(cfg.KubeSpawnDir, cfg.Name) - if err := os.RemoveAll(cDir); err != nil { - log.Fatal(errors.Wrapf(err, "error removing cluster dir at %q", cDir)) + if err := kluster.Destroy(); err != nil { + log.Fatalf("Failed to destroy cluster: %v", err) } - RemoveCniConfig() - log.Printf("%q destroyed", cfg.Name) -} -func RemoveCniConfig() { - if err := os.RemoveAll(bootstrap.VarLibCniDir); err != nil { - log.Printf("cannot remove %q: %v", bootstrap.VarLibCniDir, err) - } - if err := os.RemoveAll(bootstrap.NspawnNetPath); err != nil { - log.Printf("cannot remove %q: %v", bootstrap.NspawnNetPath, err) - } + log.Printf("Cluster %s destroyed", clusterName) } diff --git a/cmd/kube-spawn/kube-spawn.go b/cmd/kube-spawn/kube-spawn.go index 86e0572e..f3ee2b61 100644 --- a/cmd/kube-spawn/kube-spawn.go +++ b/cmd/kube-spawn/kube-spawn.go @@ -21,10 +21,9 @@ import ( "log" "os" - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/sys/unix" ) var ( @@ -43,41 +42,56 @@ You can run a release-version cluster or spawn one from your local Kubernetes re } }, } - + // set from ldflags to current git version during build version string printVersion bool + + cfgFile string ) func init() { - kubespawnCmd.PersistentFlags().StringP("dir", "d", "", "Path to kube-spawn asset directory") - kubespawnCmd.PersistentFlags().StringP("cluster-name", "c", "", "Name for the cluster") - viper.BindPFlags(kubespawnCmd.PersistentFlags()) + log.SetFlags(0) - kubespawnCmd.Flags().BoolVarP(&printVersion, "version", "V", false, "Output version information") + cobra.OnInitialize(initConfig) - kubespawnCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { - log.SetFlags(0) - } -} + kubespawnCmd.Flags().BoolVarP(&printVersion, "version", "V", false, "Output version and exit") + kubespawnCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default \"/etc/kube-spawn/config.yaml\")") + kubespawnCmd.PersistentFlags().StringP("dir", "d", "/var/lib/kube-spawn", "Path to kube-spawn asset directory") + kubespawnCmd.PersistentFlags().StringP("cluster-name", "c", "default", "Name for the cluster") -func main() { - if err := kubespawnCmd.Execute(); err != nil { - os.Exit(1) + viper.BindPFlags(kubespawnCmd.PersistentFlags()) + + kubespawnCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + cmdName := cmd.Use + if cmdName == "create" || cmdName == "destroy" || cmdName == "start" || cmdName == "stop" || cmdName == "up" { + if unix.Geteuid() != 0 { + cmd.SilenceUsage = true + return fmt.Errorf("root privileges required for command %q, aborting", cmdName) + } + } + return nil } } -func loadConfig() *config.ClusterConfiguration { - cfg, err := config.LoadConfig() - if err != nil { - log.Fatal(errors.Wrap(err, "error loading config")) +func initConfig() { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + } else { + viper.SetConfigName("config") + viper.SetConfigType("yaml") + config := fmt.Sprintf("/etc/kube-spawn") + viper.AddConfigPath(config) + } + viper.SetEnvPrefix("KUBE_SPAWN") + viper.AutomaticEnv() + if err := viper.ReadInConfig(); err == nil { + log.Printf("Using config file %q", viper.ConfigFileUsed()) } - log.Printf("using config from %s/%s", cfg.KubeSpawnDir, cfg.Name) - return cfg } -func saveConfig(cfg *config.ClusterConfiguration) { - if err := config.WriteConfigToFile(cfg); err != nil { - log.Fatal(errors.Wrap(err, "failed to write to config file")) +func main() { + if err := kubespawnCmd.Execute(); err != nil { + os.Exit(1) } } diff --git a/cmd/kube-spawn/list.go b/cmd/kube-spawn/list.go index 930a562a..ee6f24fa 100644 --- a/cmd/kube-spawn/list.go +++ b/cmd/kube-spawn/list.go @@ -17,23 +17,20 @@ limitations under the License. package main import ( + "fmt" + "io/ioutil" "log" "os" "path" - "path/filepath" - "strings" - "time" "github.com/spf13/cobra" "github.com/spf13/viper" ) -const tableFmt = "%-10s %s" - var ( listCmd = &cobra.Command{ Use: "list", - Short: "print the created environments", + Short: "List all kube-spawn clusters", Run: runList, } ) @@ -44,48 +41,22 @@ func init() { func runList(cmd *cobra.Command, args []string) { if len(args) > 0 { - log.Fatalf("too many arguments: %v", args) + log.Fatalf("Command list doesn't take arguments, got: %v", args) } - ksDir := viper.GetString("dir") - - matches, err := filepath.Glob(path.Join(ksDir, "*")) - if err != nil { - log.Fatal(err) + clusterDir := path.Join(viper.GetString("dir"), "clusters/") + entries, err := ioutil.ReadDir(clusterDir) + if err != nil && !os.IsNotExist(err) { + log.Fatalf("Failed to read cluster directory: %v", err) } - var found [][]string - for _, m := range matches { - name := filepath.Base(m) - // skip .cache - if strings.HasPrefix(name, ".") { - continue - } - fi, err := os.Stat(m) - if err != nil { - log.Fatal(err) + if len(entries) == 0 { + log.Printf("No clusters yet") + } else { + fmt.Println("Available clusters:") + for _, entry := range entries { + fmt.Printf(" %s\n", entry.Name()) } - if fi.IsDir() { - found = append(found, []string{ - fi.Name(), - fi.ModTime().Format(time.Stamp), - }) - } - } - - if len(found) < 1 { - log.Printf("no environments found") - return - } - - printTable(found) -} - -func printTable(found [][]string) { - log.Printf(tableFmt, "ENV NAME", "LAST MODIFIED") - for _, e := range found { - log.Printf(tableFmt, e[0], e[1]) } - log.Printf("\n%d environment(s) found", len(found)) } diff --git a/cmd/kube-spawn/restart.go b/cmd/kube-spawn/restart.go deleted file mode 100644 index 55baa093..00000000 --- a/cmd/kube-spawn/restart.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "log" - - "github.com/spf13/cobra" - "golang.org/x/sys/unix" -) - -var ( - restartCmd = &cobra.Command{ - Use: "restart", - Short: "Stop and start the cluster", - Long: `Stop and start the cluster. -Shortcut for running - kube-spawn stop - kube-spawn start - -You should have run 'kube-spawn create' before this.`, - Run: runRestart, - } -) - -func init() { - kubespawnCmd.AddCommand(restartCmd) - restartCmd.Flags().BoolVar(&flagSkipInit, "skip-cluster-init", false, "skips the initialization of a Kubernetes-Cluster with kubeadm") - restartCmd.Flags().BoolVarP(&flagForce, "force", "f", false, "terminate machines instead of trying graceful shutdown") -} - -func runRestart(cmd *cobra.Command, args []string) { - if unix.Geteuid() != 0 { - log.Fatalf("non-root user cannot restart clusters. abort.") - } - - if len(args) > 0 { - log.Fatalf("too many arguments: %v", args) - } - - cfg := loadConfig() - doStop(cfg, flagForce) - doStart(cfg, flagSkipInit) -} diff --git a/cmd/kube-spawn/start.go b/cmd/kube-spawn/start.go index 61bdf2d8..4c039eae 100644 --- a/cmd/kube-spawn/start.go +++ b/cmd/kube-spawn/start.go @@ -18,164 +18,53 @@ package main import ( "log" - "sync" - "time" + "path" - "github.com/pkg/errors" "github.com/spf13/cobra" - "golang.org/x/sys/unix" - - "github.com/kinvolk/kube-spawn/pkg/bootstrap" - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/kinvolk/kube-spawn/pkg/distribution" - "github.com/kinvolk/kube-spawn/pkg/machinetool" - "github.com/kinvolk/kube-spawn/pkg/nspawntool" - "github.com/kinvolk/kube-spawn/pkg/utils" - "github.com/kinvolk/kube-spawn/pkg/utils/fs" + "github.com/spf13/viper" + + "github.com/kinvolk/kube-spawn/pkg/cluster" ) var ( startCmd = &cobra.Command{ - Use: "start", - // Aliases: []string{"setup, up"}, - Short: "Start the cluster. You should have run 'kube-spawn create' before this", + Use: "start", + Short: "Start a cluster. You should have run 'kube-spawn create' before this", Run: runStart, } - flagSkipInit bool ) func init() { kubespawnCmd.AddCommand(startCmd) - startCmd.Flags().BoolVar(&flagSkipInit, "skip-cluster-init", false, "skips the initialization of a Kubernetes-Cluster with kubeadm") -} - -func runStart(cmd *cobra.Command, args []string) { - if unix.Geteuid() != 0 { - log.Fatalf("non-root user cannot start clusters. abort.") - } - if len(args) > 0 { - log.Fatalf("too many arguments: %v", args) - } + startCmd.Flags().BoolVar(&flagSkipInit, "skip-cluster-init", false, "Skips cluster initialization through kubeadm") + startCmd.Flags().IntP("nodes", "n", 3, "Number of nodes to start") + startCmd.Flags().String("cni-plugin-dir", "/opt/cni/bin", "Path to directory with CNI plugins") - cfg := loadConfig() - doStart(cfg, flagSkipInit) + viper.BindPFlags(startCmd.Flags()) } -func doStart(cfg *config.ClusterConfiguration, skipInit bool) { - if config.RunningMachines(cfg) == cfg.Nodes { - log.Print("cluster is up") - return - } - - log.Printf("using %q base image from /var/lib/machines", cfg.Image) - log.Printf("spawning cluster %q (%d machines)", cfg.Name, cfg.Nodes) - - resizeMachineDir(cfg.Image, cfg.Nodes) - - var wg sync.WaitGroup - wg.Add(cfg.Nodes) - for i := 0; i < cfg.Nodes; i++ { - go func(i int) { - defer wg.Done() - log.Printf("waiting for machine %q to start up", config.MachineName(cfg.Name, i)) - if err := nspawntool.Run(cfg, i); err != nil { - log.Print(errors.Wrap(err, "failed to start machine")) - return - } - log.Printf("machine %q started", cfg.Machines[i].Name) - - log.Printf("bootstrapping %q", cfg.Machines[i].Name) - if err := machinetool.Exec(cfg.Machines[i].Name, "/opt/kube-spawn/bootstrap.sh"); err != nil { - log.Fatal(errors.Wrap(err, "failed to bootstrap")) - } - }(i) - } - wg.Wait() - saveConfig(cfg) - - if config.RunningMachines(cfg) != cfg.Nodes { - log.Print("not all machines started") - return - } - log.Printf("cluster %q started", cfg.Name) - - if skipInit { - return - } - - if cfg.DevCluster { - // bring up the docker registry - // needed for supplying the hyperkube to kubeadm in the machines - if err := distribution.StartRegistry(); err != nil { - log.Fatal(errors.Wrap(err, "error starting registry")) - } - var err error - for i := 0; i < distribution.PushImageRetries; i++ { - err = distribution.PushImage(cfg.HyperkubeTag) - if err == nil { - break - } - time.Sleep(1 * time.Second) - } - if err != nil { - log.Fatal(errors.Wrap(err, "error pushing hyperkube image")) - } - } - - // cluster init with kubeadm from here - initMasterNode(cfg) - if cfg.Nodes > 1 { - joinWorkerNodes(cfg) - } - log.Printf("cluster %q initialized", cfg.Name) - log.Println("Note: For kubectl to work, please set $KUBECONFIG:") - log.Printf("export KUBECONFIG=%s\n", utils.GetKubeconfigPath(cfg.KubeSpawnDir, cfg.Name)) - saveConfig(cfg) -} - -func initMasterNode(cfg *config.ClusterConfiguration) error { - log.Println("[!] note: init on master can take a couple of minutes until all pods are up") - if err := nspawntool.InitializeMaster(cfg); err != nil { - log.Fatal(errors.Wrap(err, "failed to initialize master node")) - } - - return nil -} - -func joinWorkerNodes(cfg *config.ClusterConfiguration) { - var wg sync.WaitGroup - wg.Add(cfg.Nodes - 1) - for i := 1; i < cfg.Nodes; i++ { - go func(i int) { - defer wg.Done() - if err := nspawntool.JoinNode(cfg, i); err != nil { - log.Fatal(errors.Wrapf(err, "failed to join %q", cfg.Machines[i].Name)) - } - }(i) +func runStart(cmd *cobra.Command, args []string) { + if len(args) > 0 { + log.Fatalf("Command start doesn't take arguments, got: %v", args) } - wg.Wait() -} -func resizeMachineDir(baseImage string, nodesN int) { - // estimate get pool size based on sum of virtual image sizes. - var poolSize int64 - var err error + kubespawnDir := viper.GetString("dir") + clusterName := viper.GetString("cluster-name") + numberNodes := viper.GetInt("nodes") + cniPluginDir := viper.GetString("cni-plugin-dir") - if exists, err := fs.PathExists("/var/lib/machines.raw"); err != nil { - log.Fatal(errors.Wrap(err, "failed to determine if btrfs pool image exists")) - } else if !exists { - // nothing to do - return + kluster, err := cluster.New(path.Join(kubespawnDir, "clusters", clusterName), clusterName) + if err != nil { + log.Fatalf("Failed to create cluster object: %v", err) } - if poolSize, err = bootstrap.GetPoolSize(baseImage, nodesN); err != nil { - // fail hard in case of error, to avoid running unnecessary nodes - log.Fatalf("cannot get pool size: %v", err) + if err := kluster.Start(numberNodes, cniPluginDir); err != nil { + log.Fatalf("Failed to start cluster: %v", err) } - if err := bootstrap.EnlargeStoragePool(poolSize); err != nil { - log.Printf("[!] warning: cannot enlarge storage pool: %v", err) - } + log.Printf("Cluster %q initialized", clusterName) + log.Println("Export $KUBECONFIG as follows for kubectl:") + log.Printf("\n\texport KUBECONFIG=%s\n\n", kluster.AdminKubeconfigPath()) } diff --git a/cmd/kube-spawn/stop.go b/cmd/kube-spawn/stop.go index 6b78f260..3a5c36be 100644 --- a/cmd/kube-spawn/stop.go +++ b/cmd/kube-spawn/stop.go @@ -18,23 +18,20 @@ package main import ( "log" - "sync" - "time" + "path" - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/kinvolk/kube-spawn/pkg/machinetool" - "github.com/pkg/errors" "github.com/spf13/cobra" - "golang.org/x/sys/unix" + "github.com/spf13/viper" + + "github.com/kinvolk/kube-spawn/pkg/cluster" ) var ( stopCmd = &cobra.Command{ Use: "stop", - Short: "Stop the running cluster", + Short: "Stop a running cluster", Run: runStop, } - flagForce bool ) @@ -44,120 +41,24 @@ func init() { } func runStop(cmd *cobra.Command, args []string) { - if unix.Geteuid() != 0 { - log.Fatalf("non-root user cannot stop clusters. abort.") - } - if len(args) > 0 { - log.Fatalf("too many arguments: %v", args) - } - - cfg := loadConfig() - doStop(cfg, flagForce) -} - -func doStop(cfg *config.ClusterConfiguration, force bool) { - var forceTxt = "" - if force { - forceTxt = "force " - } - log.Printf("%sstopping %d machines", forceTxt, len(cfg.Machines)) - - if config.RunningMachines(cfg) != 0 { - stopMachines(cfg, force) - } else { - log.Print("nothing to stop") + log.Fatalf("Command stop doesn't take arguments, got: %v", args) } - removeImages(cfg) + kubespawnDir := viper.GetString("dir") + clusterName := viper.GetString("cluster-name") + clusterDir := path.Join(kubespawnDir, "clusters", clusterName) - // clear cluster config - cfg.Token = "" - saveConfig(cfg) -} - -func stopMachines(cfg *config.ClusterConfiguration, force bool) { - var wg sync.WaitGroup - wg.Add(len(cfg.Machines)) - for i := 0; i < len(cfg.Machines); i++ { - go func(i int) { - defer wg.Done() - if err := doGracefulStop(cfg.Machines[i].Name, force); err != nil { - return - } - cfg.Machines[i].Running = false - // machinectl output for machines with no IP - // is '-', hence use '-' here was well - cfg.Machines[i].IP = "-" - log.Printf("%q stopped", cfg.Machines[i].Name) - }(i) - } - wg.Wait() -} - -func doGracefulStop(machineName string, force bool) error { - if !force { - for retries := 0; retries < 5; retries++ { - // graceful stop - if err := machinetool.Poweroff(machineName); err != nil { - if !machinetool.IsNotKnown(err) { - log.Print(errors.Wrapf(err, "error powering off machine %q, maybe try with `kube-spawn stop -f`", machineName)) - return err - } - time.Sleep(500 * time.Millisecond) - continue - } - return nil - } - log.Printf("Tried to stop %s 5 times, but it didn't work, terminating.", machineName) - // fall back to force shutdown + kluster, err := cluster.New(clusterDir, clusterName) + if err != nil { + log.Fatalf("Failed to create cluster object: %v", err) } - // Either it's force mode from the beginning, - // or it's a fallback from a retry loop of a graceful stop. - if err := machinetool.Terminate(machineName); err != nil { - if !machinetool.IsNotKnown(err) { - log.Print(errors.Wrapf(err, "error terminating machine %q", machineName)) - return err - } - } - - return nil -} + log.Printf("Stopping cluster %s ...", clusterName) -func removeImages(cfg *config.ClusterConfiguration) { - var wg sync.WaitGroup - wg.Add(len(cfg.Machines)) - for i := 0; i < len(cfg.Machines); i++ { - go func(i int) { - defer wg.Done() - // clean up image - machName := cfg.Machines[i].Name - if machinetool.ImageExists(machName) { - for retries := 0; retries < 15; retries++ { - if err := removeImage(machName); err != nil { - log.Printf("failed to remove image %q: %v. retrying...", machName, err) - time.Sleep(500 * time.Millisecond) - continue - } - log.Printf("successfully removed image %q", machName) - break - } - } - }(i) + if err := kluster.Stop(); err != nil { + log.Fatalf("Failed to stop cluster: %v", err) } - wg.Wait() -} -func removeImage(machineName string) error { - var err error - for retries := 0; retries < 5; retries++ { - if err = machinetool.RemoveImage(machineName); err != nil { - time.Sleep(500 * time.Millisecond) - continue - } else { - return nil - } - } - return errors.Wrapf(err, "error removing machine image for %q", machineName) + log.Printf("Cluster %s stopped", clusterName) } diff --git a/cmd/kube-spawn/up.go b/cmd/kube-spawn/up.go index f31a4826..3ebde8f7 100644 --- a/cmd/kube-spawn/up.go +++ b/cmd/kube-spawn/up.go @@ -16,44 +16,31 @@ limitations under the License. package main -import ( - "log" - - "github.com/spf13/cobra" - "golang.org/x/sys/unix" -) - -var ( - upCmd = &cobra.Command{ - Use: "up", - Short: "Create a default cluster and start it", - Long: `Create a default cluster and start it. -Shortcut for running - kube-spawn create - kube-spawn start`, - Run: runUp, - } -) - -func init() { - kubespawnCmd.AddCommand(upCmd) -} - -func runUp(cmd *cobra.Command, args []string) { - if unix.Geteuid() != 0 { - log.Fatalf("non-root user cannot create & start clusters. abort.") - } - - if len(args) > 0 { - log.Fatalf("too many arguments: %v", args) - } - - doUp() -} - -func doUp() { - doCreate() - - cfg := loadConfig() - doStart(cfg, false) -} +// import ( +// "log" + +// "github.com/spf13/cobra" +// ) + +// var ( +// upCmd = &cobra.Command{ +// Use: "up", +// Short: "Create a cluster with name `default` and start it", +// Run: runUp, +// } +// ) + +// func init() { +// kubespawnCmd.AddCommand(upCmd) +// } + +// func runUp(cmd *cobra.Command, args []string) { +// if len(args) > 0 { +// log.Fatalf("Command up doesn't take arguments, got: %v", args) +// } + +// doCreate() + +// cfg := loadConfig() +// doStart(cfg, false) +// } diff --git a/doc/devel/release.md b/doc/devel/release.md index 621a04bf..32cb4ec1 100644 --- a/doc/devel/release.md +++ b/doc/devel/release.md @@ -51,7 +51,7 @@ Now we switch to the GitHub web UI to conduct the release: export KSVER="0.2.0" export NAME="kube-spawn-v$KSVER" mkdir $NAME -cp kube-spawn kube-spawn-runc cnispawn $NAME/ +cp kube-spawn $NAME/ sudo chown -R root:root $NAME/ tar czvf $NAME.tar.gz --numeric-owner $NAME/ ``` diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index 72116ec7..000e06aa 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -12,7 +12,6 @@ fix them. If you discover more, please create an issue or submit a PR. - [Running with Kubernetes 1.7.3 or newer](#running-with-kubernetes-173-or-newer) - [Getting the Kubernetes repositories](#getting-the-kubernetes-repositories) - [Setting up an insecure registry](#setting-up-an-insecure-registry) -- [Debugging kube-spawn-runc](#debugging-kube-spawn-runc) - [Inotify problems with many nodes](#inotify-problems-with-many-nodes) - [Issues with ISPs hijacking DNS requests](#issues-with-isps-hijacking-dns-requests) @@ -196,10 +195,6 @@ In that case, there are several approaches you could try out. * Check if the network interface `cni0` is available, by running `ip link | grep cni0`. * Check if the port 5000 is open, by running `ss | grep 5000`. -## Debugging `kube-spawn-runc` - -see [here](../cmd/kube-spawn-runc/README.md) - ## Inotify problems with many nodes Running a big amount of nodes (many-node clusters or many clusters) can cause inotify limits to be reached, making new nodes fail to start. diff --git a/pkg/bootstrap/cninet.go b/pkg/bootstrap/cninet.go index 36c4e745..7a696cd2 100644 --- a/pkg/bootstrap/cninet.go +++ b/pkg/bootstrap/cninet.go @@ -31,8 +31,6 @@ const LoopbackNetConf string = ` "type": "loopback" }` -const VarLibCniDir string = "/var/lib/cni/networks/kube-spawn-net" - func writeNetConf(fpath, content string) error { if _, err := os.Stat(fpath); os.IsExist(err) { return nil diff --git a/pkg/bootstrap/download.go b/pkg/bootstrap/download.go index 64d2b63f..6e95e088 100644 --- a/pkg/bootstrap/download.go +++ b/pkg/bootstrap/download.go @@ -8,7 +8,6 @@ import ( "strings" "sync" - "github.com/kinvolk/kube-spawn/pkg/config" "github.com/kinvolk/kube-spawn/pkg/utils/fs" "github.com/pkg/errors" ) @@ -30,10 +29,6 @@ var ( } ) -func getCacheDir(cfg *config.ClusterConfiguration) string { - return path.Join(cfg.KubeSpawnDir, config.CacheDir) -} - func Download(url, fpath string) error { resp, err := http.Get(url) if err != nil { @@ -47,9 +42,9 @@ func Download(url, fpath string) error { return fs.CreateFileFromReader(fpath, resp.Body) } -func DownloadK8sBins(cfg *config.ClusterConfiguration) error { +func DownloadKubernetesBinaries(k8sVersion, targetDir string) error { var err error - versionPath := path.Join(getCacheDir(cfg), cfg.KubernetesVersion) + versionPath := path.Join(targetDir, k8sVersion) if exists, err := fs.PathExists(versionPath); err != nil { return err } else if !exists { @@ -62,19 +57,16 @@ func DownloadK8sBins(cfg *config.ClusterConfiguration) error { wg.Add(len(k8sfiles)) for _, url := range k8sfiles { // replace placeholder $VERSION with actual version parameter - // TODO we need some way to validate this or a better way to get - // kubelet/kubeadm/kubectl binaries go func(url string) { defer wg.Done() - url = strings.Replace(url, "$VERSION", cfg.KubernetesVersion, 1) + url = strings.Replace(url, "$VERSION", k8sVersion, 1) inCachePath := path.Join(versionPath, path.Base(url)) if exists, err := fs.PathExists(inCachePath); err != nil { log.Printf("Error checking if path %q exists: %v\n", inCachePath, err) return } else if !exists { - log.Printf("downloading %s", path.Base(inCachePath)) - err = Download(url, inCachePath) - if err != nil { + log.Printf("Downloading %s", path.Base(inCachePath)) + if err = Download(url, inCachePath); err != nil { err = errors.Wrapf(err, "error downloading %s", url) return } @@ -85,16 +77,15 @@ func DownloadK8sBins(cfg *config.ClusterConfiguration) error { return err } -func DownloadSocatBin(cfg *config.ClusterConfiguration) error { - cachePath := getCacheDir(cfg) - if exists, err := fs.PathExists(cachePath); err != nil { +func DownloadSocatBin(targetDir string) error { + if exists, err := fs.PathExists(targetDir); err != nil { return err } else if !exists { - if err := os.MkdirAll(cachePath, 0755); err != nil { + if err := os.MkdirAll(targetDir, 0755); err != nil { return err } } - inCachePath := path.Join(cachePath, path.Base(staticSocatUrl)) + inCachePath := path.Join(targetDir, path.Base(staticSocatUrl)) if exists, err := fs.PathExists(inCachePath); err != nil { return err diff --git a/pkg/bootstrap/node.go b/pkg/bootstrap/node.go index fd6eef11..7356bce3 100644 --- a/pkg/bootstrap/node.go +++ b/pkg/bootstrap/node.go @@ -29,8 +29,7 @@ import ( "syscall" "github.com/Masterminds/semver" - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/kinvolk/kube-spawn/pkg/machinetool" + "github.com/kinvolk/kube-spawn/pkg/machinectl" "github.com/pkg/errors" ) @@ -44,100 +43,6 @@ const ( coreosStableVersion string = "1478.0.0" ) -type Node struct { - Name string - IP string -} - -func GetRunningNodes() ([]Node, error) { - var nodes []Node - - args := []string{ - "list", - "--no-legend", - } - - cmd := exec.Command("machinectl", args...) - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdin - - b, err := cmd.Output() - if err != nil { - return nil, err - } - - s := bufio.NewScanner(strings.NewReader(string(b))) - for s.Scan() { - line := strings.Fields(s.Text()) - if len(line) <= 2 { - continue - } - - // an example line from systemd v232 or newer: - // kubespawn0 container systemd-nspawn coreos 1478.0.0 10.22.0.130... - // - // systemd v231 or older: - // kubespawn0 container systemd-nspawn - - var ipaddr string - machineName := strings.TrimSpace(line[0]) - if !strings.HasPrefix(machineName, "kubespawn") { - continue - } - - if len(line) >= 6 { - ipaddr = strings.TrimSuffix(line[5], "...") - } else { - ipaddr, err = GetIPAddressLegacy(machineName) - if err != nil { - return nil, err - } - } - node := Node{ - Name: machineName, - IP: ipaddr, - } - nodes = append(nodes, node) - } - - return nodes, nil -} - -func GetIPAddressLegacy(mach string) (string, error) { - // machinectl status kubespawn0 --no-pager | grep Address - args := []string{ - "status", - mach, - "--no-pager", - } - - cmd := exec.Command("machinectl", args...) - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdin - - b, err := cmd.Output() - if err != nil { - return "", err - } - - s := bufio.NewScanner(strings.NewReader(string(b))) - for s.Scan() { - // an example line is like this: - // - // Address: 10.22.0.4 - if strings.Contains(s.Text(), "Address:") { - line := strings.TrimSpace(s.Text()) - fields := strings.Fields(line) - if len(fields) <= 1 { - continue - } - return fields[1], nil - } - } - - return "", err -} - func GetPoolSize(baseImage string, nodes int) (int64, error) { var poolSize, extraSize, biSize int64 // in bytes @@ -351,16 +256,11 @@ func runBtrfsDisableQuota() error { return nil } -func EnsureRequirements(cfg *config.ClusterConfiguration) error { +func EnsureRequirements() error { // TODO: should be moved to pkg/config/defaults.go if err := WriteNetConf(); err != nil { errors.Wrap(err, "error writing CNI configuration") } - // check if container linux base image exists - log.Printf("checking base image") - if !machinetool.ImageExists(cfg.Image) { - return fmt.Errorf("base image %q not found", cfg.Image) - } // Ensure that the system requirements are satisfied for starting // kube-spawn. It's just like running the commands below: // @@ -751,7 +651,7 @@ func pullRawCoreosImage() error { var cmdPath string var err error - // TODO: use machinetool pkg + // TODO: use machinectl pkg if cmdPath, err = exec.LookPath("machinectl"); err != nil { // fall back to an ordinary abspath to machinectl cmdPath = "/usr/bin/machinectl" @@ -789,7 +689,7 @@ func ensureCoreosVersion() { func PrepareCoreosImage() error { // If no coreos image exists, just download it - if !machinetool.ImageExists("coreos") { + if !machinectl.ImageExists("coreos") { log.Printf("pulling coreos image...") if err := pullRawCoreosImage(); err != nil { return err diff --git a/pkg/bootstrap/scripts.go b/pkg/bootstrap/scripts.go deleted file mode 100644 index 05b2d55a..00000000 --- a/pkg/bootstrap/scripts.go +++ /dev/null @@ -1,109 +0,0 @@ -package bootstrap - -import ( - "os" - "path" - - "github.com/pkg/errors" - - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/kinvolk/kube-spawn/pkg/script" - "github.com/kinvolk/kube-spawn/pkg/utils/fs" -) - -func rootfsPath(cfg *config.ClusterConfiguration) string { - return path.Join(cfg.KubeSpawnDir, cfg.Name, "rootfs") -} - -// GenerateScripts writes in //rootfs/... -// also create empty machine specific rootfs/ dirs -// -func GenerateScripts(cfg *config.ClusterConfiguration) error { - if err := os.MkdirAll(rootfsPath(cfg), 0755); err != nil { - return err - } - - if err := writeKubeadmBootstrap(cfg); err != nil { - return err - } - if err := writeKubeadmExtraArgs(cfg); err != nil { - return err - } - if err := writeKubeadmConfig(cfg); err != nil { - return err - } - if err := fs.CreateFileFromBytes(path.Join(rootfsPath(cfg), script.DockerDaemonConfigPath), []byte(script.DockerDaemonConfig)); err != nil { - return err - } - if err := fs.CreateFileFromBytes(path.Join(rootfsPath(cfg), script.DockerKubeadmExtraArgsPath), []byte(script.DockerKubeadmExtraArgs)); err != nil { - return err - } - if err := fs.CreateFileFromBytes(path.Join(rootfsPath(cfg), script.KubeletTmpfilesPath), []byte(script.KubeletTmpfiles)); err != nil { - return err - } - if cfg.ContainerRuntime == config.RuntimeRkt { - if err := fs.CreateFileFromBytes(path.Join(rootfsPath(cfg), script.RktletServicePath), []byte(script.RktletService)); err != nil { - return err - } - } - if err := fs.CreateFileFromBytes(path.Join(rootfsPath(cfg), script.WeaveNetworkdUnmaskPath), []byte(script.WeaveNetworkdUnmask)); err != nil { - return err - } - - // create empty config dirs for all nodes - for i := 0; i < cfg.Nodes; i++ { - rootDir := path.Join(cfg.KubeSpawnDir, cfg.Name, config.MachineName(cfg.Name, i), "rootfs") - if err := os.MkdirAll(path.Join(rootDir, "etc"), 0755); err != nil { - return err - } - if err := os.MkdirAll(path.Join(rootDir, "opt"), 0755); err != nil { - return err - } - if err := os.MkdirAll(path.Join(rootDir, "usr/bin"), 0755); err != nil { - return err - } - } - return nil -} - -func writeKubeadmBootstrap(cfg *config.ClusterConfiguration) error { - bootstrapScript := path.Join(rootfsPath(cfg), script.KubeadmBootstrapPath) - - buf, err := script.GetKubeadmBootstrap(script.KubeadmBootstrapOpts{ - ContainerRuntime: cfg.ContainerRuntime, - }) - if err != nil { - return errors.Wrapf(err, "error generating %q", bootstrapScript) - } - return fs.CreateFileFromReader(bootstrapScript, buf) -} - -func writeKubeadmExtraArgs(cfg *config.ClusterConfiguration) error { - extraArgsConf := path.Join(rootfsPath(cfg), script.KubeadmExtraArgsPath) - buf, err := script.GetKubeadmExtraArgs(script.KubeadmExtraArgsOpts{ - ContainerRuntime: cfg.ContainerRuntime, - UseLegacyCgroupDriver: cfg.RuntimeConfiguration.UseLegacyCgroupDriver, - CgroupsPerQOS: cfg.RuntimeConfiguration.CgroupPerQos, - FailSwapOn: cfg.RuntimeConfiguration.FailSwapOn, - RuntimeEndpoint: cfg.RuntimeConfiguration.Endpoint, - RequestTimeout: cfg.RuntimeConfiguration.Timeout, - }) - if err != nil { - return errors.Wrapf(err, "error generating %q", extraArgsConf) - } - return fs.CreateFileFromReader(extraArgsConf, buf) -} - -func writeKubeadmConfig(cfg *config.ClusterConfiguration) error { - kubeadmConf := path.Join(rootfsPath(cfg), script.KubeadmConfigPath) - - buf, err := script.GetKubeadmConfig(script.KubeadmYmlOpts{ - DevCluster: cfg.DevCluster, - KubernetesVersion: cfg.KubernetesVersion, - HyperkubeTag: cfg.HyperkubeTag, - }) - if err != nil { - return errors.Wrapf(err, "error generating %q", kubeadmConf) - } - return fs.CreateFileFromReader(kubeadmConf, buf) -} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go new file mode 100644 index 00000000..6c02e8e9 --- /dev/null +++ b/pkg/cache/cache.go @@ -0,0 +1,15 @@ +package cache + +type Cache struct { + dir string +} + +func New(dir string) (*Cache, error) { + return &Cache{ + dir: dir, + }, nil +} + +func (c *Cache) Dir() string { + return c.dir +} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go new file mode 100644 index 00000000..0d98629e --- /dev/null +++ b/pkg/cluster/cluster.go @@ -0,0 +1,660 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "os" + "os/exec" + "path" + "regexp" + "strings" + "sync" + "time" + + "github.com/Masterminds/semver" + "github.com/pkg/errors" + + "github.com/kinvolk/kube-spawn/pkg/bootstrap" + "github.com/kinvolk/kube-spawn/pkg/cache" + "github.com/kinvolk/kube-spawn/pkg/machinectl" + "github.com/kinvolk/kube-spawn/pkg/multiprint" + "github.com/kinvolk/kube-spawn/pkg/nspawntool" + "github.com/kinvolk/kube-spawn/pkg/utils/fs" +) + +type ClusterSettings struct { + CNIPluginDir string + ContainerRuntime string + HyperkubeImage string + KubernetesSourceDir string + KubernetesVersion string + RuntimeEndpoint string + RktBinaryPath string + RktStage1ImagePath string + RktletBinaryPath string + UseLegacyCgroupDriver bool +} + +type Cluster struct { + dir string + name string +} + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) +} + +const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789" + +func randString(n int) string { + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return string(b) +} + +const ( + validNameRegexpStr = "^[a-zA-Z0-9-]{1,50}$" + weaveNet = "https://github.com/weaveworks/weave/releases/download/v2.0.5/weave-daemonset-k8s-1.7.yaml" + + // Avoid token passing between master and worker nodes by using + // a hard-coded token + kubeadmToken = "aaaaaa.bbbbbbbbbbbbbbbb" +) + +var validNameRegexp = regexp.MustCompile(validNameRegexpStr) + +func ValidName(name string) bool { + return validNameRegexp.MatchString(name) +} + +func New(dir, name string) (*Cluster, error) { + if !ValidName(name) { + return nil, errors.Errorf("got invalid cluster name %q (expected %q)", name, validNameRegexpStr) + } + return &Cluster{ + dir: dir, + name: name, + }, nil +} + +func validateClusterSettings(clusterSettings *ClusterSettings) error { + if clusterSettings.KubernetesVersion == "" && (clusterSettings.HyperkubeImage == "" || clusterSettings.KubernetesSourceDir == "") { + return errors.Errorf("either kubernetes version or hyperkube image and kubernetes source dir must be given") + } + if clusterSettings.ContainerRuntime != "docker" && clusterSettings.ContainerRuntime != "rkt" { + return errors.Errorf("unsupported container runtime given: %s", clusterSettings.ContainerRuntime) + } + return nil +} + +// Create creates a new kube-spawn cluster environment. It does.. +// +// * Validate the cluster settings +// * Download the required files into the cache +// * Generate the shared, readonly rootfs directory structure (later +// mounted via overlayfs) +// * Copy the required files from the source directories and cache to +// the target locations +func (c *Cluster) Create(clusterSettings *ClusterSettings, clusterCache *cache.Cache) error { + if err := validateClusterSettings(clusterSettings); err != nil { + return err + } + if clusterCache == nil { + return errors.Errorf("no cache given but required") + } + + cacheDirKubernetes := path.Join(clusterCache.Dir(), "kubernetes") + + if clusterSettings.KubernetesSourceDir == "" { + if err := bootstrap.DownloadKubernetesBinaries(clusterSettings.KubernetesVersion, cacheDirKubernetes); err != nil { + return errors.Wrap(err, "failed to download required Kubernetes binaries") + } + } + + if err := bootstrap.DownloadSocatBin(clusterCache.Dir()); err != nil { + return errors.Wrap(err, "failed to download `socat` into cache dir") + } + + if err := os.MkdirAll(c.BaseRootfsPath(), 0755); err != nil { + return errors.Wrapf(err, "failed to create directory %q", c.BaseRootfsPath()) + } + + log.Print("Copying files for cluster ...") + + var ( + kubeletPath string + kubeadmPath string + kubectlPath string + kubeletServicePath string + kubeadmDropinPath string + ) + if clusterSettings.KubernetesSourceDir != "" { + // If Docker was used to build Kubernetes (`build/run.sh make`), + // the binaries would be in `"_output/dockerized/bin/linux/amd64`, + // look their first. If we don't find them there, try with + // `_output/bin` + kubernetesSourceBinaryDir := path.Join(clusterSettings.KubernetesSourceDir, "_output/dockerized/bin/linux/amd64") + + kubeadmPath = path.Join(kubernetesSourceBinaryDir, "kubeadm") + if exists, err := fs.PathExists(kubeadmPath); err != nil { + return errors.Wrapf(err, "Failed to stat %q: %v", kubeadmPath) + } else if !exists { + kubernetesSourceBinaryDir = path.Join(clusterSettings.KubernetesSourceDir, "_output/bin") + + kubeadmPath = path.Join(kubernetesSourceBinaryDir, "kubeadm") + if exists, err := fs.PathExists(kubeadmPath); err != nil { + return errors.Wrapf(err, "Failed to stat %q: %v", kubernetesSourceBinaryDir) + } else if !exists { + return errors.Errorf("Cannot find expected `_output` directory in %q", clusterSettings.KubernetesSourceDir) + } + + } + + kubeletPath = path.Join(kubernetesSourceBinaryDir, "kubelet") + kubectlPath = path.Join(kubernetesSourceBinaryDir, "kubectl") + + kubernetesSourceBuildDir := path.Join(clusterSettings.KubernetesSourceDir, "build") + + kubeletServicePath = path.Join(kubernetesSourceBuildDir, "debs/kubelet.service") + kubeadmDropinPath = path.Join(kubernetesSourceBuildDir, "rpms/10-kubeadm.conf") + } else { + kubeletPath = path.Join(cacheDirKubernetes, clusterSettings.KubernetesVersion, "kubelet") + kubeadmPath = path.Join(cacheDirKubernetes, clusterSettings.KubernetesVersion, "kubeadm") + kubectlPath = path.Join(cacheDirKubernetes, clusterSettings.KubernetesVersion, "kubectl") + + kubeletServicePath = path.Join(cacheDirKubernetes, clusterSettings.KubernetesVersion, "kubelet.service") + kubeadmDropinPath = path.Join(cacheDirKubernetes, clusterSettings.KubernetesVersion, "10-kubeadm.conf") + } + + type copyItem struct { + dst string + src string + } + var copyItems []copyItem + + // copyItem destinations must be relative to the cluster rootfs path + // We will prepend it later + + copyItems = append(copyItems, copyItem{dst: "/usr/bin/kubelet", src: kubeletPath}) + copyItems = append(copyItems, copyItem{dst: "/usr/bin/kubeadm", src: kubeadmPath}) + copyItems = append(copyItems, copyItem{dst: "/usr/bin/kubectl", src: kubectlPath}) + copyItems = append(copyItems, copyItem{dst: "/etc/systemd/system/kubelet.service", src: kubeletServicePath}) + copyItems = append(copyItems, copyItem{dst: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", src: kubeadmDropinPath}) + + socatPath := path.Join(clusterCache.Dir(), "socat") + copyItems = append(copyItems, copyItem{dst: "/usr/bin/socat", src: socatPath}) + + copyItems = append(copyItems, copyItem{dst: "/opt/cni/bin/bridge", src: path.Join(clusterSettings.CNIPluginDir, "bridge")}) + copyItems = append(copyItems, copyItem{dst: "/opt/cni/bin/loopback", src: path.Join(clusterSettings.CNIPluginDir, "loopback")}) + + if clusterSettings.ContainerRuntime == "rkt" { + copyItems = append(copyItems, copyItem{dst: "/usr/bin/rkt", src: clusterSettings.RktBinaryPath}) + copyItems = append(copyItems, copyItem{dst: "/usr/bin/stage1-coreos.aci", src: clusterSettings.RktStage1ImagePath}) + copyItems = append(copyItems, copyItem{dst: "/usr/bin/rktlet", src: clusterSettings.RktletBinaryPath}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var failed bool + errorChan := make(chan error) + go func() { + for { + select { + case <-ctx.Done(): + return + case err, ok := <-errorChan: + if !ok { + return + } + failed = true + log.Printf("%v", err) + } + } + }() + + var wg sync.WaitGroup + wg.Add(len(copyItems)) + for _, item := range copyItems { + go func(dst, src string) { + defer wg.Done() + dst = path.Join(c.BaseRootfsPath(), dst) + if copyErr := fs.CopyFile(src, dst); copyErr != nil { + errorChan <- errors.Wrapf(copyErr, "Failed to copy file %q -> %q", src, dst) + } + }(item.dst, item.src) + } + wg.Wait() + + if failed { + return errors.Errorf("copying necessary files didn't succeed") + } + return prepareBaseRootfs(c.BaseRootfsPath(), clusterSettings) +} + +func prepareBaseRootfs(rootfsDir string, clusterSettings *ClusterSettings) error { + log.Print("Generating configuration files from templates ...") + + clusterSettings.UseLegacyCgroupDriver = clusterSettings.ContainerRuntime == "docker" + if clusterSettings.ContainerRuntime == "rkt" { + clusterSettings.RuntimeEndpoint = "unix:///var/run/rktlet.sock" + } + + if err := fs.CreateFileFromString(path.Join(rootfsDir, "/usr/bin/kube-spawn-runc"), KubeSpawnRuncWrapperScript); err != nil { + return err + } + if err := fs.CreateFileFromString(path.Join(rootfsDir, "/etc/docker/daemon.json"), DockerDaemonConfig); err != nil { + return err + } + if err := fs.CreateFileFromString(path.Join(rootfsDir, "/etc/systemd/system/docker.service.d/20-kube-spawn.conf"), DockerSystemdDropin); err != nil { + return err + } + if err := fs.CreateFileFromString(path.Join(rootfsDir, "/etc/resolv.conf"), "nameserver 8.8.8.8"); err != nil { + return err + } + if clusterSettings.ContainerRuntime == "rkt" { + if err := fs.CreateFileFromString(path.Join(rootfsDir, "/etc/systemd/system/rktlet.service"), RktletSystemdUnit); err != nil { + return err + } + } + if err := fs.CreateFileFromString(path.Join(rootfsDir, "/etc/systemd/network/50-weave.network"), WeaveSystemdNetworkdConfig); err != nil { + return err + } + + buf, err := ExecuteTemplate(KubespawnBootstrapScriptTmpl, clusterSettings) + if err != nil { + return err + } + if err := fs.CreateFileFromReader(path.Join(rootfsDir, "/opt/kube-spawn/bootstrap.sh"), &buf); err != nil { + return err + } + + buf, err = ExecuteTemplate(KubeletSystemdDropinTmpl, clusterSettings) + if err != nil { + return err + } + if err := fs.CreateFileFromReader(path.Join(rootfsDir, "/etc/systemd/system/kubelet.service.d/20-kube-spawn.conf"), &buf); err != nil { + return err + } + + buf, err = ExecuteTemplate(KubeadmConfigTmpl, clusterSettings) + if err != nil { + return err + } + if err := fs.CreateFileFromReader(path.Join(rootfsDir, "/etc/kubeadm/kubeadm.yml"), &buf); err != nil { + return err + } + + return nil +} + +func (c *Cluster) Start(numberNodes int, cniPluginDir string) error { + if numberNodes < 1 { + return errors.Errorf("cannot start less than 1 node") + } + if err := bootstrap.PrepareCoreosImage(); err != nil { + return err + } + + if err := bootstrap.EnsureRequirements(); err != nil { + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var failed bool + errorChan := make(chan error) + go func() { + for { + select { + case <-ctx.Done(): + return + case err, ok := <-errorChan: + if !ok { + return + } + failed = true + log.Printf("%v", err) + } + } + }() + + log.Printf("Starting %d nodes in cluster %s ...", numberNodes, c.name) + + // Note: currently only a single master node is supported and + // the code written with that limitation. Supporting multiple + // master nodes shouldn't be too much work though (figure out + // multi master setup with kubeadm + use loadbalancer + use + // loadbalancer IP from worker nodes) + + var wg sync.WaitGroup + wg.Add(numberNodes) + for i := 0; i < numberNodes; i++ { + go func(nodeNumber int) { + defer wg.Done() + + var machineNameSuffix string + if nodeNumber == 0 { + machineNameSuffix = fmt.Sprintf("master-%s", randString(6)) + } else { + machineNameSuffix = fmt.Sprintf("worker-%s", randString(6)) + } + machineName := fmt.Sprintf("kube-spawn-%s-%s", c.name, machineNameSuffix) + + log.Printf("Waiting for machine %s to start up ...", machineName) + + if err := nspawntool.Run("coreos", c.BaseRootfsPath(), path.Join(c.MachineRootfsPath(), machineName), machineName, cniPluginDir); err != nil { + errorChan <- errors.Wrapf(err, "Failed to start machine %s", machineName) + return + } + + log.Printf("Started %s", machineName) + log.Printf("Bootstrapping %s ...", machineName) + + if err := machinectl.Exec(machineName, "/opt/kube-spawn/bootstrap.sh"); err != nil { + errorChan <- errors.Wrapf(err, "Failed to bootstrap machine %s", machineName) + } + }(i) + } + wg.Wait() + + if failed { + return errors.Errorf("starting the cluster didn't succeed") + } + + log.Printf("Cluster %q started", c.name) + + masterMachines, err := c.MasterMachines() + if err != nil { + return errors.Errorf("failed to get list of master machines: %v", err) + } + if len(masterMachines) == 0 { + return errors.Errorf("no master machines found") + } + + masterMachine := masterMachines[0] + + log.Println("Note: `kubeadm init` can take several minutes") + + multiPrinter := multiprint.New(ctx) + multiPrinter.RunPrintLoop() + + // Determine the kubeadm version by parsing `kubeadm version` + // We need to know in order to adjust used configuration flags + kubeadmVersion, err := kubeadmGitVersion(path.Join(c.BaseRootfsPath(), "usr/bin/kubeadm")) + if err != nil { + return errors.Wrap(err, "failed to determine kubeadm version") + } + + shortName := strings.TrimPrefix(masterMachine.Name, fmt.Sprintf("kube-spawn-%s-", c.name)) + cliWriter := multiPrinter.NewWriter(fmt.Sprintf("%s ", shortName)) + if err := kubeadmInit(kubeadmVersion, masterMachine.Name, cliWriter); err != nil { + return errors.Wrapf(err, "failed to kubeadm init %q", masterMachine.Name) + } + if err := applyNetworkPlugin(masterMachine.Name, cliWriter); err != nil { + return err + } + + adminKubeconfigSource := path.Join(c.MachineRootfsPath(), masterMachine.Name, "etc/kubernetes/admin.conf") + if err := fs.CopyFile(adminKubeconfigSource, c.AdminKubeconfigPath()); err != nil { + return err + } + + workerMachines, err := c.WorkerMachines() + if err != nil { + return err + } + + masterIP := masterMachine.IP + + wg.Add(len(workerMachines)) + for _, worker := range workerMachines { + go func(nodeName string) { + defer wg.Done() + shortName := strings.TrimPrefix(nodeName, fmt.Sprintf("kube-spawn-%s-", c.name)) + if err := kubeadmJoin(kubeadmVersion, masterIP, nodeName, multiPrinter.NewWriter(fmt.Sprintf("%s ", shortName))); err != nil { + errorChan <- errors.Wrapf(err, "Failed to kubeadm join %q", worker.Name) + } + }(worker.Name) + } + wg.Wait() + + if failed { + return errors.Errorf("provisioning the worker nodes with kubeadm didn't succeed") + } + return nil +} + +func (c *Cluster) AdminKubeconfigPath() string { + return path.Join(c.dir, "admin.kubeconfig") +} + +func (c *Cluster) AdminKubeconfig() (string, error) { + kubeconfigBytes, err := ioutil.ReadFile(c.AdminKubeconfigPath()) + if err != nil { + return "", err + } + return string(kubeconfigBytes), nil +} + +func (c *Cluster) BaseRootfsPath() string { + return path.Join(c.dir, "rootfs-base-readonly") +} + +func (c *Cluster) MachineRootfsPath() string { + return path.Join(c.dir, "rootfs-machines") +} + +func (c *Cluster) MasterMachines() ([]machinectl.Machine, error) { + return machinectl.ListByRegexp(fmt.Sprintf("^kube-spawn-%s-master-[a-z0-9]+$", c.name)) +} + +func (c *Cluster) WorkerMachines() ([]machinectl.Machine, error) { + return machinectl.ListByRegexp(fmt.Sprintf("^kube-spawn-%s-worker-[a-z0-9]+$", c.name)) +} + +func (c *Cluster) Machines() ([]machinectl.Machine, error) { + return machinectl.ListByRegexp(fmt.Sprintf("^kube-spawn-%s.*$", c.name)) +} + +func (c *Cluster) ListImages() ([]machinectl.Image, error) { + return machinectl.ListImagesByRegexp(fmt.Sprintf("^kube-spawn-%s.*$", c.name)) +} + +func (c *Cluster) Stop() error { + if err := c.StopMachines(30 * time.Second); err != nil { + return err + } + if err := c.RemoveImages(30 * time.Second); err != nil { + return err + } + // TODO(schu): remove network bits + return nil +} + +func (c *Cluster) Destroy() error { + if err := c.Stop(); err != nil { + return err + } + if err := os.RemoveAll(c.dir); err != nil { + return errors.Errorf("failed to remove cluster dir %q: %v", c.dir, err) + } + return nil +} + +func (c *Cluster) RemoveImages(timeout time.Duration) error { + images, err := c.ListImages() + if err != nil { + return err + } + if len(images) == 0 { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + tickChan := time.Tick(1 * time.Second) + + var wg sync.WaitGroup + wg.Add(len(images)) + for i, image := range images { + go func(imageName string, idx int) { + defer wg.Done() + for _ = range tickChan { + if err := machinectl.Remove(imageName); err == nil { + return + } + select { + case <-ctx.Done(): + // timeout + return + default: + } + } + }(image.Name, i) + } + wg.Wait() + + images, err = c.ListImages() + if err != nil { + return err + } + if len(images) > 0 { + return errors.Errorf("failed to remove all images (use `machinectl remove ...` to remove them manually)") + } + return nil +} + +func (c *Cluster) StopMachines(timeout time.Duration) error { + machines, err := c.Machines() + if err != nil { + return err + } + if len(machines) == 0 { + return nil + } + + for _, machine := range machines { + if err := machinectl.Poweroff(machine.Name); err != nil { + return errors.Wrapf(err, "failed to poweroff machine %q", machine.Name) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + tickChan := time.Tick(2 * time.Second) + +waitPoweroff: + for _ = range tickChan { + select { + case <-ctx.Done(): + // timeout + break waitPoweroff + default: + } + for _, machine := range machines { + if machinectl.IsRunning(machine.Name) { + continue waitPoweroff + } + } + // all machines stopped already + return nil + } + + // poweroff didn't succeed in time, terminate the machines + for _, machine := range machines { + if machinectl.IsRunning(machine.Name) { + if err := machinectl.Terminate(machine.Name); err != nil { + return errors.Wrapf(err, "failed to terminate machine %q", machine.Name) + } + } + } + return nil +} + +func kubeadmInit(kubeadmVersionStr, machineName string, outWriter io.Writer) error { + initCmd := []string{ + "/usr/bin/kubeadm", + "init", + "--config=/etc/kubeadm/kubeadm.yml", + } + kubeadmVersion, err := semver.NewVersion(kubeadmVersionStr) + if err != nil { + return err + } + isLargerEqual19, err := semver.NewConstraint(">= 1.9") + if err != nil { + return err + } + if !isLargerEqual19.Check(kubeadmVersion) { + initCmd = append(initCmd, "--skip-preflight-checks") + } else { + initCmd = append(initCmd, "--ignore-preflight-errors=all") + } + if _, err := machinectl.RunCommand(outWriter, nil, "", "shell", machineName, initCmd...); err != nil { + return errors.Wrap(err, "kubeadm init failed") + } + if _, err := machinectl.RunCommand(outWriter, nil, "", "shell", machineName, "/usr/bin/kubeadm", "token", "create", kubeadmToken, "--ttl=0"); err != nil { + return errors.Wrap(err, "failed registering token") + } + return nil +} + +func kubeadmJoin(kubeadmVersionStr, masterIP, machineName string, outWriter io.Writer) error { + joinCmd := []string{ + "/usr/bin/kubeadm", + "join", + "--token", kubeadmToken, + } + kubeadmVersion, err := semver.NewVersion(kubeadmVersionStr) + if err != nil { + return err + } + isLargerEqual19, err := semver.NewConstraint(">= 1.9") + if err != nil { + return err + } + if !isLargerEqual19.Check(kubeadmVersion) { + joinCmd = append(joinCmd, "--skip-preflight-checks") + } else { + joinCmd = append(joinCmd, + "--ignore-preflight-errors=all", + "--discovery-token-unsafe-skip-ca-verification") + } + joinCmd = append(joinCmd, fmt.Sprintf("%s:6443", masterIP)) + _, err = machinectl.RunCommand(outWriter, nil, "", "shell", machineName, joinCmd...) + return err +} + +func applyNetworkPlugin(machineName string, outWriter io.Writer) error { + _, err := machinectl.RunCommand(outWriter, nil, "", "shell", machineName, "/usr/bin/kubectl", "apply", "-f", weaveNet) + return err +} + +type kubeadmVersionType struct { + ClientVersion struct { + GitVersion string `json:"gitVersion"` + } `json:"clientVersion"` +} + +func kubeadmGitVersion(kubeadmPath string) (string, error) { + out, err := exec.Command(kubeadmPath, "version", "-o", "json").Output() + if err != nil { + return "", err + } + var kubeadmVersion kubeadmVersionType + if err := json.Unmarshal(out, &kubeadmVersion); err != nil { + return "", err + } + return kubeadmVersion.ClientVersion.GitVersion, nil +} diff --git a/pkg/cluster/clusterfiles.go b/pkg/cluster/clusterfiles.go new file mode 100644 index 00000000..246fbf11 --- /dev/null +++ b/pkg/cluster/clusterfiles.go @@ -0,0 +1,131 @@ +package cluster + +import ( + "bytes" + "text/template" +) + +func ExecuteTemplate(tmplStr string, tmplData interface{}) (bytes.Buffer, error) { + var out bytes.Buffer + tmpl, err := template.New("").Parse(tmplStr) + if err != nil { + return out, err + } + if err := tmpl.Execute(&out, tmplData); err != nil { + return out, err + } + return out, nil +} + +const DockerDaemonConfig = `{ + "insecure-registries": ["10.22.0.1:5000"], + "default-runtime": "custom", + "runtimes": { + "custom": { "path": "/usr/bin/kube-spawn-runc" } + }, + "storage-driver": "overlay2" +} +` + +const DockerSystemdDropin = `[Service] +Environment="DOCKER_OPTS=--exec-opt native.cgroupdriver=cgroupfs" +` + +const RktletSystemdUnit = `[Unit] +Description=rktlet: The rkt implementation of a Kubernetes Container Runtime +Documentation=https://github.com/kubernetes-incubator/rktlet/tree/master/docs + +[Service] +ExecStart=/usr/bin/rktlet --net=weave +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target +` + +// https://github.com/kinvolk/kube-spawn/issues/99 +// https://github.com/weaveworks/weave/issues/2601 +const WeaveSystemdNetworkdConfig = `[Match] +Name=weave datapath vethwe* + +[Link] +Unmanaged=yes +` + +const KubespawnBootstrapScriptTmpl = `#!/bin/bash + +set -euxo pipefail + +echo "root:root" | chpasswd +echo "core:core" | chpasswd + +systemctl enable kubelet.service +systemctl enable sshd.service + +{{ if eq .ContainerRuntime "docker" -}}systemctl start --no-block docker.service{{- end}} +{{ if eq .ContainerRuntime "rkt" -}}systemctl start --no-block rktlet.service +mkdir -p /usr/lib/rkt/plugins +ln -s /opt/cni/bin/ /usr/lib/rkt/plugins/net +ln -sfT /etc/cni/net.d /etc/rkt/net.d{{- end}} + +mkdir -p /var/lib/weave + +# necessary to prevent docker from being blocked +systemctl mask systemd-networkd-wait-online.service + +kubeadm reset +systemctl start --no-block kubelet.service +` + +// --fail-swap-on=false is necessary for k8s 1.8 or newer. + +// For rktlet, --container-runtime must be "remote", not "rkt". +// --container-runtime-endpoint needs to point to the unix socket, +// which rktlet listens on. + +// --cgroups-per-qos should be set to false, so that we can avoid issues with +// different formats of cgroup paths between k8s and systemd. +// --enforce-node-allocatable= is also necessary. +const KubeletSystemdDropinTmpl = `[Service] +Environment="KUBELET_CGROUP_ARGS=--cgroup-driver={{ if .UseLegacyCgroupDriver }}cgroupfs{{else}}systemd{{end}}" +Environment="KUBELET_EXTRA_ARGS=\ +{{ if ne .ContainerRuntime "docker" -}}--container-runtime=remote \ +--container-runtime-endpoint={{.RuntimeEndpoint}} \ +--runtime-request-timeout=15m {{- end}} \ +--enforce-node-allocatable= \ +--cgroups-per-qos=false \ +--fail-swap-on=false \ +--authentication-token-webhook" +` + +const KubeadmConfigTmpl = `apiVersion: kubeadm.k8s.io/v1alpha1 +authorizationMode: AlwaysAllow +apiServerExtraArgs: + insecure-port: "8080" +controllerManagerExtraArgs: +kubernetesVersion: {{.KubernetesVersion}} +schedulerExtraArgs: +{{if .HyperkubeImage -}} +unifiedControlPlaneImage: {{.HyperkubeImage}} +{{- end }} +` + +const KubeSpawnRuncWrapperScript = `#!/bin/bash +# TODO: the docker-runc wrapper ensures --no-new-keyring is +# set, otherwise Docker will attempt to use keyring syscalls +# which are not allowed in systemd-nspawn containers. It can +# be removed once we require systemd v235 or later. We then +# will be able to whitelist the required syscalls; see: +# https:#github.com/systemd/systemd/pull/6798 +set -euo pipefail +args=() +for arg in "${@}"; do + args+=("${arg}") + if [[ "${arg}" == "create" ]] || [[ "${arg}" == "run" ]]; then + args+=("--no-new-keyring") + fi +done +exec docker-runc "${args[@]}" +` diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go deleted file mode 100644 index 08402bd2..00000000 --- a/pkg/config/defaults.go +++ /dev/null @@ -1,253 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - "os" - "os/exec" - "path" - - "github.com/pkg/errors" - "github.com/spf13/viper" - - "github.com/kinvolk/kube-spawn/pkg/utils" - "github.com/kinvolk/kube-spawn/pkg/utils/fs" -) - -const ( - DefaultKubeSpawnDir = "/var/lib/kube-spawn" - DefaultClusterName = "default" - DefaultContainerRuntime = RuntimeDocker - DefaultKubernetesVersion = "v1.8.5" - DefaultBaseImage = "coreos" - - DefaultDockerRuntimeEndpoint = "" // "unix:///var/run/docker.sock" - DefaultRktRuntimeEndpoint = "unix:///var/run/rktlet.sock" - DefaultCrioRuntimeEndpoint = "unix:///var/run/crio.sock" - DefaultRuntimeTimeout = "15m" - DefaultRktStage1ImagePath = "/usr/lib/rkt/stage1-images/stage1-coreos.aci" - - CacheDir = ".cache" -) - -func SetDefaults_Viper(v *viper.Viper) { - v.SetDefault("dir", DefaultKubeSpawnDir) - v.SetDefault("cluster-name", DefaultClusterName) - v.SetDefault("container-runtime", DefaultContainerRuntime) - v.SetDefault("kubernetes-version", DefaultKubernetesVersion) - v.SetDefault("dev", false) - v.SetDefault("nodes", 2) - v.SetDefault("image", DefaultBaseImage) -} - -func SetDefaults_Kubernetes(cfg *ClusterConfiguration) error { - var ( - kubeletPath, kubeadmPath, kubectlPath string - kubeletServicePath, kubeadmDropinPath string - ) - - cacheDir := path.Join(cfg.KubeSpawnDir, CacheDir) - - if cfg.DevCluster { - cfg.KubernetesVersion = "latest" - // self-compiled k8s development tree - k8sOutputDir, err := utils.GetK8sBuildOutputDir() - if err != nil { - return errors.Wrap(err, "error getting k8s build output directory") - } - kubeletPath = path.Join(k8sOutputDir, "kubelet") - kubeadmPath = path.Join(k8sOutputDir, "kubeadm") - kubectlPath = path.Join(k8sOutputDir, "kubectl") - - k8sBuildAssetDir, err := utils.GetK8sBuildAssetDir() - if err != nil { - return errors.Wrap(err, "error getting k8s build asset directory") - } - kubeletServicePath = path.Join(k8sBuildAssetDir, "debs/kubelet.service") - kubeadmDropinPath = path.Join(k8sBuildAssetDir, "rpms/10-kubeadm.conf") - } else { - // from download cache - kubeletPath = path.Join(cacheDir, cfg.KubernetesVersion, "kubelet") - kubeadmPath = path.Join(cacheDir, cfg.KubernetesVersion, "kubeadm") - kubectlPath = path.Join(cacheDir, cfg.KubernetesVersion, "kubectl") - kubeletServicePath = path.Join(cacheDir, cfg.KubernetesVersion, "kubelet.service") - kubeadmDropinPath = path.Join(cacheDir, cfg.KubernetesVersion, "10-kubeadm.conf") - } - - cfg.Copymap = []Pathmap{ - {Dst: "/usr/bin/kubelet", Src: kubeletPath}, - {Dst: "/usr/bin/kubeadm", Src: kubeadmPath}, - {Dst: "/usr/bin/kubectl", Src: kubectlPath}, - {Dst: "/etc/systemd/system/kubelet.service", Src: kubeletServicePath}, - {Dst: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", Src: kubeadmDropinPath}, - // NOTE: workaround for making kubelet work with port-forward - {Dst: "/usr/bin/socat", Src: path.Join(cacheDir, "socat")}, - } - - if cfg.DevCluster || utils.CheckVersionConstraint(cfg.KubernetesVersion, ">=1.8.0") { - cfg.TokenGroupsOption = "--groups=system:bootstrappers:kubeadm:default-node-token" - } - - return nil -} - -func SetDefaults_RuntimeConfiguration(cfg *ClusterConfiguration) error { - if cfg.RuntimeConfiguration.Timeout == "" { - cfg.RuntimeConfiguration.Timeout = DefaultRuntimeTimeout - } - - // NOTE: K8s 1.8 or newer fails to run by default when swap is enabled. - // So we should disable the feature with an option "--fail-swap-on=false". - if !cfg.DevCluster && utils.CheckVersionConstraint(cfg.KubernetesVersion, "<1.8.0") { - cfg.RuntimeConfiguration.FailSwapOn = true - } - - var err error - switch cfg.ContainerRuntime { - case RuntimeDocker: - err = SetDefaults_DockerRuntime(cfg) - case RuntimeRkt: - err = SetDefaults_RktRuntime(cfg) - case RuntimeCrio: - err = SetDefaults_CrioRuntime(cfg) - default: - return fmt.Errorf("runtime %q not supported", cfg.ContainerRuntime) - } - if err != nil { - return err - } - - // NOTE: using docker/rkt in our nodes we run out of space quick - // TODO: can this be moved to the runtime functions below? - cfg.Machines = make([]MachineConfiguration, cfg.Nodes) - for i := 0; i < cfg.Nodes; i++ { - cfg.Machines[i].Name = MachineName(cfg.Name, i) - mountPath := path.Join(cfg.KubeSpawnDir, cfg.Name, cfg.Machines[i].Name, "mount") - - var pm Pathmap - switch cfg.ContainerRuntime { - case RuntimeDocker: - pm = Pathmap{Dst: "/var/lib/docker", Src: mountPath} - case RuntimeRkt: - pm = Pathmap{Dst: "/var/lib/rktlet", Src: mountPath} - case RuntimeCrio: - pm = Pathmap{Dst: "/var/lib/containers", Src: mountPath} - } - cfg.Machines[i].Bindmount.ReadWrite = append(cfg.Machines[i].Bindmount.ReadWrite, pm) - - // create dirs from above if they don't exist already - // TODO: should we create the dirs from here or move this to the check pkg - for _, pm := range cfg.Machines[i].Bindmount.ReadWrite { - if exists, err := fs.PathExists(pm.Src); err != nil { - return errors.Wrap(err, "cannot determine if path exists") - } else if !exists { - if err := os.MkdirAll(pm.Src, 0755); err != nil { - return err - } - } - } - } - - return nil -} - -func SetDefaults_DockerRuntime(cfg *ClusterConfiguration) error { - var err error - cfg.RuntimeConfiguration.Endpoint = DefaultDockerRuntimeEndpoint - // note: cgroup driver defaults to systemd on most systems, but there's - // an issue of runc <=1.0.0-rc2 that conflicts with --cgroup-driver=systemd, - // so we should use legacy driver "cgroupfs". - cfg.RuntimeConfiguration.UseLegacyCgroupDriver = true - return err -} - -func SetDefaults_RktRuntime(cfg *ClusterConfiguration) error { - var err error - cfg.RuntimeConfiguration.Endpoint = DefaultRktRuntimeEndpoint - - if cfg.RuntimeConfiguration.Rkt.RktBin == "" { - cfg.RuntimeConfiguration.Rkt.RktBin, err = exec.LookPath("rkt") - if err != nil { - return err - } - } - if cfg.RuntimeConfiguration.Rkt.RktletBin == "" { - cfg.RuntimeConfiguration.Rkt.RktletBin, err = exec.LookPath("rktlet") - if err != nil { - return err - } - } - if cfg.RuntimeConfiguration.Rkt.Stage1Image == "" { - cfg.RuntimeConfiguration.Rkt.Stage1Image = DefaultRktStage1ImagePath - } - - pms := []Pathmap{ - {Dst: "/usr/bin/rkt", Src: cfg.RuntimeConfiguration.Rkt.RktBin}, - {Dst: "/usr/bin/rktlet", Src: cfg.RuntimeConfiguration.Rkt.RktletBin}, - {Dst: path.Join("/usr/bin/", path.Base(cfg.RuntimeConfiguration.Rkt.Stage1Image)), Src: cfg.RuntimeConfiguration.Rkt.Stage1Image}, - {Dst: "/usr/lib/rkt/plugins/net", Src: cfg.CNIPluginDir}, - } - cfg.Bindmount.ReadOnly = append(cfg.Bindmount.ReadOnly, pms...) - return err -} - -func SetDefaults_CrioRuntime(cfg *ClusterConfiguration) error { - // note: This lays the groundwork for supporting cri-o - // in the future. - // As of now it is not expected to work. - // https://github.com/kubernetes-incubator/cri-o/blob/master/kubernetes.md - // - var err error - cfg.RuntimeConfiguration.Endpoint = DefaultCrioRuntimeEndpoint - // cgroup driver defaults to systemd on most systems, but there's - // an issue of runc <=1.0.0-rc2 that conflicts with --cgroup-driver=systemd, - // so we should use legacy driver "cgroupfs". - cfg.RuntimeConfiguration.UseLegacyCgroupDriver = true - - if cfg.RuntimeConfiguration.Crio.CrioBin == "" { - cfg.RuntimeConfiguration.Crio.CrioBin, err = exec.LookPath("crio") - if err != nil { - return err - } - } - if cfg.RuntimeConfiguration.Crio.RuncBin == "" { - cfg.RuntimeConfiguration.Crio.RuncBin, err = exec.LookPath("runc") - if err != nil { - return err - } - } - if cfg.RuntimeConfiguration.Crio.ConmonBin == "" { - cfg.RuntimeConfiguration.Crio.ConmonBin, err = exec.LookPath("conmon") - if err != nil { - return err - } - } - - pms := []Pathmap{ - {Dst: "/usr/bin/crio", Src: cfg.RuntimeConfiguration.Crio.CrioBin}, - {Dst: "/usr/bin/runc", Src: cfg.RuntimeConfiguration.Crio.RuncBin}, - {Dst: "/usr/bin/conmon", Src: cfg.RuntimeConfiguration.Crio.ConmonBin}, - } - cfg.Bindmount.ReadOnly = append(cfg.Bindmount.ReadOnly, pms...) - return err -} - -func SetDefaults_BindmountConfiguration(cfg *ClusterConfiguration) error { - cfg.Bindmount.ReadWrite = append(cfg.Bindmount.ReadWrite, Pathmap{Dst: "/opt/cni/bin", Src: cfg.CNIPluginDir}) - return nil -} diff --git a/pkg/config/types.go b/pkg/config/types.go deleted file mode 100644 index 7a758de1..00000000 --- a/pkg/config/types.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -const ( - RuntimeDocker = "docker" - RuntimeRkt = "rkt" - RuntimeCrio = "crio" -) - -// ClusterConfiguration holds the state of a cluster -// with all the information needed to -// run kube-spawn -// -type ClusterConfiguration struct { - KubeSpawnDir string `toml:"dir" mapstructure:"dir"` - - CNIPluginDir string `toml:"cni-plugin-dir" mapstructure:"cni-plugin-dir"` - - Name string `toml:"cluster-name" mapstructure:"cluster-name"` - ContainerRuntime string `toml:"container-runtime" mapstructure:"container-runtime"` - KubernetesVersion string `toml:"kubernetes-version" mapstructure:"kubernetes-version"` - Image string `toml:"image" mapstructure:"image"` - Nodes int `toml:"nodes" mapstructure:"nodes"` - - // DevCluster indicates if we should run - // from a local kubernetes build - DevCluster bool `toml:"dev" mapstructure:"dev"` - HyperkubeTag string `toml:"hyperkube-tag" mapstructure:"hyperkube-tag"` - - RuntimeConfiguration RuntimeConfiguration `toml:"runtime-config,omitempty" mapstructure:"runtime-config"` - - // Files to be copied from host to overlay of all machines - Copymap []Pathmap `toml:"-" mapstructure:"copymap"` - - // TODO: rename Bindmap - Bindmount BindmountConfiguration `toml:"bindmount" comment:"syntax: = " mapstructure:"bindmount"` - - // Token is the token generated on kubeadm init - // used to join more workers to the cluster - Token string `toml:"token, omitempty" mapstructure:"token"` - TokenGroupsOption string `toml:"token-groups-option, omitempty" mapstructure:"token-groups-option"` - - Machines []MachineConfiguration `toml:"machines, omitempty" mapstructure:"machines"` -} - -// RuntimeConfiguration holds the variables for -// running the cluster with a container runtime -// -type RuntimeConfiguration struct { - Endpoint string `toml:"endpoint,omitempty" mapstructure:"endpoint"` - Timeout string `toml:"timeout" mapstructure:"timeout"` - UseLegacyCgroupDriver bool `toml:"use-legacy-cgroup-driver" mapstructure:"use-legacy-cgroup-driver"` - CgroupPerQos bool `toml:"cgroup-per-qos" mapstructure:"cgroup-per-qos"` - FailSwapOn bool `toml:"fail-swap-on" mapstructure:"fail-swap-on"` - - Rkt RuntimeConfigurationRkt `toml:"-" mapstructure:"rkt"` - Crio RuntimeConfigurationCrio `toml:"-" mapstructure:"crio"` -} - -type RuntimeConfigurationRkt struct { - RktBin string `toml:"rkt-bin" mapstructure:"rkt-bin"` - Stage1Image string `toml:"stage1-image" mapstructure:"stage1-image"` - RktletBin string `toml:"rktlet-bin" mapstructure:"rktlet-bin"` -} - -type RuntimeConfigurationCrio struct { - CrioBin string `toml:"crio-bin" mapstructure:"crio-bin"` - RuncBin string `toml:"runc-bin" mapstructure:"runc-bin"` - ConmonBin string `toml:"conmon-bin" mapstructure:"conmon-bin"` -} - -// BindmountConfiguration contains the bind args -// for systemd-nspawn -type BindmountConfiguration struct { - ReadOnly []Pathmap `toml:"read-only" mapstructure:"read-only"` - ReadWrite []Pathmap `toml:"read-write" mapstructure:"read-write"` -} - -type MachineConfiguration struct { - Running bool `toml:"running" comment:"autogenerated. do not edit!" mapstructure:"running"` - Name string `toml:"name" comment:"autogenerated. do not edit!" mapstructure:"name"` - IP string `toml:"ip" comment:"autogenerated. do not edit!" mapstructure:"ip"` - Bindmount BindmountConfiguration `toml:"bindmount" comment:"syntax: = " mapstructure:"bindmount"` -} - -// Pathmap represents mapping a path on one machine -// to a path on another machine -type Pathmap struct { - Src string `toml:"src" mapstructure:"src"` - Dst string `toml:"dst" mapstructure:"dst"` -} diff --git a/pkg/config/utils.go b/pkg/config/utils.go deleted file mode 100644 index 1af12386..00000000 --- a/pkg/config/utils.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - "os" - "path" - - toml "github.com/pelletier/go-toml" - "github.com/pkg/errors" - "github.com/spf13/viper" - - "github.com/kinvolk/kube-spawn/pkg/utils/fs" -) - -const ( - Filename = "kspawn.toml" - - machineNameTemplate = "kubespawn%s%d" -) - -func MachineName(clusterName string, no int) string { - return fmt.Sprintf(machineNameTemplate, clusterName, no) -} - -// TODO: this is not enough. -// need to always check machined or we might lose track in case of errors -func RunningMachines(cfg *ClusterConfiguration) int { - var n int - for _, m := range cfg.Machines { - if m.Running { - n++ - } - } - return n -} - -func LoadConfig() (*ClusterConfiguration, error) { - cfgFile := path.Join(viper.GetString("dir"), viper.GetString("cluster-name"), Filename) - viper.SetConfigFile(cfgFile) - - var err error - var cfg = &ClusterConfiguration{} - err = viper.ReadInConfig() - if err := viper.Unmarshal(cfg); err != nil { - return nil, errors.Wrap(err, "unable to decode viper config") - } - return cfg, err -} - -func IsNotFound(err error) bool { - switch err.(type) { - case viper.ConfigFileNotFoundError: - return true - default: - return os.IsNotExist(err) - } -} - -func WriteConfigToFile(cfg *ClusterConfiguration) error { - cfgFilepath := path.Join(cfg.KubeSpawnDir, cfg.Name, Filename) - raw, err := toml.Marshal(*cfg) - if err != nil { - return errors.Wrap(err, "unable to encode cluster config") - } - return fs.CreateFileFromBytes(cfgFilepath, raw) -} diff --git a/pkg/distribution/registry.go b/pkg/distribution/registry.go deleted file mode 100644 index c5c02d5c..00000000 --- a/pkg/distribution/registry.go +++ /dev/null @@ -1,130 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package distribution - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "io" - "log" - "strings" - - "github.com/docker/docker/api/types" - containertypes "github.com/docker/docker/api/types/container" - networktypes "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" - "github.com/docker/go-connections/nat" -) - -const PushImageRetries = 10 - -func dockerProgress(rc io.ReadCloser) error { - var status string - - bufReader := bufio.NewReader(rc) - for { - line, _, err := bufReader.ReadLine() - if err == io.EOF { - return nil - } else if err != nil { - return err - } - - var jsonLine map[string]interface{} - if err := json.Unmarshal(line, &jsonLine); err != nil { - return err - } - - if errMsg, ok := jsonLine["error"]; ok { - if errMsgStr, ok := errMsg.(string); ok { - return fmt.Errorf(errMsgStr) - } - } - - if s, ok := jsonLine["status"]; ok { - if s.(string) != status { - status = s.(string) - log.Println(status) - } - } - } -} - -func StartRegistry() error { - ctx := context.Background() - - cli, err := client.NewEnvClient() - if err != nil { - return err - } - - log.Println("Pulling registry image") - readerCloser, err := cli.ImagePull(ctx, "docker.io/library/registry:2", types.ImagePullOptions{}) - if err != nil { - return err - } - defer readerCloser.Close() - - if err := dockerProgress(readerCloser); err != nil { - return err - } - - if _, err := cli.ContainerCreate(context.Background(), &containertypes.Config{ - Image: "registry:2", - }, &containertypes.HostConfig{ - PortBindings: nat.PortMap{"5000/tcp": []nat.PortBinding{nat.PortBinding{ - HostIP: "0.0.0.0", - HostPort: "5000", - }}}, - }, &networktypes.NetworkingConfig{}, "registry"); err != nil && !strings.Contains(err.Error(), "Conflict") { - return err - } - return cli.ContainerStart(ctx, "registry", types.ContainerStartOptions{}) -} - -func PushImage(tag string) error { - cli, err := client.NewEnvClient() - if err != nil { - return err - } - - if err := cli.ImageTag( - context.Background(), - "gcr.io/google-containers/hyperkube-amd64:"+tag, - "10.22.0.1:5000/hyperkube-amd64:"+tag, - ); err != nil { - return err - } - - log.Println("Pushing hyperkube image to local registry") - readerCloser, err := cli.ImagePush(context.Background(), "10.22.0.1:5000/hyperkube-amd64", types.ImagePushOptions{ - All: true, - // RegistryAuth header cannot be empty, even if no authentication is used at all... - RegistryAuth: "123", - }) - if err != nil { - return err - } - defer readerCloser.Close() - - if err := dockerProgress(readerCloser); err != nil { - return err - } - return nil -} diff --git a/pkg/machinectl/machinectl.go b/pkg/machinectl/machinectl.go new file mode 100644 index 00000000..7933f815 --- /dev/null +++ b/pkg/machinectl/machinectl.go @@ -0,0 +1,173 @@ +package machinectl + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os/exec" + "regexp" + "strings" +) + +type Machine struct { + Name string + IP string +} + +type Image struct { + Name string +} + +func List() ([]Machine, error) { + var machines []Machine + out, err := exec.Command("machinectl", "list", "--no-legend").Output() + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(out)) + for scanner.Scan() { + // Example `machinectl list --no-legend` output: + // kube-spawn-default-worker-fpllng container systemd-nspawn coreos 1478.0.0 10.22.0.130... + line := strings.Fields(scanner.Text()) + if len(line) < 6 { + return nil, fmt.Errorf("got unexpected output from `machinectl list --no-legend`: %s", line) + } + machine := Machine{ + Name: strings.TrimSpace(line[0]), + IP: strings.TrimSuffix(line[5], "..."), + } + machines = append(machines, machine) + } + return machines, nil +} + +func ListByRegexp(expStr string) ([]Machine, error) { + machines, err := List() + if err != nil { + return nil, err + } + exp, err := regexp.Compile(expStr) + if err != nil { + return nil, err + } + var matching []Machine + for _, machine := range machines { + if exp.MatchString(machine.Name) { + matching = append(matching, machine) + } + } + return matching, nil +} + +func ListImages() ([]Image, error) { + var images []Image + out, err := exec.Command("machinectl", "list-images", "--no-legend").Output() + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewReader(out)) + for scanner.Scan() { + // Example `machinectl list-images --no-legend` output: + // kube-spawn-default-worker-zyyios raw no 1.4G n/a Fri 2018-01-26 10:54:43 CET + line := strings.Fields(scanner.Text()) + if len(line) < 1 { + return nil, fmt.Errorf("got unexpected output from `machinectl list-images --no-legend`: %s", line) + } + image := Image{ + Name: strings.TrimSpace(line[0]), + } + images = append(images, image) + } + return images, nil +} + +func ListImagesByRegexp(expStr string) ([]Image, error) { + images, err := ListImages() + if err != nil { + return nil, err + } + exp, err := regexp.Compile(expStr) + if err != nil { + return nil, err + } + var matching []Image + for _, image := range images { + if exp.MatchString(image.Name) { + matching = append(matching, image) + } + } + return matching, nil +} + +func RunCommand(stdout, stderr io.Writer, opts, cmd, machine string, args ...string) ([]byte, error) { + mPath, err := exec.LookPath("machinectl") + if err != nil { + return nil, err + } + cmdArgs := []string{mPath} + if opts != "" { + cmdArgs = append(cmdArgs, opts) + } + cmdArgs = append(cmdArgs, cmd) + cmdArgs = append(cmdArgs, machine) + + run := exec.Cmd{ + Path: mPath, + Args: cmdArgs, + Stdout: stdout, + Stderr: stderr, + } + run.Args = append(run.Args, args...) + + var buf []byte + + if stdout != nil { + err = run.Run() + } else { + buf, err = run.Output() + } + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + return nil, fmt.Errorf("%q failed: %s", strings.Join(run.Args, " "), exitErr.Stderr) + } + return nil, fmt.Errorf("%q failed: %s", strings.Join(run.Args, " "), err) + } + return buf, nil +} + +func Exec(machine string, cmd ...string) error { + _, err := RunCommand(nil, nil, "", "shell", machine, cmd...) + return err +} + +func Clone(base, dest string) error { + _, err := RunCommand(nil, nil, "", "clone", base, dest) + return err +} + +func Poweroff(machine string) error { + _, err := RunCommand(nil, nil, "", "poweroff", machine) + return err +} + +func Terminate(machine string) error { + _, err := RunCommand(nil, nil, "", "terminate", machine) + return err +} + +func Remove(image string) error { + _, err := RunCommand(nil, nil, "", "remove", image) + return err +} + +func IsRunning(machine string) bool { + check := exec.Command("systemctl", "--machine", machine, "status", "basic.target", "--state=running") + check.Run() + return check.ProcessState.Success() +} + +func ImageExists(image string) bool { + _, err := RunCommand(nil, nil, "", "show-image", image) + return err == nil +} diff --git a/pkg/machinetool/machinetool.go b/pkg/machinetool/machinetool.go deleted file mode 100644 index 7db39c88..00000000 --- a/pkg/machinetool/machinetool.go +++ /dev/null @@ -1,96 +0,0 @@ -package machinetool - -import ( - "fmt" - "io" - "os" - "os/exec" - "regexp" - "strings" -) - -func machinectl(stdout, stderr io.Writer, opts, cmd, machine string, args ...string) ([]byte, error) { - mPath, err := exec.LookPath("machinectl") - if err != nil { - return nil, err - } - cmdArgs := []string{mPath} - if opts != "" { - cmdArgs = append(cmdArgs, opts) - } - cmdArgs = append(cmdArgs, cmd) - cmdArgs = append(cmdArgs, machine) - - run := exec.Cmd{ - Path: mPath, - Args: cmdArgs, - Stdout: stdout, - Stderr: stderr, - } - run.Args = append(run.Args, args...) - - var buf []byte - - if stdout != nil { - err = run.Run() - } else { - buf, err = run.Output() - } - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - return nil, fmt.Errorf("%q failed: %s", strings.Join(run.Args, " "), exitErr.Stderr) - } - return nil, fmt.Errorf("%q failed: %s", strings.Join(run.Args, " "), err) - } - return buf, nil -} - -func Shell(opts, machine string, cmd ...string) error { - _, err := machinectl(os.Stdout, os.Stderr, opts, "shell", machine, cmd...) - return err -} - -func Output(cmd, machine string, args ...string) ([]byte, error) { - return machinectl(nil, nil, "", cmd, machine, args...) -} - -func Exec(machine string, cmd ...string) error { - _, err := machinectl(nil, nil, "", "shell", machine, cmd...) - return err -} - -func Clone(base, dest string) error { - _, err := machinectl(nil, nil, "", "clone", base, dest) - return err -} - -func Poweroff(machine string) error { - _, err := machinectl(nil, nil, "", "poweroff", machine) - return err -} - -func Terminate(machine string) error { - _, err := machinectl(nil, nil, "", "terminate", machine) - return err -} - -func RemoveImage(image string) error { - _, err := machinectl(nil, nil, "", "remove", image) - return err -} - -func IsRunning(machine string) bool { - check := exec.Command("systemctl", "--machine", machine, "status", "basic.target", "--state=running") - check.Run() - return check.ProcessState.Success() -} - -func ImageExists(image string) bool { - _, err := machinectl(nil, nil, "", "show-image", image) - return err == nil -} - -func IsNotKnown(err error) bool { - re := regexp.MustCompile(`(.*)No (machine|image) '(.*)' known`) - return re.MatchString(err.Error()) -} diff --git a/pkg/multiprint/multiprint.go b/pkg/multiprint/multiprint.go new file mode 100644 index 00000000..76278b7c --- /dev/null +++ b/pkg/multiprint/multiprint.go @@ -0,0 +1,85 @@ +package multiprint + +import ( + "bufio" + "bytes" + "context" + "fmt" + "strings" +) + +type message struct { + prefix string + value []byte +} + +type Multiprint struct { + ctx context.Context + messageChan chan message +} + +type Writer struct { + ctx context.Context + messageChan chan message + prefix string + cancelled bool +} + +func New(ctx context.Context) *Multiprint { + return &Multiprint{ + ctx: ctx, + messageChan: make(chan message), + } +} + +func (m *Multiprint) RunPrintLoop() { + go func() { + var previousPrefix, prefix string + for { + select { + case <-m.ctx.Done(): + return + case message, ok := <-m.messageChan: + if !ok { + return + } + if previousPrefix != message.prefix { + previousPrefix = message.prefix + prefix = message.prefix + } + scanner := bufio.NewScanner(bytes.NewBuffer(message.value)) + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + fmt.Printf("%s%s\n", prefix, text) + prefix = strings.Repeat(" ", len(prefix)) + } + } + } + }() +} + +func (m *Multiprint) NewWriter(prefix string) *Writer { + writer := &Writer{ + ctx: m.ctx, + messageChan: m.messageChan, + prefix: prefix, + } + go func() { + select { + case <-writer.ctx.Done(): + writer.cancelled = true + } + }() + return writer +} + +func (w *Writer) Write(p []byte) (n int, err error) { + if w.cancelled { + return 0, fmt.Errorf("writer was cancelled") + } + w.messageChan <- message{prefix: w.prefix, value: p} + return len(p), nil +} diff --git a/pkg/nspawntool/binds.go b/pkg/nspawntool/binds.go deleted file mode 100644 index 31194cff..00000000 --- a/pkg/nspawntool/binds.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nspawntool - -import ( - "fmt" - - "github.com/kinvolk/kube-spawn/pkg/config" -) - -const bindro string = "--bind-ro=" -const bindrw string = "--bind=" - -func optionsFromBindmountConfig(bm config.BindmountConfiguration) []string { - var opts []string - robinds := generateBinds(bindro, bm.ReadOnly) - opts = append(opts, robinds...) - rwbinds := generateBinds(bindrw, bm.ReadWrite) - opts = append(opts, rwbinds...) - return opts -} - -func generateBinds(prefix string, binds []config.Pathmap) []string { - var opts []string - for _, pm := range binds { - opts = append(opts, fmt.Sprintf("%s%s:%s", prefix, pm.Src, pm.Dst)) - } - return opts -} diff --git a/pkg/nspawntool/kubeadm.go b/pkg/nspawntool/kubeadm.go deleted file mode 100644 index d7942fb0..00000000 --- a/pkg/nspawntool/kubeadm.go +++ /dev/null @@ -1,109 +0,0 @@ -package nspawntool - -import ( - "fmt" - "path" - "strings" - - "github.com/kinvolk/kube-spawn/pkg/bootstrap" - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/kinvolk/kube-spawn/pkg/machinetool" - "github.com/kinvolk/kube-spawn/pkg/utils" - "github.com/kinvolk/kube-spawn/pkg/utils/fs" - "github.com/pkg/errors" -) - -const weaveNet = "https://github.com/weaveworks/weave/releases/download/v2.0.5/weave-daemonset-k8s-1.7.yaml" - -func InitializeMaster(cfg *config.ClusterConfiguration) error { - // TODO: do we need a switch to turn off printing to stdout? - var initCmd []string - var shellOpts string - if cfg.DevCluster { - // KUBE_HYPERKUBE_IMAGE is used in old kubeadm versions. For new kubeadm >= 1.8, - // we set 'unifiedControlPlaneImage' in kubeadm.yml, see pkg/script/kubeadm-config.go - shellOpts = fmt.Sprintf(`--setenv=KUBE_HYPERKUBE_IMAGE="10.22.0.1:5000/hyperkube-amd64:%s"`, cfg.HyperkubeTag) - } - initCmd = append(initCmd, []string{ - "/usr/bin/kubeadm", "init", "--skip-preflight-checks", - "--config=/etc/kubeadm/kubeadm.yml"}...) - - if err := machinetool.Shell(shellOpts, cfg.Machines[0].Name, initCmd...); err != nil { - return err - } - if err := machinetool.Shell(shellOpts, cfg.Machines[0].Name, "/usr/bin/kubectl", "apply", "-f", weaveNet); err != nil { - return err - } - - // generate and register a token for joining - tok, err := machinetool.Output("shell", cfg.Machines[0].Name, "/usr/bin/kubeadm", "token", "generate") - if err != nil { - return errors.Wrap(err, "failed generating token") - } - cfg.Token = strings.TrimSpace(string(tok)) - if err := machinetool.Exec(cfg.Machines[0].Name, "/usr/bin/kubeadm", "token", "create", cfg.Token, "--ttl=0", cfg.TokenGroupsOption); err != nil { - cfg.Token = "" // clear unregistered token before exit - return errors.Wrap(err, "failed registering token") - } - - // find IP of master - ipStr, err := getIP(cfg.Machines[0].Name) - if err != nil { - return errors.Wrap(err, "failed to retrieve ip address") - } - cfg.Machines[0].IP = ipStr - - kubeConfigSrc := path.Join(cfg.KubeSpawnDir, cfg.Name, cfg.Machines[0].Name, "rootfs/etc/kubernetes/admin.conf") - kubeConfigDst := path.Join(cfg.KubeSpawnDir, cfg.Name, "kubeconfig") - if err := fs.CopyFile(kubeConfigSrc, kubeConfigDst); err != nil { - return errors.Wrap(err, "failed copying kubeconfig to host") - } - return nil -} - -func getIP(masterName string) (string, error) { - nodes, err := bootstrap.GetRunningNodes() - if err != nil { - return "", err - } - - for _, n := range nodes { - if n.Name == masterName { - return n.IP, nil - } - } - return "", errors.New("could not find machine") -} - -func JoinNode(cfg *config.ClusterConfiguration, mNo int) error { - if cfg.Token == "" { - return errors.New("no token found") - } - masterIP := cfg.Machines[0].IP - if masterIP == "" || masterIP == "-" { - return errors.New("no master IP found") - } - - var joinCmd []string - var shellOpts string - if cfg.DevCluster { - // TODO: remove this or implement config for it - shellOpts = `--setenv=KUBE_HYPERKUBE_IMAGE="10.22.0.1:5000/hyperkube-amd64"` - } - joinCmd = append(joinCmd, []string{ - "/usr/bin/kubeadm", "join", "--skip-preflight-checks", - "--token", cfg.Token}...) - - // --discovery-token-unsafe-skip-ca-verification appeared in Kubernetes 1.8 - // See: https://github.com/kubernetes/kubernetes/pull/49520 - // It is mandatory since Kubernetes 1.9 - // See: https://github.com/kubernetes/kubernetes/pull/55468 - // Test is !<1.8 instead of >=1.8 in order to handle non-semver version 'latest' - if !utils.CheckVersionConstraint(cfg.KubernetesVersion, "<1.8") { - joinCmd = append(joinCmd, "--discovery-token-unsafe-skip-ca-verification") - } - - joinCmd = append(joinCmd, masterIP+":6443") - - return machinetool.Shell(shellOpts, cfg.Machines[mNo].Name, joinCmd...) -} diff --git a/pkg/nspawntool/run.go b/pkg/nspawntool/run.go index 5146bb51..f07eeee5 100644 --- a/pkg/nspawntool/run.go +++ b/pkg/nspawntool/run.go @@ -20,62 +20,76 @@ import ( "encoding/json" "fmt" "io/ioutil" - "log" "os" "path" - "path/filepath" "time" cnitypes "github.com/containernetworking/cni/pkg/types" cniversion "github.com/containernetworking/cni/pkg/version" "github.com/pkg/errors" - "github.com/kinvolk/kube-spawn/pkg/config" - "github.com/kinvolk/kube-spawn/pkg/machinetool" + "github.com/kinvolk/kube-spawn/pkg/machinectl" "github.com/kinvolk/kube-spawn/pkg/utils" ) -func Run(cfg *config.ClusterConfiguration, mNo int) error { - // This check is necessary to avoid panic with "index out of range". - if mNo > len(cfg.Machines)-1 { - return fmt.Errorf("cannot get a machine kubespawn%d", mNo) +func Run(machinectlImage, lowerRootPath, upperRootPath, machineName, cniPluginDir string) error { + if machinectl.IsRunning(machineName) { + return errors.Errorf("a machine with name %q is running already", machineName) } - if cfg.Machines[mNo].Running { - return nil + if err := machinectl.Clone(machinectlImage, machineName); err != nil { + return errors.Wrap(err, "error cloning image") } - if err := machinetool.Clone(cfg.Image, cfg.Machines[mNo].Name); err != nil { - return errors.Wrap(err, "error cloning image") + if err := os.MkdirAll(lowerRootPath, 0755); err != nil { + return err } - lowerRoot, err := filepath.Abs(path.Join(cfg.KubeSpawnDir, cfg.Name, "rootfs")) - if err != nil { + if err := os.MkdirAll(upperRootPath, 0755); err != nil { return err } - upperRoot, err := filepath.Abs(path.Join(cfg.KubeSpawnDir, cfg.Name, cfg.Machines[mNo].Name, "rootfs")) - if err != nil { + + // Create all directories which will be overlay mounts (see below) + // Otherwise systemd-nspawn will fail: + // `overlayfs: failed to resolve '/var/lib/kube-spawn/...'` + if err := os.MkdirAll(path.Join(upperRootPath, "etc"), 0755); err != nil { + return err + } + if err := os.MkdirAll(path.Join(upperRootPath, "opt"), 0755); err != nil { + return err + } + if err := os.MkdirAll(path.Join(upperRootPath, "usr/bin"), 0755); err != nil { return err } + // Create all directories that will be bind mounted + bindmountDirs := []string{ + "/var/lib/docker", + "/var/lib/rktlet", + } + for _, d := range bindmountDirs { + if err := os.MkdirAll(path.Join(upperRootPath, d), 0755); err != nil { + return err + } + } + args := []string{ "cni-spawn", - "--cni-plugin-dir", cfg.CNIPluginDir, + "--cni-plugin-dir", cniPluginDir, "--", - "--machine", cfg.Machines[mNo].Name, - optionsOverlay("--overlay", "/etc", lowerRoot, upperRoot), - optionsOverlay("--overlay", "/opt", lowerRoot, upperRoot), - optionsOverlay("--overlay", "/usr/bin", lowerRoot, upperRoot), + "--machine", machineName, + optionsOverlay("--overlay", "/etc", lowerRootPath, upperRootPath), + optionsOverlay("--overlay", "/opt", lowerRootPath, upperRootPath), + optionsOverlay("--overlay", "/usr/bin", lowerRootPath, upperRootPath), } - args = append(args, optionsFromBindmountConfig(cfg.Bindmount)...) - args = append(args, optionsFromBindmountConfig(cfg.Machines[mNo].Bindmount)...) + for _, d := range bindmountDirs { + args = append(args, fmt.Sprintf("--bind=%s:%s", path.Join(upperRootPath, d), d)) + } c := utils.Command("kube-spawn", args...) c.Stderr = os.Stderr - // log.Printf(">>> runnning: %q", strings.Join(c.Args, " ")) - stdout, err := c.StdoutPipe() if err != nil { return errors.Wrap(err, "error creating stdout pipe") @@ -92,28 +106,23 @@ func Run(cfg *config.ClusterConfiguration, mNo int) error { } if _, err := cniversion.NewResult(cniversion.Current(), cniDataJSON); err != nil { - log.Printf("unexpected result output: %s", cniDataJSON) - return errors.Wrap(err, "unable to parse result") + return errors.Wrapf(err, "unable to parse CNI data %q", cniDataJSON) } if err := c.Wait(); err != nil { var cniError cnitypes.Error if err := json.Unmarshal(cniDataJSON, &cniError); err != nil { - return errors.Wrap(err, "error unmarshaling cni error") + return errors.Wrapf(err, "error unmarshaling CNI error %q", cniDataJSON) } return errors.Wrap(&cniError, "error running cnispawn") } - if err := waitMachinesRunning(cfg.Machines[mNo].Name); err != nil { - return err - } - cfg.Machines[mNo].Running = true - return nil + return waitMachinesRunning(machineName) } func waitMachinesRunning(machineName string) error { for retries := 0; retries <= 30; retries++ { - if machinetool.IsRunning(machineName) { + if machinectl.IsRunning(machineName) { return nil } time.Sleep(2 * time.Second) diff --git a/pkg/script/docker-daemon-config.go b/pkg/script/docker-daemon-config.go deleted file mode 100644 index 40362277..00000000 --- a/pkg/script/docker-daemon-config.go +++ /dev/null @@ -1,13 +0,0 @@ -package script - -const DockerDaemonConfigPath = "/etc/docker/daemon.json" - -const DockerDaemonConfig = `{ - "insecure-registries": ["10.22.0.1:5000"], - "default-runtime": "custom", - "runtimes": { - "custom": { "path": "/usr/bin/kube-spawn-runc" } - }, - "storage-driver": "overlay2" -} -` diff --git a/pkg/script/docker-kubeadm-extra-args.go b/pkg/script/docker-kubeadm-extra-args.go deleted file mode 100644 index 1c4776ae..00000000 --- a/pkg/script/docker-kubeadm-extra-args.go +++ /dev/null @@ -1,7 +0,0 @@ -package script - -const DockerKubeadmExtraArgsPath = "/etc/systemd/system/docker.service.d/20-kubeadm-extra-args.conf" - -const DockerKubeadmExtraArgs = `[Service] -Environment="DOCKER_OPTS=--exec-opt native.cgroupdriver=cgroupfs" -` diff --git a/pkg/script/kubeadm-bootstrap.go b/pkg/script/kubeadm-bootstrap.go deleted file mode 100644 index 5e3fbaa3..00000000 --- a/pkg/script/kubeadm-bootstrap.go +++ /dev/null @@ -1,37 +0,0 @@ -package script - -import "bytes" - -const KubeadmBootstrapPath = "/opt/kube-spawn/bootstrap.sh" - -const kubeadmBootstrapTmpl = `#!/bin/sh - -set -ex - -echo "root:k8s" | chpasswd -echo "core:core" | chpasswd - -systemctl enable kubelet.service -systemctl enable sshd.service - -{{ if eq .ContainerRuntime "docker" -}}systemctl start --no-block docker.service{{- end}} -{{ if eq .ContainerRuntime "crio" -}}systemctl start --no-block crio.service{{- end}} -{{ if eq .ContainerRuntime "rkt" -}}systemctl start --no-block rktlet.service -ln -sfT /etc/cni/net.d /etc/rkt/net.d{{- end}} - -mkdir -p /var/lib/weave - -# necessary to prevent docker from being blocked. -systemctl mask systemd-networkd-wait-online.service - -kubeadm reset -systemctl start --no-block kubelet.service -` - -type KubeadmBootstrapOpts struct { - ContainerRuntime string -} - -func GetKubeadmBootstrap(opts KubeadmBootstrapOpts) (*bytes.Buffer, error) { - return render(kubeadmBootstrapTmpl, opts) -} diff --git a/pkg/script/kubeadm-config.go b/pkg/script/kubeadm-config.go deleted file mode 100644 index 88bfa375..00000000 --- a/pkg/script/kubeadm-config.go +++ /dev/null @@ -1,29 +0,0 @@ -package script - -import ( - "bytes" -) - -const KubeadmConfigPath = "/etc/kubeadm/kubeadm.yml" - -const kubeadmConfigTmpl = `apiVersion: kubeadm.k8s.io/v1alpha1 -authorizationMode: AlwaysAllow -apiServerExtraArgs: - insecure-port: "8080" -controllerManagerExtraArgs: -kubernetesVersion: {{.KubernetesVersion}} -schedulerExtraArgs: -{{if .DevCluster -}} -unifiedControlPlaneImage: 10.22.0.1:5000/hyperkube-amd64:{{.HyperkubeTag}} -{{- end }} -` - -type KubeadmYmlOpts struct { - DevCluster bool - KubernetesVersion string - HyperkubeTag string -} - -func GetKubeadmConfig(opts KubeadmYmlOpts) (*bytes.Buffer, error) { - return render(kubeadmConfigTmpl, opts) -} diff --git a/pkg/script/kubeadm-extra-args.go b/pkg/script/kubeadm-extra-args.go deleted file mode 100644 index 880f3693..00000000 --- a/pkg/script/kubeadm-extra-args.go +++ /dev/null @@ -1,43 +0,0 @@ -package script - -import ( - "bytes" -) - -const KubeadmExtraArgsPath = "/etc/systemd/system/kubelet.service.d/20-kubeadm-extra-args.conf" - -// NOTE: --fail-swap-on=false is necessary for k8s 1.8 or newer, -// and the option is not available at all in k8s 1.7 or older. -// With that option, kubelet 1.7 or older will not run at all. - -// For rktlet, --container-runtime must be "remote", not "rkt". -// --container-runtime-endpoint needs to point to the unix socket, -// which rktlet listens on. - -// --cgroups-per-qos should be set to false, so that we can avoid issues with -// different formats of cgroup paths between k8s and systemd. -// --enforce-node-allocatable= is also necessary. -const kubeadmExtraArgsTmpl string = `[Service] -Environment="KUBELET_CGROUP_ARGS=--cgroup-driver={{ if .UseLegacyCgroupDriver }}cgroupfs{{else}}systemd{{end}}" -Environment="KUBELET_EXTRA_ARGS=\ -{{ if ne .ContainerRuntime "docker" -}}--container-runtime=remote \ ---container-runtime-endpoint={{.RuntimeEndpoint}} \ ---runtime-request-timeout={{.RequestTimeout}} {{- end}} \ ---enforce-node-allocatable= \ -{{ printf "--cgroups-per-qos=%t" .CgroupsPerQOS }} \ -{{ if not .FailSwapOn -}}--fail-swap-on=false {{- end}} \ ---authentication-token-webhook" -` - -type KubeadmExtraArgsOpts struct { - ContainerRuntime string - UseLegacyCgroupDriver bool - CgroupsPerQOS bool - FailSwapOn bool - RuntimeEndpoint string - RequestTimeout string -} - -func GetKubeadmExtraArgs(opts KubeadmExtraArgsOpts) (*bytes.Buffer, error) { - return render(kubeadmExtraArgsTmpl, opts) -} diff --git a/pkg/script/kubeadm-extra-runtime.go b/pkg/script/kubeadm-extra-runtime.go deleted file mode 100644 index 52707ec2..00000000 --- a/pkg/script/kubeadm-extra-runtime.go +++ /dev/null @@ -1,17 +0,0 @@ -package script - -const kubeadmExtraRuntimeTmpl string = ` -{{ if .RktRuntime -}}--container-runtime=remote \ ---container-runtime-endpoint={{.RuntimeEndpoint}}{{- end}} \ ---runtime-request-timeout={{.RequestTimeout}}" -` - -// For rktlet, --container-runtime must be "remote", not "rkt". -// --container-runtime-endpoint needs to point to the unix socket, -// which rktlet listens on. - -type KubeadmExtraRuntimeOpts struct { - RktRuntime bool - RuntimeEndpoint string - RequestTimeout string -} diff --git a/pkg/script/kubelet-tmpfiles.go b/pkg/script/kubelet-tmpfiles.go deleted file mode 100644 index a1885ab0..00000000 --- a/pkg/script/kubelet-tmpfiles.go +++ /dev/null @@ -1,5 +0,0 @@ -package script - -const KubeletTmpfilesPath = "/etc/tmpfiles.d/kubelet.conf" - -const KubeletTmpfiles = `d /var/lib/kubelet 0755 - - -` diff --git a/pkg/script/rktlet-service.go b/pkg/script/rktlet-service.go deleted file mode 100644 index e2ba1ffd..00000000 --- a/pkg/script/rktlet-service.go +++ /dev/null @@ -1,17 +0,0 @@ -package script - -const RktletServicePath = "/etc/systemd/system/rktlet.service" - -const RktletService = `[Unit] -Description=rktlet: The rkt implementation of a Kubernetes Container Runtime -Documentation=https://github.com/kubernetes-incubator/rktlet/tree/master/docs - -[Service] -ExecStart=/usr/bin/rktlet --net=weave -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -WantedBy=multi-user.target -` diff --git a/pkg/script/script.go b/pkg/script/script.go deleted file mode 100644 index 4887ef43..00000000 --- a/pkg/script/script.go +++ /dev/null @@ -1,16 +0,0 @@ -package script - -import ( - "bytes" - "fmt" - "text/template" -) - -func render(tmpl string, opts interface{}) (*bytes.Buffer, error) { - var buf = new(bytes.Buffer) - t := template.Must(template.New(fmt.Sprintf("%T", opts)).Parse(tmpl)) - if err := t.Execute(buf, opts); err != nil { - return nil, err - } - return buf, nil -} diff --git a/pkg/script/weave-networkd.go b/pkg/script/weave-networkd.go deleted file mode 100644 index 36cb5b80..00000000 --- a/pkg/script/weave-networkd.go +++ /dev/null @@ -1,10 +0,0 @@ -package script - -const WeaveNetworkdUnmaskPath = "/etc/systemd/network/50-weave.network" - -const WeaveNetworkdUnmask = `[Match] -Name=weave datapath vethwe* - -[Link] -Unmanaged=yes -` diff --git a/pkg/utils/fs/fs.go b/pkg/utils/fs/fs.go index a98311f7..d603bcf6 100644 --- a/pkg/utils/fs/fs.go +++ b/pkg/utils/fs/fs.go @@ -52,8 +52,8 @@ func CreateFileFromReader(path string, reader io.Reader) error { return nil } -func CreateFileFromBytes(path string, data []byte) error { - buf := bytes.NewBuffer(data) +func CreateFileFromString(path string, content string) error { + buf := bytes.NewBuffer([]byte(content)) return CreateFileFromReader(path, buf) } diff --git a/pkg/utils/kubernetes.go b/pkg/utils/kubernetes.go deleted file mode 100644 index 8843eee8..00000000 --- a/pkg/utils/kubernetes.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2017 Kinvolk GmbH - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "fmt" - "log" - "os" - "path" - "path/filepath" - "syscall" - "unsafe" - - "golang.org/x/sys/unix" -) - -const ( - ksHiddenDir string = ".kube-spawn" - ksRelPath string = "src/github.com/kinvolk/kube-spawn" -) - -var ( - homePath string = os.Getenv("HOME") - goPath string = os.Getenv("GOPATH") -) - -func CheckValidDir(inPath string) error { - if fi, err := os.Stat(inPath); os.IsNotExist(err) { - return err - } else if !fi.IsDir() { - return fmt.Errorf("%q is not a directory.", inPath) - } - return nil -} - -func CheckValidFile(inPath string) error { - if fi, err := os.Stat(inPath); os.IsNotExist(err) { - return err - } else if !fi.Mode().IsRegular() { - return fmt.Errorf("%q is not a file.", inPath) - } - return nil -} - -func GetValidGoPath() (string, error) { - if err := CheckValidDir(goPath); err != nil { - // fall back to $HOME/go - goPath = path.Join(homePath, "go") - if err := CheckValidDir(goPath); err != nil { - return "", err - } - } - - return goPath, nil -} - -func GetKubeconfigPath(kubeSpawnDir, clusterName string) string { - kcRelPath := filepath.Join(clusterName, "kubeconfig") - kcUserPath := filepath.Join(ksHiddenDir, kcRelPath) - kcSystemPath := filepath.Join(kubeSpawnDir, kcRelPath) - - kcPath := kcSystemPath - if err := CheckValidFile(kcPath); err != nil { - // fall back to $GOPATH/src/github.com/kinvolk/kube-spawn/.kube-spawn/default/kubeconfig - kcPath = filepath.Join(goPath, ksRelPath, kcUserPath) - log.Printf("fall back to %s...\n", kcPath) - - if err := CheckValidFile(kcPath); err != nil { - // fall back to $HOME/go/src/github.com/kinvolk/kube-spawn/.kube-spawn/default/kubeconfig - kcPath = filepath.Join(homePath, "go", ksRelPath, kcUserPath) - log.Printf("fall back to %s...\n", kcPath) - if err := CheckValidFile(kcPath); err != nil { - return "" - } - } - } - - return kcPath -} - -func GetK8sBuildOutputDir() (string, error) { - goPath, err := GetValidGoPath() - if err != nil { - return "", err - } - k8sRepoPath := filepath.Join(goPath, "/src/k8s.io/kubernetes") - // first try to use "_output/dockerized/bin/linux/amd64" - outputPath := filepath.Join(k8sRepoPath, "_output/dockerized/bin/linux/amd64") - if err := CheckValidDir(outputPath); err != nil { - // fall back to "_output/bin" - outputPath = filepath.Join(k8sRepoPath, "_output/bin") - if err := CheckValidDir(outputPath); err != nil { - return "", err - } - } - - return outputPath, nil -} - -func GetK8sBuildAssetDir() (string, error) { - goPath, err := GetValidGoPath() - if err != nil { - return "", err - } - k8sAssetPath := filepath.Join(goPath, "/src/k8s.io/kubernetes/build") - if err := CheckValidDir(k8sAssetPath); err != nil { - return "", err - } - return k8sAssetPath, nil -} - -// IsTerminal returns true if the given file descriptor is a terminal. -func IsTerminal(fd uintptr) bool { - var termios syscall.Termios - _, _, err := unix.Syscall(unix.SYS_IOCTL, fd, uintptr(syscall.TCGETS), uintptr(unsafe.Pointer(&termios))) - return err == 0 -} diff --git a/pkg/utils/terminal.go b/pkg/utils/terminal.go new file mode 100644 index 00000000..9efc6351 --- /dev/null +++ b/pkg/utils/terminal.go @@ -0,0 +1,31 @@ +/* +Copyright 2017 Kinvolk GmbH + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +// IsTerminal returns true if the given file descriptor is a terminal. +func IsTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := unix.Syscall(unix.SYS_IOCTL, fd, uintptr(syscall.TCGETS), uintptr(unsafe.Pointer(&termios))) + return err == 0 +} diff --git a/pkg/utils/version.go b/pkg/utils/version.go deleted file mode 100644 index ef242c8c..00000000 --- a/pkg/utils/version.go +++ /dev/null @@ -1,13 +0,0 @@ -package utils - -import "github.com/Masterminds/semver" - -func CheckVersionConstraint(version, constraint string) bool { - v, err := semver.NewVersion(version) - if err != nil { - return false - } - - c, _ := semver.NewConstraint(constraint) - return c.Check(v) -}