RFC: kube-bootstrap

cluster/aws/templates/download-release.sh

@@ -14,21 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Download and install release
-
-# This script assumes that the environment variable MASTER_RELEASE_TAR contains
-# the release tar to download and unpack.  It is meant to be pushed to the
-# master and run.
+# Download release
 
 echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
 download-or-bust "$SERVER_BINARY_TAR_URL"
 
 echo "Downloading binary release tar ($SALT_TAR_URL)"
 download-or-bust "$SALT_TAR_URL"
-
-echo "Unpacking Salt tree"
-rm -rf kubernetes
-tar xzf "${SALT_TAR_URL##*/}"
-
-echo "Running release install script"
-sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"

cluster/aws/templates/install-release.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Untar downloaded files & install release
+
+echo "Unpacking Salt tree"
+rm -rf kubernetes
+tar xzf "${SALT_TAR_URL##*/}"
+
+echo "Running release install script"
+sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"

cluster/aws/templates/master-bootstrap.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "Extracting & running kube-bootstrap"
+rm -f kube-bootstrap
+tar xzf "${SERVER_BINARY_TAR_URL##*/}" --strip-components=3 kubernetes/server/bin/kube-bootstrap
+
+mkdir -p /etc/kubernetes/
+echo "${BOOTSTRAP_JSON}" >/etc/kubernetes/bootstrap.json
+
+./kube-bootstrap --config /etc/kubernetes/bootstrap.json

cluster/aws/templates/setup-master-pd.sh

@@ -17,25 +17,6 @@
 # Format and mount the disk, create directories on it for all of the master's
 # persistent data, and link them to where they're used.
 
-echo "Waiting for master pd to be attached"
-attempt=0
-while true; do
-  echo Attempt "$(($attempt+1))" to check for /dev/xvdb
-  if [[ -e /dev/xvdb ]]; then
-    echo "Found /dev/xvdb"
-    break
-  fi
-  attempt=$(($attempt+1))
-  sleep 1
-done
-
-# Mount Master Persistent Disk
-echo "Mounting master-pd"
-mkdir -p /mnt/master-pd
-mkfs -t ext4 /dev/xvdb
-echo "/dev/xvdb  /mnt/master-pd  ext4  noatime  0 0" >> /etc/fstab
-mount /mnt/master-pd
-
 # Contains all the data stored in etcd
 mkdir -m 700 -p /mnt/master-pd/var/etcd
 # Contains the dynamically generated apiserver auth certs and keys

cluster/aws/util.sh

@@ -857,12 +857,21 @@ function start-master() {
   service_ip=$(echo "${octets[*]}" | sed 's/ /./g')
   MASTER_EXTRA_SANS="IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}"
 
+  # Build bootstrap config
+  NL=$'\n'
+  BOOTSTRAP_JSON="{"${NL}
+  BOOTSTRAP_JSON+=' "MasterCIDR": "'${MASTER_IP_RANGE}'",'${NL}
+  BOOTSTRAP_JSON+=' "ClusterID": "'${CLUSTER_ID}'",'${NL}
+  BOOTSTRAP_JSON+=' "MasterVolume": "'${MASTER_DISK_ID}'",'${NL}
+  BOOTSTRAP_JSON+=' "CloudProvider": "aws"'${NL}
+  BOOTSTRAP_JSON+="}"
 
   (
     # We pipe this to the ami as a startup script in the user-data field.  Requires a compatible ami
     echo "#! /bin/bash"
     echo "mkdir -p /var/cache/kubernetes-install"
     echo "cd /var/cache/kubernetes-install"
+    echo "readonly BOOTSTRAP_JSON='${BOOTSTRAP_JSON}'"
     echo "readonly SALT_MASTER='${MASTER_INTERNAL_IP}'"
     echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
     echo "readonly NODE_INSTANCE_PREFIX='${NODE_INSTANCE_PREFIX}'"
@@ -898,13 +907,22 @@ function start-master() {
     echo "readonly OPENCONTRAIL_PUBLIC_SUBNET='${OPENCONTRAIL_PUBLIC_SUBNET:-}'"
     echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/download-release.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/master-bootstrap.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/setup-master-pd.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh"
-    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/download-release.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/install-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/salt-master.sh"
   ) > "${KUBE_TEMP}/master-start.sh"
 
+  # We're running right up against the 16KB limit
+  # Remove all comment lines and then put back the bin/bash shebang
+  sed -i -e 's/^[[:blank:]]*#.*$//' -e '/^[[:blank:]]*$/d' "${KUBE_TEMP}/master-start.sh"
+  sed -i '1i #! /bin/bash' "${KUBE_TEMP}/master-start.sh"
+
+  cp "${KUBE_TEMP}/master-start.sh" /tmp
+
   echo "Starting Master"
   master_id=$($AWS_CMD run-instances \
     --image-id $AWS_IMAGE \
@@ -936,21 +954,12 @@ function start-master() {
         exit 1
       fi
     else
-      # We are not able to add an elastic ip, a route or volume to the instance until that instance is in "running" state.
       wait-for-instance-running $master_id
 
       KUBE_MASTER=${MASTER_NAME}
       KUBE_MASTER_IP=$(assign-elastic-ip $ip $master_id)
       echo -e " ${color_green}[master running @${KUBE_MASTER_IP}]${color_norm}"
 
-      # This is a race between instance start and volume attachment.  There appears to be no way to start an AWS instance with a volume attached.
-      # To work around this, we wait for volume to be ready in setup-master-pd.sh
-      echo "Attaching persistent data volume (${MASTER_DISK_ID}) to master"
-      $AWS_CMD attach-volume --volume-id ${MASTER_DISK_ID} --device /dev/sdb --instance-id ${master_id}
-
-      sleep 10
-      $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MASTER_IP_RANGE} --instance-id $master_id > $LOG
-
       break
     fi
     echo -e " ${color_yellow}[master not working yet]${color_norm}"
@@ -1325,6 +1334,7 @@ function kube-push {
     echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/download-release.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/install-release.sh"
     echo "echo Executing configuration"
     echo "sudo salt '*' mine.update"
     echo "sudo salt --force-color '*' state.highstate"

cmd/kube-bootstrap/app/plugins.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+// This file exists to force the desired plugin implementations to be linked.
+import (
+	//Cloud providers
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
+)

cmd/kube-bootstrap/app/server.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package app does all of the work necessary to configure and run a
+// Kubernetes app process.
+package app
+
+import (
+	"k8s.io/kubernetes/pkg/bootstrap"
+
+	"github.com/golang/glog"
+	"github.com/spf13/pflag"
+	"time"
+)
+
+// BootstrapServerConfig contains configures and runs a Kubernetes proxy server
+type BootstrapServerConfig struct {
+	ConfigFile string
+}
+
+type BootstrapServer struct {
+	Config       *BootstrapServerConfig
+	Bootstrapper *bootstrap.Bootstrapper
+}
+
+// AddFlags adds flags for a specific BootstrapServer to the specified FlagSet
+func (s *BootstrapServerConfig) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&s.ConfigFile, "config", s.ConfigFile, "The path to the configuration file.")
+}
+
+func NewBootstrapServerConfig() *BootstrapServerConfig {
+	return &BootstrapServerConfig{
+		ConfigFile: "/etc/kubernetes/config",
+	}
+}
+
+func NewBootstrapServer(config *BootstrapServerConfig, bootstrapper *bootstrap.Bootstrapper) (*BootstrapServer, error) {
+	return &BootstrapServer{
+		Config:       config,
+		Bootstrapper: bootstrapper,
+	}, nil
+}
+
+// NewBootstrapServerDefault creates a new BootstrapServer BootstrapServer with default parameters.
+func NewBootstrapServerDefault(config *BootstrapServerConfig) (*BootstrapServer, error) {
+	bootstrapper := bootstrap.NewBootstrapper(config.ConfigFile)
+	return NewBootstrapServer(config, bootstrapper)
+}
+
+// Run runs the specified BootstrapServer, which bootstraps the machine.
+// If it encounters an error, it will sleep and retry.
+// Thus on exit the bootstrap has succeeded.
+func (s *BootstrapServer) Run(_ []string) error {
+	for {
+		err := s.Bootstrapper.RunOnce()
+		if err == nil {
+			return nil
+		}
+		glog.Warning("error during bootstrapping (will sleep and retry): %v", err)
+		time.Sleep(10 * time.Second)
+	}
+}

cmd/kube-bootstrap/bootstrap.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+
+	"k8s.io/kubernetes/cmd/kube-bootstrap/app"
+	"k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/pkg/version/verflag"
+
+	"github.com/spf13/pflag"
+)
+
+func main() {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	config := app.NewBootstrapServerConfig()
+	config.AddFlags(pflag.CommandLine)
+
+	util.InitFlags()
+	util.InitLogs()
+	defer util.FlushLogs()
+
+	verflag.PrintAndExitIfRequested()
+
+	s, err := app.NewBootstrapServerDefault(config)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	if err = s.Run(pflag.CommandLine.Args()); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

hack/lib/golang.sh

@@ -30,6 +30,7 @@ kube::golang::server_targets() {
   local targets=(
     cmd/kube-proxy
     cmd/kube-apiserver
+    cmd/kube-bootstrap
     cmd/kube-controller-manager
     cmd/kubelet
     cmd/kubemark