diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index ba1080edea48..ff850d8e68cd 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -14,7 +14,7 @@ {% set srv_sshproxy_path = "/srv/sshproxy" -%} {% if grains.cloud is defined -%} - {% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller', 'azure-legacy'] -%} + {% if grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%} {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} {% endif -%} @@ -22,7 +22,7 @@ {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% endif -%} - {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} + {% if grains.cloud in [ 'vsphere', 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%} diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index ecf83f69462f..227fa71465f2 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -41,7 +41,7 @@ {% set srv_kube_path = "/srv/kubernetes" -%} {% if grains.cloud is defined -%} - {% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller', 'azure-legacy'] -%} + {% if grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%} {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} {% endif -%} {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%} @@ -50,7 +50,7 @@ {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% endif -%} - {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} + {% if grains.cloud in [ 'vsphere', 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%} diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 374d6922e132..ad01f65547b6 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -48,12 +48,12 @@ {% endif -%} {% set cloud_provider = "" -%} -{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'vsphere', 'photon-controller', 'azure-legacy'] -%} +{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%} {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} {% endif -%} {% set cloud_config = "" -%} -{% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%} +{% if grains.cloud in [ 'openstack', 'vsphere' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% endif -%} diff --git a/cluster/vsphere/config-common.sh b/cluster/vsphere/config-common.sh index 3cd82618df36..f2c4bffd4636 100644 --- a/cluster/vsphere/config-common.sh +++ b/cluster/vsphere/config-common.sh @@ -17,12 +17,19 @@ SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR -C" # These need to be set -#export GOVC_URL= -#export GOVC_DATACENTER= -#export GOVC_DATASTORE= -#export GOVC_NETWORK= -#export GOVC_GUEST_LOGIN= +# export GOVC_URL='hostname' # hostname of the vc +# export GOVC_USERNAME='username' # username for logging into the vsphere. +# export GOVC_PASSWORD='password' # password for the above username +# export GOVC_NETWORK='Network Name' # Name of the network the vms should join. Many times it could be "VM Network" +# export GOVC_DATASTORE='target datastore' +# To get resource pool via govc: govc ls -l 'host/*' | grep ResourcePool | awk '{print $1}' | xargs -n1 -t govc pool.info +# export GOVC_RESOURCE_POOL='resource pool or cluster with access to datastore' +# export GOVC_GUEST_LOGIN='kube:kube' # Used for logging into kube.vmdk during deployment. +# export GOVC_PORT=443 # The port to be used by vSphere cloud provider plugin +# To get datacente via govc: govc datacenter.info +# export GOVC_DATACENTER='ha-datacenter' # The datacenter to be used by vSphere cloud provider plugin +# export GOVC_GUEST_LOGIN='kube:kube' # Used for logging into kube.vmdk during deployment. # Set GOVC_INSECURE if the host in GOVC_URL is using a certificate that cannot # be verified (i.e. a self-signed certificate), but IS trusted. -#export GOVC_INSECURE=1 +# export GOVC_INSECURE=1 diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh index e646dfa5000b..83c47b79f06a 100755 --- a/cluster/vsphere/config-default.sh +++ b/cluster/vsphere/config-default.sh @@ -17,6 +17,7 @@ NUM_NODES=4 DISK=./kube/kube.vmdk GUEST_ID=debian7_64Guest +ENABLE_UUID=TRUE INSTANCE_PREFIX=kubernetes MASTER_TAG="${INSTANCE_PREFIX}-master" diff --git a/cluster/vsphere/templates/salt-master.sh b/cluster/vsphere/templates/salt-master.sh index 0d86d6b6fb43..871b715d2805 100755 --- a/cluster/vsphere/templates/salt-master.sh +++ b/cluster/vsphere/templates/salt-master.sh @@ -20,6 +20,22 @@ sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list # Prepopulate the name of the Master mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf +CLOUD_CONFIG=/etc/vsphere_cloud.config + +# Configuration to initialize vsphere cloud provider +cat < $CLOUD_CONFIG +[Global] + user = $GOVC_USERNAME + password = $GOVC_PASSWORD + server = $GOVC_URL + port = $GOVC_PORT + insecure-flag = $GOVC_INSECURE + datacenter = $GOVC_DATACENTER + datastore = $GOVC_DATASTORE + +[Disk] + scsicontrollertype = pvscsi +EOF cat </etc/salt/minion.d/grains.conf grains: @@ -29,6 +45,7 @@ grains: cloud: vsphere master_extra_sans: $MASTER_EXTRA_SANS kube_user: $KUBE_USER + cloud_config: $CLOUD_CONFIG EOF # Auto accept all keys from minions that try to join @@ -53,5 +70,5 @@ EOF # # -M installs the master set +x -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -M -X +curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -M -X stable 2016.3.2 set -x diff --git a/cluster/vsphere/templates/salt-minion.sh b/cluster/vsphere/templates/salt-minion.sh index 4ba1a4f275d9..2b4e40d725ce 100755 --- a/cluster/vsphere/templates/salt-minion.sh +++ b/cluster/vsphere/templates/salt-minion.sh @@ -30,6 +30,23 @@ echo "master: $KUBE_MASTER" > /etc/salt/minion.d/master.conf # Turn on debugging for salt-minion # echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion +# Configuration to initialize vsphere cloud provider +CLOUD_CONFIG=/etc/vsphere_cloud.config + +cat < $CLOUD_CONFIG +[Global] + user = $GOVC_USERNAME + password = $GOVC_PASSWORD + server = $GOVC_URL + port = $GOVC_PORT + insecure-flag = $GOVC_INSECURE + datacenter = $GOVC_DATACENTER + datastore = $GOVC_DATASTORE + +[Disk] + scsicontrollertype = pvscsi +EOF + # Our minions will have a pool role to distinguish them from the master. # # Setting the "minion_ip" here causes the kubelet to use its IP for @@ -37,15 +54,15 @@ echo "master: $KUBE_MASTER" > /etc/salt/minion.d/master.conf # cat </etc/salt/minion.d/grains.conf grains: - hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}') roles: - kubernetes-pool - kubernetes-pool-vsphere cloud: vsphere + cloud_config: $CLOUD_CONFIG EOF # Install Salt # # We specify -X to avoid a race condition that can cause minion failure to # install. See https://github.com/saltstack/salt-bootstrap/issues/270 -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X +curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X stable 2016.3.2 diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh index e32379670c25..deab193d7d0f 100755 --- a/cluster/vsphere/util.sh +++ b/cluster/vsphere/util.sh @@ -180,12 +180,17 @@ function kube-up-vm { -debug \ -disk="${DISK}" \ -g="${GUEST_ID}" \ + -on=false \ -link=true \ "$@" \ "${vm_name}" + govc vm.change -e="disk.enableUUID=${ENABLE_UUID}" -vm="${vm_name}" + + govc vm.power -on=true "${vm_name}" + # Retrieve IP first, to confirm the guest operations agent is running. - govc vm.ip "${vm_name}" > /dev/null + CURRENT_NODE_IP=$(govc vm.ip "${vm_name}") govc guest.mkdir \ -l "kube:kube" \ @@ -395,6 +400,13 @@ function kube-up { echo "readonly MASTER_HTPASSWD='${htpasswd}'" echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'" echo "readonly MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'" + echo "readonly GOVC_USERNAME='${GOVC_USERNAME}'" + echo "readonly GOVC_PASSWORD='${GOVC_PASSWORD}'" + echo "readonly GOVC_URL='${GOVC_URL}'" + echo "readonly GOVC_PORT='${GOVC_PORT}'" + echo "readonly GOVC_INSECURE='${GOVC_INSECURE}'" + echo "readonly GOVC_DATACENTER='${GOVC_DATACENTER}'" + echo "readonly GOVC_DATASTORE='${GOVC_DATASTORE}'" grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/create-dynamic-salt-files.sh" grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/install-release.sh" grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-master.sh" @@ -416,11 +428,23 @@ function kube-up { echo "KUBE_MASTER=${KUBE_MASTER}" echo "KUBE_MASTER_IP=${KUBE_MASTER_IP}" echo "NODE_IP_RANGE=$NODE_IP_RANGES" + echo "readonly GOVC_USERNAME='${GOVC_USERNAME}'" + echo "readonly GOVC_PASSWORD='${GOVC_PASSWORD}'" + echo "readonly GOVC_URL='${GOVC_URL}'" + echo "readonly GOVC_PORT='${GOVC_PORT}'" + echo "readonly GOVC_INSECURE='${GOVC_INSECURE}'" + echo "readonly GOVC_DATACENTER='${GOVC_DATACENTER}'" + echo "readonly GOVC_DATASTORE='${GOVC_DATASTORE}'" grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-minion.sh" ) > "${KUBE_TEMP}/node-start-${i}.sh" ( kube-up-vm "${NODE_NAMES[$i]}" -c ${NODE_CPU-1} -m ${NODE_MEMORY_MB-1024} + add_to_hosts="${NODE_NAMES[$i]} ${CURRENT_NODE_IP}" + node_ip_file=${NODE_NAMES[$i]}-ip + echo "sudo bash -c \"echo $add_to_hosts >> /etc/hosts\"" > ${KUBE_TEMP}/${node_ip_file} + kube-scp ${KUBE_MASTER_IP} ${KUBE_TEMP}/${node_ip_file} /tmp/ + kube-ssh ${KUBE_MASTER_IP} "bash /tmp/${node_ip_file}" kube-run "${NODE_NAMES[$i]}" "${KUBE_TEMP}/node-start-${i}.sh" ) & done @@ -441,18 +465,18 @@ function kube-up { printf "Waiting for salt-master to be up on ${KUBE_MASTER} ...\n" remote-pgrep ${KUBE_MASTER_IP} "salt-master" - printf "Waiting for all packages to be installed on ${KUBE_MASTER} ...\n" - kube-check ${KUBE_MASTER_IP} 'sudo salt "kubernetes-master" state.highstate -t 30 | grep -E "Failed:[[:space:]]+0"' - local i for (( i=0; i<${#NODE_NAMES[@]}; i++)); do printf "Waiting for salt-minion to be up on ${NODE_NAMES[$i]} ....\n" remote-pgrep ${KUBE_NODE_IP_ADDRESSES[$i]} "salt-minion" - printf "Waiting for all salt packages to be installed on ${NODE_NAMES[$i]} .... \n" - kube-check ${KUBE_MASTER_IP} 'sudo salt '"${NODE_NAMES[$i]}"' state.highstate -t 30 | grep -E "Failed:[[:space:]]+0"' printf " OK\n" done + printf "Waiting for init highstate to be done on all nodes (this can take a few minutes) ...\n" + kube-check ${KUBE_MASTER_IP} 'sudo salt '\''*'\'' state.show_highstate -t 50' + printf "Waiting for all packages to be installed on all nodes (this can take a few minutes) ...\n" + kube-check ${KUBE_MASTER_IP} 'sudo salt '\''*'\'' state.highstate -t 50 | grep -E "Failed:[[:space:]]+0"' + echo echo "Waiting for master and node initialization." diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index c284e88a4efb..0bfaf96e14fd 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -67,6 +67,8 @@ cluster/ubuntu/util.sh: local node_ip=${1} cluster/vagrant/provision-utils.sh: api_servers: '$(echo "$MASTER_IP" | sed -e "s/'/''/g")' cluster/vagrant/provision-utils.sh: node_ip: '$(echo "$MASTER_IP" | sed -e "s/'/''/g")' cluster/vagrant/provision-utils.sh: runtime_config: '$(echo "$RUNTIME_CONFIG" | sed -e "s/'/''/g")' +cluster/vsphere/templates/salt-master.sh: cloud_config: $CLOUD_CONFIG +cluster/vsphere/templates/salt-minion.sh: cloud_config: $CLOUD_CONFIG cluster/vsphere/templates/salt-minion.sh: hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}') examples/cluster-dns/images/frontend/client.py: service_address = socket.gethostbyname(hostname) examples/storage/cassandra/image/run.sh: cluster_name \ diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index edaf74ee40bc..d5aeacbaae54 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -215,7 +215,7 @@ func readInstance(cfg *VSphereConfig) (string, string, error) { var rp mo.ResourcePool err = s.Properties(ctx, *vm.ResourcePool, []string{"parent"}, &rp) if err == nil { - var ccr mo.ClusterComputeResource + var ccr mo.ComputeResource err = s.Properties(ctx, *rp.Parent, []string{"name"}, &ccr) if err == nil { cluster = ccr.Name