Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove credentials on worker nodes for vSphere cloud provider. #43545

Merged
merged 1 commit into from
Apr 11, 2017
Merged

Remove credentials on worker nodes for vSphere cloud provider. #43545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
240 changes: 111 additions & 129 deletions pkg/cloudprovider/providers/vsphere/vsphere.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"fmt"
"io"
"io/ioutil"
"net"
"net/url"
"path"
"path/filepath"
Expand Down Expand Up @@ -68,6 +69,8 @@ const (
RoundTripperDefaultCount = 3
DummyVMName = "kubernetes-helper-vm"
VSANDatastoreType = "vsan"
MAC_OUI_VC = "00:50:56"
MAC_OUI_ESX = "00:0c:29"
)

// Controller types that are currently supported for hot attach of disks
Expand Down Expand Up @@ -128,6 +131,10 @@ type VSphereConfig struct {
// property in VmConfigInfo, or also set as vc.uuid in VMX file.
// If not set, will be fetched from the machine via sysfs (requires root)
VMUUID string `gcfg:"vm-uuid"`
// VMName is the VM name of virtual machine
// Combining the WorkingDir and VMName can form a unique InstanceID.
// When vm-name is set, no username/password is required on worker nodes.
VMName string `gcfg:"vm-name"`
}

Network struct {
Expand Down Expand Up @@ -280,14 +287,22 @@ func newVSphere(cfg VSphereConfig) (*VSphere, error) {
glog.Warningf("port is a deprecated field in vsphere.conf and will be removed in future release.")
}

c, err := newClient(context.TODO(), &cfg)
if err != nil {
return nil, err
}
var c *govmomi.Client
var id string
if cfg.Global.VMName == "" {
// if VMName is not set in the cloud config file, each nodes (including worker nodes) need credentials to obtain VMName from vCenter
glog.V(4).Infof("Cannot find VMName from cloud config file, start obtaining it from vCenter")
c, err := newClient(context.TODO(), &cfg)
if err != nil {
return nil, err
}

id, err := getVMName(c, &cfg)
if err != nil {
return nil, err
id, err = getVMName(c, &cfg)
if err != nil {
return nil, err
}
} else {
id = cfg.Global.VMName
}

vs := VSphere{
Expand All @@ -311,7 +326,9 @@ func checkControllerSupported(ctrlType string) bool {
}

func logout(vs *VSphere) {
vs.client.Logout(context.TODO())
if vs.client != nil {
vs.client.Logout(context.TODO())
}
}

func newClient(ctx context.Context, cfg *VSphereConfig) (*govmomi.Client, error) {
Expand Down Expand Up @@ -396,119 +413,102 @@ func getVirtualMachineByName(ctx context.Context, cfg *VSphereConfig, c *govmomi
return vm, nil
}

func getVirtualMachineManagedObjectReference(ctx context.Context, c *govmomi.Client, vm *object.VirtualMachine, field string, dst interface{}) error {
collector := property.DefaultCollector(c.Client)

// Retrieve required field from VM object
err := collector.RetrieveOne(ctx, vm.Reference(), []string{field}, dst)
if err != nil {
return err
}
return nil
// Instances returns an implementation of Instances for vSphere.
func (vs *VSphere) Instances() (cloudprovider.Instances, bool) {
return vs, true
}

// Returns names of running VMs inside VM folder.
func getInstances(ctx context.Context, cfg *VSphereConfig, c *govmomi.Client, filter string) ([]string, error) {
f := find.NewFinder(c.Client, true)
dc, err := f.Datacenter(ctx, cfg.Global.Datacenter)
if err != nil {
return nil, err
}

f.SetDatacenter(dc)

vmRegex := cfg.Global.WorkingDir + filter

//TODO: get all vms inside subfolders
vms, err := f.VirtualMachineList(ctx, vmRegex)
if err != nil {
return nil, err
}

var vmRef []types.ManagedObjectReference
for _, vm := range vms {
vmRef = append(vmRef, vm.Reference())
}

pc := property.DefaultCollector(c.Client)
func getLocalIP() ([]v1.NodeAddress, error) {
addrs := []v1.NodeAddress{}

var vmt []mo.VirtualMachine
err = pc.Retrieve(ctx, vmRef, []string{"name", "summary"}, &vmt)
ifaces, err := net.Interfaces()
if err != nil {
glog.Errorf("net.Interfaces() failed for NodeAddresses - %v", err)
return nil, err
}

var vmList []string
for _, vm := range vmt {
if vm.Summary.Runtime.PowerState == ActivePowerState {
vmList = append(vmList, vm.Name)
} else if vm.Summary.Config.Template == false {
glog.Warningf("VM %s, is not in %s state", vm.Name, ActivePowerState)
for _, i := range ifaces {
localAddrs, err := i.Addrs()
if err != nil {
glog.Warningf("Failed to extract addresses for NodeAddresses - %v", err)
} else {
for _, addr := range localAddrs {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
// Filter external IP by MAC address OUIs from vCenter and from ESX
var addressType v1.NodeAddressType
if strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_VC) ||
strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_ESX) {
addressType = v1.NodeExternalIP
} else {
addressType = v1.NodeInternalIP
}
v1.AddToNodeAddresses(&addrs,
v1.NodeAddress{
Type: addressType,
Address: ipnet.IP.String(),
},
)
glog.V(4).Infof("Find local IP address %v and set type to %v", ipnet.IP.String(), addressType)
}
}
}
}
}
return vmList, nil
}

type Instances struct {
client *govmomi.Client
cfg *VSphereConfig
localInstanceID string
return addrs, nil
}

// Instances returns an implementation of Instances for vSphere.
func (vs *VSphere) Instances() (cloudprovider.Instances, bool) {
// getVMandMO returns the VM object and required field from the VM object
func (vs *VSphere) getVMandMO(ctx context.Context, nodeName k8stypes.NodeName, field string) (vm *object.VirtualMachine, mvm *mo.VirtualMachine, err error) {
// Ensure client is logged in and session is valid
err := vSphereLogin(context.TODO(), vs)
err = vSphereLogin(ctx, vs)
if err != nil {
glog.Errorf("Failed to login into vCenter - %v", err)
return nil, false
return nil, nil, err
}
return &Instances{vs.client, vs.cfg, vs.localInstanceID}, true
}

// List returns names of VMs (inside vm folder) by applying filter and which are currently running.
func (vs *VSphere) list(filter string) ([]k8stypes.NodeName, error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

vmList, err := getInstances(ctx, vs.cfg, vs.client, filter)
vm, err = getVirtualMachineByName(ctx, vs.cfg, vs.client, nodeName)
if err != nil {
return nil, err
if _, ok := err.(*find.NotFoundError); ok {
return nil, nil, cloudprovider.InstanceNotFound
}
return nil, nil, err
}

glog.V(3).Infof("Found %d instances matching %s: %s",
len(vmList), filter, vmList)

var nodeNames []k8stypes.NodeName
for _, n := range vmList {
nodeNames = append(nodeNames, k8stypes.NodeName(n))
// Retrieve required field from VM object
var movm mo.VirtualMachine
collector := property.DefaultCollector(vs.client.Client)
err = collector.RetrieveOne(ctx, vm.Reference(), []string{field}, &movm)
if err != nil {
return nil, nil, err
}
return nodeNames, nil

return vm, &movm, nil
}

// NodeAddresses is an implementation of Instances.NodeAddresses.
func (i *Instances) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, error) {
func (vs *VSphere) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, error) {
if vs.localInstanceID == nodeNameToVMName(nodeName) {
/* Get local IP addresses if node is local node */
return getLocalIP()
}

addrs := []v1.NodeAddress{}

// Create context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

vm, err := getVirtualMachineByName(ctx, i.cfg, i.client, nodeName)
_, mvm, err := vs.getVMandMO(ctx, nodeName, "guest.net")
if err != nil {
return nil, err
}

var mvm mo.VirtualMachine
err = getVirtualMachineManagedObjectReference(ctx, i.client, vm, "guest.net", &mvm)
if err != nil {
return nil, err
glog.Errorf("Failed to getVMandMO for NodeAddresses: err %v", err)
return addrs, err
}

// retrieve VM's ip(s)
for _, v := range mvm.Guest.Net {
var addressType v1.NodeAddressType
if i.cfg.Network.PublicNetwork == v.Network {
if vs.cfg.Network.PublicNetwork == v.Network {
addressType = v1.NodeExternalIP
} else {
addressType = v1.NodeInternalIP
Expand All @@ -528,16 +528,16 @@ func (i *Instances) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress,
// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
func (vs *VSphere) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
return []v1.NodeAddress{}, errors.New("unimplemented")
}

func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error {
func (vs *VSphere) AddSSHKeyToAllInstances(user string, keyData []byte) error {
return errors.New("unimplemented")
}

func (i *Instances) CurrentNodeName(hostname string) (k8stypes.NodeName, error) {
return k8stypes.NodeName(i.localInstanceID), nil
func (vs *VSphere) CurrentNodeName(hostname string) (k8stypes.NodeName, error) {
return vmNameToNodeName(vs.localInstanceID), nil
}

// nodeNameToVMName maps a NodeName to the vmware infrastructure name
Expand All @@ -551,22 +551,18 @@ func vmNameToNodeName(vmName string) k8stypes.NodeName {
}

// ExternalID returns the cloud provider ID of the node with the specified Name (deprecated).
func (i *Instances) ExternalID(nodeName k8stypes.NodeName) (string, error) {
func (vs *VSphere) ExternalID(nodeName k8stypes.NodeName) (string, error) {
if vs.localInstanceID == nodeNameToVMName(nodeName) {
return vs.cfg.Global.WorkingDir + vs.localInstanceID, nil
}

// Create context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

vm, err := getVirtualMachineByName(ctx, i.cfg, i.client, nodeName)
if err != nil {
if _, ok := err.(*find.NotFoundError); ok {
return "", cloudprovider.InstanceNotFound
}
return "", err
}

var mvm mo.VirtualMachine
err = getVirtualMachineManagedObjectReference(ctx, i.client, vm, "summary", &mvm)
vm, mvm, err := vs.getVMandMO(ctx, nodeName, "summary")
if err != nil {
glog.Errorf("Failed to getVMandMO for ExternalID: err %v", err)
return "", err
}

Expand All @@ -584,22 +580,18 @@ func (i *Instances) ExternalID(nodeName k8stypes.NodeName) (string, error) {
}

// InstanceID returns the cloud provider ID of the node with the specified Name.
func (i *Instances) InstanceID(nodeName k8stypes.NodeName) (string, error) {
func (vs *VSphere) InstanceID(nodeName k8stypes.NodeName) (string, error) {
if vs.localInstanceID == nodeNameToVMName(nodeName) {
return vs.cfg.Global.WorkingDir + vs.localInstanceID, nil
}

// Create context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

vm, err := getVirtualMachineByName(ctx, i.cfg, i.client, nodeName)
if err != nil {
if _, ok := err.(*find.NotFoundError); ok {
return "", cloudprovider.InstanceNotFound
}
return "", err
}

var mvm mo.VirtualMachine
err = getVirtualMachineManagedObjectReference(ctx, i.client, vm, "summary", &mvm)
vm, mvm, err := vs.getVMandMO(ctx, nodeName, "summary")
if err != nil {
glog.Errorf("Failed to getVMandMO for InstanceID: err %v", err)
return "", err
}

Expand All @@ -619,11 +611,11 @@ func (i *Instances) InstanceID(nodeName k8stypes.NodeName) (string, error) {
// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here
func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) {
func (vs *VSphere) InstanceTypeByProviderID(providerID string) (string, error) {
return "", errors.New("unimplemented")
}

func (i *Instances) InstanceType(name k8stypes.NodeName) (string, error) {
func (vs *VSphere) InstanceType(name k8stypes.NodeName) (string, error) {
return "", nil
}

Expand Down Expand Up @@ -940,7 +932,7 @@ func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (b
vSphereInstance = nodeNameToVMName(nodeName)
}

nodeExist, err := vs.NodeExists(vs.client, nodeName)
nodeExist, err := vs.NodeExists(nodeName)
if err != nil {
glog.Errorf("Failed to check whether node exist. err: %s.", err)
return false, err
Expand Down Expand Up @@ -991,7 +983,7 @@ func (vs *VSphere) DisksAreAttached(volPaths []string, nodeName k8stypes.NodeNam
vSphereInstance = nodeNameToVMName(nodeName)
}

nodeExist, err := vs.NodeExists(vs.client, nodeName)
nodeExist, err := vs.NodeExists(nodeName)

if err != nil {
glog.Errorf("Failed to check whether node exist. err: %s.", err)
Expand Down Expand Up @@ -1332,27 +1324,17 @@ func (vs *VSphere) DeleteVolume(vmDiskPath string) error {

// NodeExists checks if the node with given nodeName exist.
// Returns false if VM doesn't exist or VM is in powerOff state.
func (vs *VSphere) NodeExists(c *govmomi.Client, nodeName k8stypes.NodeName) (bool, error) {
func (vs *VSphere) NodeExists(nodeName k8stypes.NodeName) (bool, error) {
if nodeName == "" {
return false, nil
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

vm, err := getVirtualMachineByName(ctx, vs.cfg, c, nodeName)
if err != nil {
if _, ok := err.(*find.NotFoundError); ok {
return false, nil
}
glog.Errorf("Failed to get virtual machine object for node %+q. err %s", nodeName, err)
return false, err
}

var mvm mo.VirtualMachine
err = getVirtualMachineManagedObjectReference(ctx, c, vm, "summary", &mvm)
_, mvm, err := vs.getVMandMO(ctx, nodeName, "summary")
if err != nil {
glog.Errorf("Failed to get virtual machine object reference for node %+q. err %s", nodeName, err)
glog.Errorf("Failed to getVMandMO for NodeExists: err %v", err)
return false, err
}

Expand Down