Add vSphere integration to k8s charms

cluster/juju/layers/kubernetes-master/layer.yaml

@@ -18,6 +18,7 @@ includes:
   - 'interface:aws-integration'
   - 'interface:gcp-integration'
   - 'interface:openstack-integration'
+  - 'interface:vsphere-integration'
 options:
   basic:
     packages:

cluster/juju/layers/kubernetes-master/metadata.yaml

@@ -47,6 +47,8 @@ requires:
     interface: gcp-integration
   openstack:
     interface: openstack-integration
+  vsphere:
+    interface: vsphere-integration
 resources:
   kubectl:
     type: file

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -510,12 +510,20 @@ def set_final_status():
         return
 
     req_sent = is_state('kubernetes-master.cloud-request-sent')
+    # openstack and vsphere have admin perms; cloud req is not required
     openstack_joined = is_state('endpoint.openstack.joined')
-    cloud_req = req_sent or openstack_joined
+    vsphere_joined = is_state('endpoint.vsphere.joined')
+    cloud_req = req_sent or openstack_joined or vsphere_joined
     aws_ready = is_state('endpoint.aws.ready')
     gcp_ready = is_state('endpoint.gcp.ready')
     openstack_ready = is_state('endpoint.openstack.ready')
-    cloud_ready = aws_ready or gcp_ready or openstack_ready
+    vsphere_ready = is_state('endpoint.vsphere.ready')
+    if vsphere_ready and get_version('kube-apiserver') < (1, 12):
+        msg = 'vSphere integration requires K8s 1.12 or greater'
+        hookenv.status_set('blocked', msg)
+        return
+
+    cloud_ready = aws_ready or gcp_ready or openstack_ready or vsphere_ready
     if cloud_req and not cloud_ready:
         hookenv.status_set('waiting', 'waiting for cloud integration')
 
@@ -1338,6 +1346,11 @@ def configure_apiserver(etcd_connection_string):
         cloud_config_path = _cloud_config_path('kube-apiserver')
         api_opts['cloud-provider'] = 'openstack'
         api_opts['cloud-config'] = str(cloud_config_path)
+    elif (is_state('endpoint.vsphere.ready') and
+          get_version('kube-apiserver') >= (1, 12)):
+        cloud_config_path = _cloud_config_path('kube-apiserver')
+        api_opts['cloud-provider'] = 'vsphere'
+        api_opts['cloud-config'] = str(cloud_config_path)
 
     audit_root = '/root/cdk/audit'
     os.makedirs(audit_root, exist_ok=True)
@@ -1395,6 +1408,11 @@ def configure_controller_manager():
         cloud_config_path = _cloud_config_path('kube-controller-manager')
         controller_opts['cloud-provider'] = 'openstack'
         controller_opts['cloud-config'] = str(cloud_config_path)
+    elif (is_state('endpoint.vsphere.ready') and
+          get_version('kube-apiserver') >= (1, 12)):
+        cloud_config_path = _cloud_config_path('kube-controller-manager')
+        controller_opts['cloud-provider'] = 'vsphere'
+        controller_opts['cloud-config'] = str(cloud_config_path)
 
     configure_kubernetes_service('kube-controller-manager', controller_opts,
                                  'controller-manager-extra-args')
@@ -1665,7 +1683,8 @@ def clear_requested_integration():
 
 @when_any('endpoint.aws.ready',
           'endpoint.gcp.ready',
-          'endpoint.openstack.ready')
+          'endpoint.openstack.ready',
+          'endpoint.vsphere.ready')
 @when_not('kubernetes-master.restarted-for-cloud')
 def restart_for_cloud():
     if is_state('endpoint.gcp.ready'):
@@ -1674,6 +1693,9 @@ def restart_for_cloud():
     elif is_state('endpoint.openstack.ready'):
         _write_openstack_snap_config('kube-apiserver')
         _write_openstack_snap_config('kube-controller-manager')
+    elif is_state('endpoint.vsphere.ready'):
+        _write_vsphere_snap_config('kube-apiserver')
+        _write_vsphere_snap_config('kube-controller-manager')
     set_state('kubernetes-master.restarted-for-cloud')
     remove_state('kubernetes-master.components.started')  # force restart
 
@@ -1736,3 +1758,39 @@ def _write_openstack_snap_config(component):
         'tenant-name = {}'.format(openstack.project_name),
         'domain-name = {}'.format(openstack.user_domain_name),
     ]))
+
+
+def _write_vsphere_snap_config(component):
+    # vsphere requires additional cloud config
+    vsphere = endpoint_from_flag('endpoint.vsphere.ready')
+
+    # NB: vsphere provider will ask kube-apiserver and -controller-manager to
+    # find a uuid from sysfs unless a global config value is set. Our strict
+    # snaps cannot read sysfs, so let's do it in the charm. An invalid uuid is
+    # not fatal for storage, but it will muddy the logs; try to get it right.
+    uuid_file = '/sys/class/dmi/id/product_uuid'
+    try:
+        with open(uuid_file, 'r') as f:
+            uuid = f.read().strip()
+    except IOError as err:
+        hookenv.log("Unable to read UUID from sysfs: {}".format(err))
+        uuid = 'UNKNOWN'
+
+    cloud_config_path = _cloud_config_path(component)
+    cloud_config_path.write_text('\n'.join([
+        '[Global]',
+        'insecure-flag = true',
+        'datacenters = "{}"'.format(vsphere.datacenter),
+        'vm-uuid = "VMware-{}"'.format(uuid),
+        '[VirtualCenter "{}"]'.format(vsphere.vsphere_ip),
+        'user = {}'.format(vsphere.user),
+        'password = {}'.format(vsphere.password),
+        '[Workspace]',
+        'server = {}'.format(vsphere.vsphere_ip),
+        'datacenter = "{}"'.format(vsphere.datacenter),
+        'default-datastore = "{}"'.format(vsphere.datastore),
+        'folder = "kubernetes"',
+        'resourcepool-path = ""',
+        '[Disk]',
+        'scsicontrollertype = "pvscsi"',
+    ]))

cluster/juju/layers/kubernetes-worker/layer.yaml

@@ -16,6 +16,7 @@ includes:
   - 'interface:aws-integration'
   - 'interface:gcp-integration'
   - 'interface:openstack-integration'
+  - 'interface:vsphere-integration'
   - 'interface:mount'
 config:
   deletes:

cluster/juju/layers/kubernetes-worker/metadata.yaml

@@ -36,6 +36,8 @@ requires:
     interface: gcp-integration
   openstack:
     interface: openstack-integration
+  vsphere:
+    interface: vsphere-integration
   nfs:
     interface: mount
 provides:

cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py

@@ -714,6 +714,15 @@ def configure_kubelet(dns, ingress_ip):
         cloud_config_path = _cloud_config_path('kubelet')
         kubelet_opts['cloud-provider'] = 'openstack'
         kubelet_opts['cloud-config'] = str(cloud_config_path)
+    elif is_state('endpoint.vsphere.joined'):
+        # vsphere just needs to be joined on the worker (vs 'ready')
+        cloud_config_path = _cloud_config_path('kubelet')
+        kubelet_opts['cloud-provider'] = 'vsphere'
+        # NB: vsphere maps node product-id to its uuid (no config file needed).
+        uuid_file = '/sys/class/dmi/id/product_uuid'
+        with open(uuid_file, 'r') as f:
+            uuid = f.read().strip()
+        kubelet_opts['provider-id'] = 'vsphere://{}'.format(uuid)
 
     if get_version('kubelet') >= (1, 10):
         # Put together the KubeletConfiguration data
@@ -1193,6 +1202,8 @@ def get_node_name():
         cloud_provider = 'gce'
     elif is_state('endpoint.openstack.ready'):
         cloud_provider = 'openstack'
+    elif is_state('endpoint.vsphere.ready'):
+        cloud_provider = 'vsphere'
     if cloud_provider == 'aws':
         return getfqdn().lower()
     else: