Merge pull request #5653 from freenas/default-storage-class

NAS-107517 / 20.10 / Setup default storage class for k8s

src/middlewared/middlewared/etc_files/docker.py

@@ -24,7 +24,6 @@ def nvidia_configuration(middleware):
 
 def gpu_configuration(middleware):
     available_gpu = middleware.call_sync('device.get_info', 'GPU')
-
     if available_gpu['vendor'] == 'NVIDIA':
         return nvidia_configuration(middleware)
 

src/middlewared/middlewared/plugins/device_/gpu_linux.py

@@ -0,0 +1,33 @@
+import re
+
+from middlewared.service import Service
+from middlewared.utils import run
+
+
+RE_VENDOR = re.compile(r'description:\s*VGA compatible controller[\s\S]*vendor:\s*(.*)')
+
+
+class DeviceService(Service):
+
+    GPU = None
+
+    async def available_gpu(self):
+        if self.GPU:
+            return self.GPU
+
+        not_available = {'available': False, 'vendor': None}
+        cp = await run(['lshw', '-numeric', '-C', 'display'], check=False)
+        if cp.returncode:
+            self.logger.error('Unable to retrieve GPU details: %s', cp.stderr.decode())
+            return not_available
+
+        vendor = RE_VENDOR.findall(cp.stdout.decode())
+        if not vendor:
+            self.GPU = not_available
+        else:
+            # We only support nvidia based GPU's right now based on equipment available
+            if 'nvidia' in vendor[0].lower():
+                self.GPU = {'available': True, 'vendor': 'NVIDIA'}
+            else:
+                self.GPU = not_available
+        return self.GPU

src/middlewared/middlewared/plugins/kubernetes_linux/k8s/api_client.py

@@ -12,7 +12,11 @@ async def api_client(context=None, api_client_kwargs=None):
     context = context or {}
     context['core_api'] = True
     api_cl = ApiClient(**(api_client_kwargs or {}))
-    user_context = {'core_api': client.CoreV1Api(api_cl), 'apps_api': client.AppsV1Api(api_cl)}
+    user_context = {
+        'core_api': client.CoreV1Api(api_cl),
+        'apps_api': client.AppsV1Api(api_cl),
+        'storage_api': client.StorageV1Api(api_cl),
+    }
     for k in filter(lambda k: context[k], context):
         if k == 'node':
             user_context[k] = await get_node(user_context['core_api'])

src/middlewared/middlewared/plugins/kubernetes_linux/lifecycle.py

@@ -35,6 +35,7 @@ async def post_start_internal(self):
         node_config = await self.middleware.call('k8s.node.config')
         await self.middleware.call('k8s.cni.setup_cni')
         await self.middleware.call('k8s.gpu.setup')
+        await self.middleware.call('k8s.storage_class.setup_default_storage_class')
         await self.middleware.call(
             'k8s.node.remove_taints', [
                 k['key'] for k in (node_config['spec']['taints'] or []) if k['key'] in ('ix-svc-start', 'ix-svc-stop')
@@ -127,7 +128,7 @@ async def create_update_k8s_datasets(self, k8s_ds):
 
     @private
     async def kubernetes_datasets(self, k8s_ds):
-        return [k8s_ds] + [os.path.join(k8s_ds, d) for d in ('docker', 'k3s', 'releases')]
+        return [k8s_ds] + [os.path.join(k8s_ds, d) for d in ('docker', 'k3s', 'releases', 'default_volumes')]
 
 
 async def _event_system(middleware, event_type, args):

src/middlewared/middlewared/plugins/kubernetes_linux/storage_classes.py

@@ -0,0 +1,76 @@
+import os
+
+from kubernetes_asyncio import client
+
+from middlewared.service import CallError, CRUDService, filterable
+from middlewared.utils import filter_list
+
+from .k8s import api_client
+
+
+DEFAULT_STORAGE_CLASS = 'openebs-zfspv-default'
+
+
+class KubernetesStorageClassService(CRUDService):
+
+    class Config:
+        namespace = 'k8s.storage_class'
+        private = True
+
+    @filterable
+    async def query(self, filters=None, options=None):
+        async with api_client() as (api, context):
+            return filter_list(
+                [d.to_dict() for d in (await context['storage_api'].list_storage_class()).items],
+                filters, options
+            )
+
+    async def do_create(self, data):
+        async with api_client() as (api, context):
+            try:
+                await context['storage_api'].create_storage_class(data)
+            except client.exceptions.ApiException as e:
+                raise CallError(f'Failed to create storage class: {e}')
+
+    async def do_update(self, name, data):
+        async with api_client() as (api, context):
+            try:
+                await context['storage_api'].patch_storage_class(name, data)
+            except client.exceptions.ApiException as e:
+                raise CallError(f'Failed to create storage class: {e}')
+
+    async def do_delete(self, name):
+        async with api_client() as (api, context):
+            try:
+                await context['storage_api'].delete_storage_class(name)
+            except client.exceptions.ApiException as e:
+                raise CallError(f'Failed to delete storage class: {e}')
+
+    async def setup_default_storage_class(self):
+        try:
+            await self.setup_default_storage_class_internal()
+        except Exception as e:
+            # Let's not make this fatal as workloads managed by us will still be functional
+            self.logger.error('Failed to setup default storage class: %s', e)
+
+    async def setup_default_storage_class_internal(self):
+        storage_ds = os.path.join((await self.middleware.call('kubernetes.config'))['dataset'], 'default_volumes')
+        config = {
+            'apiVersion': 'storage.k8s.io/v1',
+            'kind': 'StorageClass',
+            'metadata': {
+                'name': DEFAULT_STORAGE_CLASS,
+                'annotations': {'storageclass.kubernetes.io/is-default-class': 'true'}
+            },
+            'parameters': {'fstype': 'zfs', 'poolname': storage_ds},
+            'provisioner': 'zfs.csi.openebs.io',
+            'allowVolumeExpansion': True,
+        }
+
+        if await self.query([
+            ['metadata.annotations.storageclass\\.kubernetes\\.io/is-default-class', '=', 'true'],
+            ['metadata.name', '=', DEFAULT_STORAGE_CLASS],
+        ]):
+            await self.middleware.call('k8s.storage_class.update', DEFAULT_STORAGE_CLASS, config)
+        else:
+            await self.middleware.call('k8s.storage_class.create', config)