Add ability to deploy Ceph via Rook on k3s

This adds the `--deploy-ses` option to `sesdev create k3s`,
which will pull registry.suse.com/ses/7.1/charts/rook-ceph,
then deploy Rook, which in turn will deploy Ceph, and have
it eat all the disks on all the workers.

Signed-off-by: Tim Serong <tserong@suse.com>

sesdev/__init__.py

@@ -775,9 +775,13 @@ def _create_command(deployment_id, settings_dict):
                 click.echo("  $ sesdev tunnel {} suma".format(deployment_id))
                 click.echo()
             elif dep.settings.version == 'k3s':
-                # Nothing extra to print
-                # TODO: really?
-                pass
+                if dep.settings.caasp_deploy_ses:
+                    click.echo("Rook will be off doing its magic dance now.")
+                    click.echo("After logging into the cluster, try these:")
+                    click.echo("")
+                    click.echo("  # kubectl -n rook-ceph logs -l app=rook-ceph-operator")
+                    click.echo("  # kubectl -n rook-ceph get pods")
+                    click.echo()
             else:
                 click.echo("Or, access the Ceph Dashboard with:")
                 click.echo()
@@ -927,6 +931,8 @@ def caasp4(deployment_id, **kwargs):
 @click.argument('deployment_id', required=False)
 @common_create_options
 @libvirt_options
+@click.option("--deploy-ses", is_flag=True, default=False,
+              help="Deploy SES using rook in k3s")
 @click.option("--k3s-version", default=None,
               help='k3s version to install (defaults to latest stable)')
 def k3s(deployment_id, **kwargs):

seslib/deployment.py

@@ -265,7 +265,7 @@ def __maybe_tweak_roles(self):
     def __maybe_adjust_num_disks(self):
         single_node = self.settings.single_node or len(self.settings.roles) == 1
         storage_nodes = self.node_counts["storage"]
-        if self.settings.version in ['caasp4'] and self.settings.caasp_deploy_ses:
+        if self.settings.version in ['caasp4', 'k3s'] and self.settings.caasp_deploy_ses:
             if single_node:
                 storage_nodes = 1
             else:

seslib/settings.py

@@ -15,9 +15,12 @@
         'help': 'Enable/disable AppArmor',
         'default': True,
     },
+    # This should rightly now be named something more generic, e.g:
+    # "kube_deploy_ses", but that will confuse sesdev when it looks
+    # at settings of existing deployments, so it can stay as is.
     'caasp_deploy_ses': {
         'type': bool,
-        'help': 'Deploy SES using rook in CaasP',
+        'help': 'Deploy SES using rook in CaasP or k3s',
         'default': False,
     },
     'ceph_salt_git_repo': {

seslib/templates/k3s/provision.sh.j2

@@ -79,6 +79,37 @@ helm version
 # by default, which of course won't work.
 echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> /etc/profile.local
 
+{% if caasp_deploy_ses %}
+
+mkdir -p ~/cluster
+cd ~/cluster
+helm pull oci://registry.suse.com/ses/7.1/charts/rook-ceph
+# Currently rook-ceph-1.10.1.tgz
+# TODO: can we ask helm for the name of the file?  Or just assume it's
+# the only file currently in the directory?
+tar -xzf rook-ceph-*.tgz
+
+kubectl create namespace rook-ceph
+
+# The earlier addition of KUBECONFIG to /etc/profile.local won't help
+# us in this current session, so need to set it explicitly here.
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+helm install -n rook-ceph rook-ceph ./rook-ceph/
+
+echo "Waiting for the rook operator"
+kubectl wait --timeout=5m --namespace rook-ceph --for=condition=Ready pod  -l "app=rook-ceph-operator"
+
+echo "Let rook take all nodes that aren't the master"
+kubectl label node -l 'node-role.kubernetes.io/master!=true' node-role.rook-ceph/cluster=any
+
+echo "Creating ceph cluster..."
+kubectl create -f rook-ceph/examples/cluster.yaml
+
+# The above will take some time (maybe 20 minutes in my testing),
+# so let's not wait for it :-)
+
+{% endif %} {# caasp_deploy_ses #}
+
 {% else %} {# node == master #}
 
 function get_k3s_token {
@@ -112,4 +143,4 @@ rm /tmp/k3s_token
 
 curl -sfL https://get.k3s.io | K3S_URL=https://{{ master.fqdn }}:6443 sh -
 
-{% endif %}
+{% endif %}