Merge d9aeb69 into 34e1b41

reanahub · Sep 14, 2018 · cf28cd5 · cf28cd5
2 parents 34e1b41 + d9aeb69
commit cf28cd5
Show file tree

Hide file tree

Showing 11 changed files with 131 additions and 58 deletions.
diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst
@@ -138,8 +138,38 @@ Deploy on CERN infrastructure
             -subj "/CN=reana.cern.ch"
       $ kubectl create secret tls reana-ssl-secrets
             --key /tmp/tls.key --cert /tmp/tls.crt
-
-6. Since Python3 does not come by default we have to use the `slc` command to
+6. Create the shared volume:
+
+   .. code-block:: console
+
+      $ manila create --share-type "Geneva CephFS Testing"
+            --name reana cephfs 10
+      $ # wait until gets created
+      $ manila access-allow reana cephx reana-user
+      $ manila share-export-location-list reana-dev
+      +--------------------------------------+------------------------------------------------------------------------------------------------------------------+-----------+
+      | ID                                   | Path                                                                                                             | Preferred |
+      +--------------------------------------+------------------------------------------------------------------------------------------------------------------+-----------+
+      | 455rc38d-c1d2-4837-abba-76c25505bc02 | 142.143.144.45:5565,142.143.144.46:5565,142.143.144.47:5565/<shared_volume/path>                                 | False     |
+      +--------------------------------------+------------------------------------------------------------------------------------------------------------------+-----------+
+      $ manila access-list reana-dev
+      +--------------------------------------+-------------+------------+--------------+--------+------------------------------------------+----------------------------+----------------------------+
+      | id                                   | access_type | access_to  | access_level | state  | access_key                               | created_at                 | updated_at                 |
+      +--------------------------------------+-------------+------------+--------------+--------+------------------------------------------+----------------------------+----------------------------+
+      | bf2b1e34-abba-4096-9e4e-1aa4aacdc6d0 | cephx       | user       | rw           | active | ABBAffyBbad7fdsaAfepl4MFKabbse2/UFOR1A== | 2018-06-12T22:22:15.000000 | 2018-06-12T22:22:17.000000 |
+      +--------------------------------------+-------------+------------+--------------+--------+------------------------------------------+----------------------------+----------------------------+
+      # create the secret which allows access to the manila share
+      # using the provided script from kubernetes/cloud-provider-openstack repository
+      $ git clone https://github.com/kubernetes/cloud-provider-openstack
+      $ cd cloud-provider-openstack/examples/manila-provisioner
+      $ ./generate-secrets.sh -n my-manila-secrets | ./filter-secrets.sh > ceph-secret.yaml
+      $ kubectl create -f ceph-secret.yaml
+      secret "ceph-secret" created
+
+   Set the `root_path` variable in storageclasses/ceph.yaml
+   to the created `<share_volume/path>`.
+
+7. Since Python3 does not come by default we have to use the `slc` command to
    activate it and we create a virtual environment for REANA:
 
    .. code-block:: console
@@ -148,23 +178,33 @@ Deploy on CERN infrastructure
       $ virtualenv reana
       $ source reana/bin/activate
 
-7. Install `reana-cluster`:
+8. Install `reana-cluster` (since `reana-commons` is not yet released we have to
+   install it manually):
 
    .. code-block:: console
 
       (reana) $ pip install reana-cluster
 
-8.  Set the database URI and instantiate REANA cluster:
+9. Set the database URI and instantiate REANA cluster:
+
+   .. code-block:: console
+
+      (reana) $ export REANA_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://reana:reana@<db-server>:5432/reana
+      (reana) $ curl https://raw.githubusercontent.com/reanahub/reana-cluster/master/reana_cluster/configurations/reana-cluster.yaml
+            --output reana-cluster.yaml
+
+   Edit ``reana-cluster.yaml`` adding the ``cephfs_monitors`` obtained
+   in the step 5 and instatiate the cluster.
 
    .. code-block:: console
 
-      (reana) $ reana-cluster init
+      (reana) $ reana-cluster -f reana-cluster.yaml --prod init
 
 9. Make REANA accessible from outside:
 
    .. code-block:: console
 
-      (reana) $ curl http://test-reana.cern.ch/api/ping
+      (reana) $ curl http://reana.cern.ch/api/ping
       {"message": "OK", "status": "200"}
 
 

diff --git a/reana_cluster/backends/kubernetes/k8s.py b/reana_cluster/backends/kubernetes/k8s.py
@@ -104,6 +104,7 @@ def __init__(self,
         self._versionapi = k8s_client.VersionApi()
         self._extbetav1api = k8s_client.ExtensionsV1beta1Api()
         self._rbacauthorizationv1api = k8s_client.RbacAuthorizationV1Api()
+        self._storagev1api = k8s_client.StorageV1Api()
 
         self.k8s_api_client_config = k8s_api_client_config
 
@@ -380,6 +381,15 @@ def init(self):
                             namespace=manifest['metadata'].get('namespace',
                                                                'default'))
 
+                elif manifest['kind'] == 'StorageClass':
+                    self._storagev1api.create_storage_class(body=manifest)
+
+                elif manifest['kind'] == 'PersistentVolumeClaim':
+                    self._corev1api.create_namespaced_persistent_volume_claim(
+                        body=manifest,
+                        namespace=manifest['metadata'].get('namespace',
+                                                           'default'))
+
             except ApiException as e:  # Handle K8S API errors
 
                 if e.status == 409:
@@ -534,6 +544,19 @@ def down(self):
                             namespace=manifest['metadata'].get('namespace',
                                                                'default'))
 
+                elif manifest['kind'] == 'StorageClass':
+                    self._storagev1api.delete_storage_class(
+                            name=manifest['metadata']['name'],
+                            body=k8s_client.V1DeleteOptions())
+
+                elif manifest['kind'] == 'PersistentVolumeClaim':
+                    self._corev1api.\
+                        delete_namespaced_persistent_volume_claim(
+                            name=manifest['metadata']['name'],
+                            body=k8s_client.V1DeleteOptions(),
+                            namespace=manifest['metadata'].get('namespace',
+                                                               'default'))
+
             except ApiException as e:  # Handle K8S API errors
 
                 if e.status == 409:  # Conflict, object probably already exists

diff --git a/reana_cluster/backends/kubernetes/templates/backend_conf.yaml b/reana_cluster/backends/kubernetes/templates/backend_conf.yaml
@@ -20,3 +20,9 @@
 {% include 'clusterrole/service-reader.yaml' %}
 {% include 'clusterrolebinding/service-reader.yaml' %}
 {% include 'ingress/reana-ingress.yaml' %}
+{%- if DEPLOYMENT == 'prod' %}
+{% include 'storageclasses/ceph.yaml' %}
+{% include 'persistentvolumeclaims/ceph.yaml' %}
+{%- else %}
+{% include 'deployments/wdb-template.yaml' %}
+{%- endif %}
diff --git a/reana_cluster/backends/kubernetes/templates/deployments/cwl-default-worker.yaml b/reana_cluster/backends/kubernetes/templates/deployments/cwl-default-worker.yaml
@@ -43,16 +43,9 @@ spec:
       volumes:
       - name: default-shared-volume
         {%- if DEPLOYMENT == 'prod' %}
-        cephfs:
-          monitors:
-          {% for monitor in CEPHFS_MONITORS  %}
-           - {{monitor}}
-          {% endfor %}
-          path: "{{ROOT_PATH}}"
-          user: {{CEPHFS_USER}}
-          secretRef:
-            name: {{CEPHFS_SECRET_NAME}}
-          readOnly: {{CEPHFS_READONLY}}
+        persistentVolumeClaim:
+          claimName: manila-cephfs-pvc
+          readOnly: false
         {% else %}
         hostPath:
             path: "{{ROOT_PATH}}"

diff --git a/reana_cluster/backends/kubernetes/templates/deployments/serial-default-worker.yaml b/reana_cluster/backends/kubernetes/templates/deployments/serial-default-worker.yaml
@@ -43,16 +43,9 @@ spec:
       volumes:
       - name: default-shared-volume
         {%- if DEPLOYMENT == 'prod' %}
-        cephfs:
-          monitors:
-          {% for monitor in CEPHFS_MONITORS  %}
-           - {{monitor}}
-          {% endfor %}
-          path: "{{ROOT_PATH}}"
-          user: {{CEPHFS_USER}}
-          secretRef:
-            name: {{CEPHFS_SECRET_NAME}}
-          readOnly: {{CEPHFS_READONLY}}
+        persistentVolumeClaim:
+          claimName: manila-cephfs-pvc
+          readOnly: false
         {% else %}
         hostPath:
             path: "{{ROOT_PATH}}"

diff --git a/reana_cluster/backends/kubernetes/templates/deployments/server-template.yaml b/reana_cluster/backends/kubernetes/templates/deployments/server-template.yaml
@@ -44,16 +44,9 @@ spec:
       volumes:
       - name: reana-shared-volume
         {% if DEPLOYMENT == 'prod' %}
-        cephfs:
-          monitors:
-          {% for monitor in CEPHFS_MONITORS  %}
-           - {{monitor}}
-          {% endfor %}
-          path: {{ROOT_PATH}}
-          user: {{CEPHFS_USER}}
-          secretRef:
-            name: {{CEPHFS_SECRET_NAME}}
-          readOnly: {{CEPHFS_READONLY}}
+        persistentVolumeClaim:
+          claimName: manila-cephfs-pvc
+          readOnly: false
         {% else %}
         hostPath:
           path: {{ROOT_PATH}}

diff --git a/reana_cluster/backends/kubernetes/templates/deployments/workflow-controller-template.yaml b/reana_cluster/backends/kubernetes/templates/deployments/workflow-controller-template.yaml
@@ -64,16 +64,9 @@ spec:
       volumes:
       - name: reana-shared-volume
         {% if DEPLOYMENT == 'prod' %}
-        cephfs:
-          monitors:
-          {% for monitor in CEPHFS_MONITORS  %}
-           - {{monitor}}
-          {% endfor %}
-          path: {{ROOT_PATH}}
-          user: {{CEPHFS_USER}}
-          secretRef:
-            name: {{CEPHFS_SECRET_NAME}}
-          readOnly: {{CEPHFS_READONLY}}
+        persistentVolumeClaim:
+          claimName: manila-cephfs-pvc
+          readOnly: false
         {% else %}
         hostPath:
           path: {{ROOT_PATH}}

diff --git a/reana_cluster/backends/kubernetes/templates/deployments/yadage-default-worker.yaml b/reana_cluster/backends/kubernetes/templates/deployments/yadage-default-worker.yaml
@@ -43,16 +43,9 @@ spec:
       volumes:
       - name: default-shared-volume
         {%- if DEPLOYMENT == 'prod' %}
-        cephfs:
-          monitors:
-          {% for monitor in CEPHFS_MONITORS  %}
-           - {{monitor}}
-          {% endfor %}
-          path: "{{ROOT_PATH}}"
-          user: {{CEPHFS_USER}}
-          secretRef:
-            name: {{CEPHFS_SECRET_NAME}}
-          readOnly: {{CEPHFS_READONLY}}
+        persistentVolumeClaim:
+          claimName: manila-cephfs-pvc
+          readOnly: false
         {% else %}
         hostPath:
             path: "{{ROOT_PATH}}"

diff --git a/reana_cluster/backends/kubernetes/templates/persistentvolumeclaims/ceph.yaml b/reana_cluster/backends/kubernetes/templates/persistentvolumeclaims/ceph.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: manila-cephfs-pvc
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: manila-csicephfs-share
diff --git a/reana_cluster/backends/kubernetes/templates/storageclasses/ceph.yaml b/reana_cluster/backends/kubernetes/templates/storageclasses/ceph.yaml
@@ -0,0 +1,20 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: manila-csicephfs-share
+provisioner: manila-provisioner
+parameters:
+  type: "Geneva CephFS Testing"
+  zones: nova
+  osSecretName: ceph-secret
+  osSecretNamespace: default
+  protocol: CEPHFS
+  backend: csi-cephfs
+  csi-driver: csi-cephfsplugin
+  os-shareID: #
+  os-shareAccessID: #
+  # monitors: {{ CEPHFS_MONITORS|join(',') }}
+  provisionVolume: "false"
+  rootPath: # /volumes/_nogroup/<uuid>
+reclaimPolicy: Delete
diff --git a/reana_cluster/configurations/reana-cluster.yaml b/reana_cluster/configurations/reana-cluster.yaml
@@ -4,7 +4,10 @@ cluster:
   db_config: &db_base_config
     - REANA_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://reana:reana@db:5432/reana"
   root_path: "/reana"
-  db_persistence_path: "/reanadb"
+  cephfs_monitors:
+    - 188.184.86.25:6790
+    - 188.184.94.56:6790
+    - 188.185.66.208:6790
 
 components:
   reana-workflow-controller:
@@ -21,12 +24,13 @@ components:
     environment:
       - <<: *db_base_config
       - SHARED_VOLUME_PATH: "/reana"
-      - REANA_STORAGE_BACKEND: "LOCAL"
+      - REANA_STORAGE_BACKEND: "CEPHFS"
 
   reana-server:
     type: "docker"
     image: "reanahub/reana-server:0.3.1"
     environment:
+      - SHARED_VOLUME_PATH: "/reana"
       - <<: *db_base_config
 
   reana-message-broker:
@@ -45,16 +49,19 @@ components:
     environment:
       - <<: *db_base_config
       - ZMQ_PROXY_CONNECT: "tcp://zeromq-msg-proxy.default.svc.cluster.local:8666"
+      - SHARED_VOLUME_PATH: "/reana"
 
   reana-workflow-engine-cwl:
     type: "docker"
     image: "reanahub/reana-workflow-engine-cwl:0.3.1"
     environment:
       - <<: *db_base_config
       - ZMQ_PROXY_CONNECT: "tcp://zeromq-msg-proxy.default.svc.cluster.local:8666"
+      - SHARED_VOLUME_PATH: "/reana"
 
   reana-workflow-engine-serial:
     type: "docker"
     image: "reanahub/reana-workflow-engine-serial:0.3.1"
     environment:
       - ZMQ_PROXY_CONNECT: "tcp://zeromq-msg-proxy.default.svc.cluster.local:8666"
+      - SHARED_VOLUME_PATH: "/reana"