Merge pull request #144 from francescotimperi/affinity

Implements nuvolaris/nuvolaris#350
nuvolaris · Feb 17, 2024 · aa6af25 · aa6af25
2 parents da1b875 + 7e77a5c
commit aa6af25
Show file tree

Hide file tree

Showing 21 changed files with 585 additions and 31 deletions.
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -22,7 +22,7 @@ vars:
   N: ""
   T: ""
   P: ""
-  BASETAG: 3.0.0-beta
+  BASETAG: 3.1.0-mastrogpt
   KUBE:
     sh: ./detect.sh
   NS: "nuvolaris"

diff --git a/deploy/nuvolaris-permissions/whisk-crd.yaml b/deploy/nuvolaris-permissions/whisk-crd.yaml
@@ -77,10 +77,10 @@ spec:
                     ingresslb:
                       description: allow to set the location of the ingress lb in the form namespace/service-name. Default to auto which will force the operator to use default values based on the environment.
                       type: string
-                    affinity-enabled:
+                    affinity:
                       description: enable/disable openwhisk configuration with node affinity. This flag can be true only on multinode cluster deployment. Default to false
                       type: boolean
-                    toleration-enabled:
+                    tolerations:
                       description: enable/disable openwhisk toleration. This flag can be true only on multinode cluster deployment. Default to false
                       type: boolean                                                                                                                 
                   required:

diff --git a/deploy/openwhisk-standalone/standalone-sts.yaml b/deploy/openwhisk-standalone/standalone-sts.yaml
@@ -18,6 +18,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: controller
+  namespace: nuvolaris
   labels:
     name: controller
     app: controller

diff --git a/deploy/postgres-operator-deploy/postgres.yaml b/deploy/postgres-operator-deploy/postgres.yaml
@@ -26,7 +26,7 @@ spec:
    image: postgres:14.1
    database:
       size: 200Mi
-   customConfig: nuvolaris-postgres-conf 
+   customConfig: nuvolaris-postgres-conf    
    env:
       - name: POSTGRES_PASSWORD
         valueFrom:

diff --git a/nuvolaris/couchdb.py b/nuvolaris/couchdb.py
@@ -63,13 +63,17 @@ def create(owner=None):
         "container_cpu_lim": cfg.get('configs.couchdb.resources.cpu-lim') or "1",
         "container_mem_req": cfg.get('configs.couchdb.resources.mem-req') or "1G",
         "container_mem_lim": cfg.get('configs.couchdb.resources.mem-lim') or "2G",
-        "container_manage_resources": cfg.exists('configs.couchdb.resources.cpu-req')
+        "container_manage_resources": cfg.exists('configs.couchdb.resources.cpu-req')    
     }
 
-    kus.processTemplate("couchdb","couchdb-set-tpl.yaml",data,"couchdb-set_generated.yaml")
+    tplp = ["set-attach.yaml"]
+    util.couch_affinity_tolerations_data(data)
+    if(data['affinity'] or data['tolerations']):
+       tplp.append("affinity-tolerance-sts-core-attach.yaml") 
 
+    kus.processTemplate("couchdb","couchdb-set-tpl.yaml",data,"couchdb-set_generated.yaml")
     kust =  kus.secretLiteral("couchdb-auth", user, pasw)
-    kust += kus.patchTemplate("couchdb", "set-attach.yaml", data) 
+    kust += kus.patchTemplates("couchdb",tplp,data)
     spec = kus.restricted_kustom_list("couchdb", kust, templates=["couchdb-init.yaml"],templates_filter=["couchdb-set_generated.yaml","couchdb-svc.yaml"],data=data)
 
     if owner:

diff --git a/nuvolaris/ferretdb.py b/nuvolaris/ferretdb.py
@@ -42,6 +42,7 @@ def enrich_ferretdb_data(data):
 
      # use nuvolaris postgres db as default configuration for FERRETDB
      data['ferretdb_postgres_url'] = postgres.get_base_postgres_url(data)
+     util.ferretb_affinity_tolerations_data(data)
 
 def create(owner=None):
     """
@@ -53,6 +54,9 @@ def create(owner=None):
     enrich_ferretdb_data(data)
 
     tplp = ["security-set-attach.yaml","set-attach.yaml","ferretdb-sts.yaml"]
+    if(data['affinity'] or data['tolerations']):
+       tplp.append("affinity-tolerance-sts-core-attach.yaml")
+
     mkust = kus.patchTemplates("ferretdb", tplp, data)    
     mspec = kus.kustom_list("ferretdb", mkust, templates=[], data=data)
 

diff --git a/nuvolaris/minio.py b/nuvolaris/minio.py
@@ -68,6 +68,9 @@ def create(owner=None):
 
     tplp = ["00-minio-pvc.yaml","01-minio-dep.yaml","02-minio-svc.yaml"]
 
+    if(data['affinity'] or data['tolerations']):
+       tplp.append("affinity-tolerance-dep-core-attach.yaml")    
+
     kust = kus.patchTemplates("minio", tplp, data)
     spec = kus.kustom_list("minio", kust, templates=[], data=data)
 

diff --git a/nuvolaris/minio_static.py b/nuvolaris/minio_static.py
@@ -30,19 +30,12 @@
 def create(owner=None):
     logging.info(f"*** configuring nuvolaris nginx static provider")
     runtime = cfg.get('nuvolaris.kube')
-    data = {
-        "name":"nuvolaris-static",
-        "container":"nuvolaris-static",
-        "size":1,
-        "storageClass": cfg.get('nuvolaris.storageclass'),
-        "dir":"/var/cache/nginx",
-        "minio_host": cfg.get('minio.host') or "minio",
-        "minio_port": cfg.get('minio.port') or "9000",
-        "applypodsecurity": util.get_enable_pod_security()
-    }
-
+    data = util.get_minio_static_config_data()    
     tplp = ["nginx-static-cm.yaml","nginx-static-sts.yaml","security-set-attach.yaml","set-attach.yaml"]
 
+    if(data['affinity'] or data['tolerations']):
+       tplp.append("affinity-tolerance-sts-core-attach.yaml")     
+
     kust = kus.patchTemplates("nginx-static", tplp, data)    
     spec = kus.kustom_list("nginx-static", kust, templates=[], data=data)
 

diff --git a/nuvolaris/openwhisk_standalone.py b/nuvolaris/openwhisk_standalone.py
@@ -31,9 +31,13 @@ def create(owner=None):
 
     whisk_image = data["controller_image"]
     whisk_tag = data["controller_tag"]
-
     config = kus.image(whisk_image, newTag=whisk_tag)
-    config += kus.patchTemplates("openwhisk-standalone", ["standalone-sts.yaml"], data) 
+
+    tplp = ["standalone-sts.yaml"]
+    if(data['affinity'] or data['tolerations']):
+       tplp.append("affinity-tolerance-sts-core-attach.yaml")
+
+    config += kus.patchTemplates("openwhisk-standalone", tplp, data)
     spec = kus.kustom_list("openwhisk-standalone", config, templates=[], data=data)
 
     if owner:

diff --git a/nuvolaris/postgres_operator.py b/nuvolaris/postgres_operator.py
@@ -32,10 +32,10 @@ def create(owner=None):
     """
     Deploys the postgres using kubegres operator and wait for the operator to be ready.
     """
-    logging.info("*** creating kubegres-operator")    
-    data = util.get_postgres_config_data()
-
-    spec = kus.kustom_list("postgres-operator")
+    logging.info("*** creating kubegres-operator")        
+    pg_cm_data = util.postgres_manager_affinity_tolerations_data()
+    pg_op_kust = kus.patchTemplates("postgres-operator",templates=["affinity-tolerance-dep-core-attach.yaml"], data=pg_cm_data)
+    spec = kus.kustom_list("postgres-operator",pg_op_kust, templates=[], data={})
 
     if owner:
         kopf.append_owner_reference(spec['items'], owner)
@@ -49,7 +49,7 @@ def create(owner=None):
     util.wait_for_pod_ready("{.items[?(@.metadata.labels.control-plane == 'controller-manager')].metadata.name}")
 
     logging.info("*** creating a postgres instance")
-
+    data = util.get_postgres_config_data()    
     mkust = kus.patchTemplates("postgres-operator-deploy",templates=["postgres.yaml"], data=data)
     mkust += kus.patchGenericEntry("Secret","postgres-nuvolaris-secret","/stringData/superUserPassword",data['postgres_root_password'])
     mkust += kus.patchGenericEntry("Secret","postgres-nuvolaris-secret","/stringData/replicationUserPassword",data['postgres_root_replica_password'])        

diff --git a/nuvolaris/redis.py b/nuvolaris/redis.py
@@ -61,8 +61,10 @@ def create(owner=None):
     if data['persistence']:
        tplp.append("set-attach.yaml")
 
-    kust = kus.patchTemplates("redis",tplp , data)
-
+    if(data['affinity'] or data['tolerations']):
+       tplp.append("affinity-tolerance-sts-core-attach.yaml")
+
+    kust = kus.patchTemplates("redis",tplp , data)    
     spec = kus.kustom_list("redis", kust, templates=["redis-conf.yaml"], data=data)
 
     if owner:

diff --git a/nuvolaris/templates/affinity-tolerance-dep-core-attach.yaml b/nuvolaris/templates/affinity-tolerance-dep-core-attach.yaml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{name}}
+  namespace: nuvolaris
+spec:
+  template:
+    spec:
+      affinity:
+      {% if affinity %} 
+        # prefer to not run on an invoker node (only prefer because of single node clusters)
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: nuvolaris-role
+                operator: NotIn
+                values:
+                - {{affinity_invoker_node_label}}
+          - weight: 80
+            preference:
+              matchExpressions:
+              - key: nuvolaris-role
+                operator: In
+                values:
+                - {{affinity_core_node_label}}
+        # Fault tolerance: prevent multiple instances of {{pod_anti_affinity_name}} from running on the same node
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: name
+                operator: In
+                values:
+                - {{pod_anti_affinity_name}}
+            topologyKey: "kubernetes.io/hostname"
+      {% endif %}               
+      tolerations:
+      {% if tolerations %}   
+        - key: "nuvolaris-role"
+          operator: "Equal"
+          value: {{toleration_role}}
+          effect: "NoSchedule"
+      {% endif %}            
diff --git a/nuvolaris/templates/affinity-tolerance-sts-core-attach.yaml b/nuvolaris/templates/affinity-tolerance-sts-core-attach.yaml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{name}}
+  namespace: nuvolaris
+spec:
+  template:
+    spec:
+      affinity:
+      {% if affinity %} 
+        # prefer to not run on an invoker node (only prefer because of single node clusters)
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: nuvolaris-role
+                operator: NotIn
+                values:
+                - {{affinity_invoker_node_label}}
+          - weight: 80
+            preference:
+              matchExpressions:
+              - key: nuvolaris-role
+                operator: In
+                values:
+                - {{affinity_core_node_label}}
+        # Fault tolerance: prevent multiple instances of {{pod_anti_affinity_name}} from running on the same node
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: name
+                operator: In
+                values:
+                - {{pod_anti_affinity_name}}
+            topologyKey: "kubernetes.io/hostname"
+      {% endif %}               
+      tolerations:
+      {% if tolerations %}   
+        - key: "nuvolaris-role"
+          operator: "Equal"
+          value: {{toleration_role}}
+          effect: "NoSchedule"
+      {% endif %}            
diff --git a/nuvolaris/templates/affinity-tolerance-sts-invoker-attach.yaml b/nuvolaris/templates/affinity-tolerance-sts-invoker-attach.yaml
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{name}}
+  namespace: nuvolaris
+spec:
+  template:
+    spec:
+      affinity:
+      {% if affinity %}
+        # run only on nodes labeled with nuvolaris-role={{affinity_invoker_node_label}}
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: nuvolaris-role
+                operator: In
+                values:
+                - {{affinity_invoker_node_label}}
+        # Fault tolerance: prevent multiple instances of {{pod-anti-affinity-name}} from running on the same node
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: name
+                operator: In
+                values:
+                - {{pod_anti_affinity_name}}
+            topologyKey: "kubernetes.io/hostname"
+      {% endif %}      
+      tolerations:
+      {% if tolerations %}
+        - key: "nuvolaris-role"
+          operator: "Equal"
+          value: {{toleration_role}}
+          effect: "NoSchedule"
+      {% endif %}    
diff --git a/nuvolaris/templates/postgres.yaml b/nuvolaris/templates/postgres.yaml
@@ -27,7 +27,35 @@ spec:
    database:
       size: {{size}}Gi
       storageClassName: {{storageClass}}
-   customConfig: nuvolaris-postgres-conf 
+   customConfig: nuvolaris-postgres-conf
+   scheduler:
+      affinity:
+      {% if affinity %}
+        # prefer to not run on an invoker node (only prefer because of single node clusters)
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: nuvolaris-role
+                operator: NotIn
+                values:
+                - {{affinity_invoker_node_label}}
+          - weight: 80
+            preference:
+              matchExpressions:
+              - key: nuvolaris-role
+                operator: In
+                values:
+                - {{affinity_core_node_label}}
+      {% endif %}               
+      tolerations:
+      {% if tolerations %}   
+        - key: "nuvolaris-role"
+          operator: "Equal"
+          value: {{toleration_role}}
+          effect: "NoSchedule"
+      {% endif %}   
    env:
       - name: POSTGRES_PASSWORD
         valueFrom:

diff --git a/nuvolaris/templates/standalone-sts.yaml b/nuvolaris/templates/standalone-sts.yaml
@@ -21,6 +21,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: controller
+  namespace: nuvolaris
   labels:
     name: controller
     app: controller