feat: configure standardResources via controller configmap (#1490)

Signed-off-by: Nishchith Shetty <inishchith@gmail.com>
numaproj · Feb 11, 2024 · c71b2ab · c71b2ab
1 parent ce0a315
commit c71b2ab
Show file tree

Hide file tree

Showing 25 changed files with 1,020 additions and 499 deletions.
diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json
@@ -17838,6 +17838,9 @@
     },
     "io.numaproj.numaflow.v1alpha1.GetDaemonDeploymentReq": {
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
@@ -17858,7 +17861,8 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "type": "object"
     },
@@ -17912,6 +17916,9 @@
         "ConfigReloaderImage": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Labels": {
           "additionalProperties": {
             "type": "string"
@@ -17962,7 +17969,8 @@
         "ServerEncryptionSecretName",
         "ConfigMapName",
         "PvcNameIfNeeded",
-        "StartCommand"
+        "StartCommand",
+        "DefaultResources"
       ],
       "type": "object"
     },
@@ -17998,6 +18006,9 @@
         "CredentialSecretName": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "HealthConfigMapName": {
           "type": "string"
         },
@@ -18059,12 +18070,16 @@
         "PvcNameIfNeeded",
         "ConfConfigMapName",
         "ScriptsConfigMapName",
-        "HealthConfigMapName"
+        "HealthConfigMapName",
+        "DefaultResources"
       ],
       "type": "object"
     },
     "io.numaproj.numaflow.v1alpha1.GetSideInputDeploymentReq": {
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
@@ -18085,12 +18100,16 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "type": "object"
     },
     "io.numaproj.numaflow.v1alpha1.GetVertexPodSpecReq": {
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
@@ -18115,7 +18134,8 @@
         "Image",
         "PullPolicy",
         "Env",
-        "SideInputsStoreName"
+        "SideInputsStoreName",
+        "DefaultResources"
       ],
       "type": "object"
     },

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -17846,9 +17846,13 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "type": "array",
           "items": {
@@ -17916,7 +17920,8 @@
         "ServerEncryptionSecretName",
         "ConfigMapName",
         "PvcNameIfNeeded",
-        "StartCommand"
+        "StartCommand",
+        "DefaultResources"
       ],
       "properties": {
         "ClientPort": {
@@ -17933,6 +17938,9 @@
         "ConfigReloaderImage": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Labels": {
           "type": "object",
           "additionalProperties": {
@@ -18011,7 +18019,8 @@
         "PvcNameIfNeeded",
         "ConfConfigMapName",
         "ScriptsConfigMapName",
-        "HealthConfigMapName"
+        "HealthConfigMapName",
+        "DefaultResources"
       ],
       "properties": {
         "ConfConfigMapName": {
@@ -18020,6 +18029,9 @@
         "CredentialSecretName": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "HealthConfigMapName": {
           "type": "string"
         },
@@ -18073,9 +18085,13 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "type": "array",
           "items": {
@@ -18100,9 +18116,13 @@
         "Image",
         "PullPolicy",
         "Env",
-        "SideInputsStoreName"
+        "SideInputsStoreName",
+        "DefaultResources"
       ],
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "type": "array",
           "items": {

diff --git a/config/advanced-install/namespaced-controller-wo-crds.yaml b/config/advanced-install/namespaced-controller-wo-crds.yaml
@@ -133,6 +133,11 @@ metadata:
 apiVersion: v1
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/config/base/controller-manager/numaflow-controller-config.yaml b/config/base/controller-manager/numaflow-controller-config.yaml
@@ -4,6 +4,11 @@ metadata:
   name: numaflow-controller-config
 data:
   controller-config.yaml: |+
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/config/install.yaml b/config/install.yaml
@@ -16301,6 +16301,11 @@ metadata:
 apiVersion: v1
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml
@@ -16204,6 +16204,11 @@ metadata:
 apiVersion: v1
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/docs/APIs.md b/docs/APIs.md
@@ -1384,6 +1384,15 @@ Kubernetes core/v1.PullPolicy </a> </em>
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetJetStreamServiceSpecReq">
@@ -1555,6 +1564,15 @@ Description
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetRedisServiceSpecReq">
@@ -1719,6 +1737,15 @@ Description
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetSideInputDeploymentReq">
@@ -1772,6 +1799,15 @@ Kubernetes core/v1.PullPolicy </a> </em>
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetVertexPodSpecReq">
@@ -1832,6 +1868,15 @@ Kubernetes core/v1.PullPolicy </a> </em>
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GroupBy">

diff --git a/docs/operations/controller-configmap.md b/docs/operations/controller-configmap.md
@@ -15,11 +15,37 @@ metadata:
   name: numaflow-controller-config
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        ...
     isbsvc:
       jetstream:
         ...
 ```
 
+### Default Controller Configuration
+
+Currently, we support configuring the init and main container resources for steps across all the pipelines. The configuration is under `defaults` key in the ConfigMap.
+
+For example, to set the default container resources for steps across all the pipelines:
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: numaflow-controller-config
+data:
+  controller-config.yaml: |
+    defaults:
+      containerResources: |
+        limits:
+          memory: "256Mi"
+          cpu: "200m"
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+```
+
+
 ### ISB Service Configuration
 
 One of the important configuration items in the ConfigMap is about [ISB Service](../core-concepts/inter-step-buffer-service.md). We currently use 3rd party technologies such as `JetStream` to implement ISB Services, if those applications have new releases, to make them available in Numaflow, the new versions need to be added in the ConfigMap.

diff --git a/docs/operations/numaflow-controller-config.yaml b/docs/operations/numaflow-controller-config.yaml
@@ -4,6 +4,11 @@ metadata:
   name: numaflow-controller-config
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/pkg/apis/numaflow/v1alpha1/const.go b/pkg/apis/numaflow/v1alpha1/const.go
@@ -19,9 +19,6 @@ package v1alpha1
 import (
 	"fmt"
 	"time"
-
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 const (
@@ -171,12 +168,4 @@ const (
 var (
 	MessageTagDrop = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__
 	MessageTagAll  = fmt.Sprintf("%U__ALL__", '\\')  // U+005C__ALL__
-
-	// the standard resources used by the `init` and `main`containers.
-	standardResources = corev1.ResourceRequirements{
-		Requests: corev1.ResourceList{
-			"cpu":    resource.MustParse("100m"),
-			"memory": resource.MustParse("128Mi"),
-		},
-	}
 )