feat: configure standardResources via controller configmap (#1490)

Signed-off-by: Nishchith Shetty <inishchith@gmail.com>
numaproj · Feb 5, 2024 · 20cf66d · 20cf66d
1 parent 345e7ca
commit 20cf66d
Show file tree

Hide file tree

Showing 25 changed files with 1,032 additions and 511 deletions.
diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json
@@ -17847,6 +17847,9 @@
     },
     "io.numaproj.numaflow.v1alpha1.GetDaemonDeploymentReq": {
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
@@ -17867,7 +17870,8 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "type": "object"
     },
@@ -17921,6 +17925,9 @@
         "ConfigReloaderImage": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Labels": {
           "additionalProperties": {
             "type": "string"
@@ -17971,7 +17978,8 @@
         "ServerEncryptionSecretName",
         "ConfigMapName",
         "PvcNameIfNeeded",
-        "StartCommand"
+        "StartCommand",
+        "DefaultResources"
       ],
       "type": "object"
     },
@@ -18007,6 +18015,9 @@
         "CredentialSecretName": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "HealthConfigMapName": {
           "type": "string"
         },
@@ -18068,12 +18079,16 @@
         "PvcNameIfNeeded",
         "ConfConfigMapName",
         "ScriptsConfigMapName",
-        "HealthConfigMapName"
+        "HealthConfigMapName",
+        "DefaultResources"
       ],
       "type": "object"
     },
     "io.numaproj.numaflow.v1alpha1.GetSideInputDeploymentReq": {
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
@@ -18094,12 +18109,16 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "type": "object"
     },
     "io.numaproj.numaflow.v1alpha1.GetVertexPodSpecReq": {
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
@@ -18124,7 +18143,8 @@
         "Image",
         "PullPolicy",
         "Env",
-        "SideInputsStoreName"
+        "SideInputsStoreName",
+        "DefaultResources"
       ],
       "type": "object"
     },

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -17855,9 +17855,13 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "type": "array",
           "items": {
@@ -17925,7 +17929,8 @@
         "ServerEncryptionSecretName",
         "ConfigMapName",
         "PvcNameIfNeeded",
-        "StartCommand"
+        "StartCommand",
+        "DefaultResources"
       ],
       "properties": {
         "ClientPort": {
@@ -17942,6 +17947,9 @@
         "ConfigReloaderImage": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Labels": {
           "type": "object",
           "additionalProperties": {
@@ -18020,7 +18028,8 @@
         "PvcNameIfNeeded",
         "ConfConfigMapName",
         "ScriptsConfigMapName",
-        "HealthConfigMapName"
+        "HealthConfigMapName",
+        "DefaultResources"
       ],
       "properties": {
         "ConfConfigMapName": {
@@ -18029,6 +18038,9 @@
         "CredentialSecretName": {
           "type": "string"
         },
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "HealthConfigMapName": {
           "type": "string"
         },
@@ -18082,9 +18094,13 @@
         "ISBSvcType",
         "Image",
         "PullPolicy",
-        "Env"
+        "Env",
+        "DefaultResources"
       ],
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "type": "array",
           "items": {
@@ -18109,9 +18125,13 @@
         "Image",
         "PullPolicy",
         "Env",
-        "SideInputsStoreName"
+        "SideInputsStoreName",
+        "DefaultResources"
       ],
       "properties": {
+        "DefaultResources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
         "Env": {
           "type": "array",
           "items": {

diff --git a/config/advanced-install/namespaced-controller-wo-crds.yaml b/config/advanced-install/namespaced-controller-wo-crds.yaml
@@ -133,6 +133,11 @@ metadata:
 apiVersion: v1
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/config/base/controller-manager/numaflow-controller-config.yaml b/config/base/controller-manager/numaflow-controller-config.yaml
@@ -4,6 +4,11 @@ metadata:
   name: numaflow-controller-config
 data:
   controller-config.yaml: |+
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/config/install.yaml b/config/install.yaml
@@ -16325,6 +16325,11 @@ metadata:
 apiVersion: v1
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml
@@ -16228,6 +16228,11 @@ metadata:
 apiVersion: v1
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/docs/APIs.md b/docs/APIs.md
@@ -1416,6 +1416,15 @@ Kubernetes core/v1.PullPolicy </a> </em>
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetJetStreamServiceSpecReq">
@@ -1587,6 +1596,15 @@ Description
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetRedisServiceSpecReq">
@@ -1751,6 +1769,15 @@ Description
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetSideInputDeploymentReq">
@@ -1804,6 +1831,15 @@ Kubernetes core/v1.PullPolicy </a> </em>
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetVertexPodSpecReq">
@@ -1864,6 +1900,15 @@ Kubernetes core/v1.PullPolicy </a> </em>
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>DefaultResources</code></br> <em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements </a> </em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GroupBy">

diff --git a/docs/operations/controller-configmap.md b/docs/operations/controller-configmap.md
@@ -15,11 +15,37 @@ metadata:
   name: numaflow-controller-config
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        ...
     isbsvc:
       jetstream:
         ...
 ```
 
+### Default Controller Configuration
+
+Currently, we support configuring the init and main container resources for steps across all the pipelines. The configuration is under `defaults` key in the ConfigMap.
+
+For example, to set the default container resources for steps across all the pipelines:
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: numaflow-controller-config
+data:
+  controller-config.yaml: |
+    defaults:
+      containerResources: |
+        limits:
+          memory: "256Mi"
+          cpu: "200m"
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+```
+
+
 ### ISB Service Configuration
 
 One of the important configuration items in the ConfigMap is about [ISB Service](../core-concepts/inter-step-buffer-service.md). We currently use 3rd party technologies such as `JetStream` to implement ISB Services, if those applications have new releases, to make them available in Numaflow, the new versions need to be added in the ConfigMap.

diff --git a/docs/operations/numaflow-controller-config.yaml b/docs/operations/numaflow-controller-config.yaml
@@ -4,6 +4,11 @@ metadata:
   name: numaflow-controller-config
 data:
   controller-config.yaml: |
+    defaults:
+      containerResources: |
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
     isbsvc:
       redis:
         # Default Redis settings, could be overridden by InterStepBufferService specs

diff --git a/pkg/apis/numaflow/v1alpha1/const.go b/pkg/apis/numaflow/v1alpha1/const.go
@@ -19,9 +19,6 @@ package v1alpha1
 import (
 	"fmt"
 	"time"
-
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 const (
@@ -183,12 +180,4 @@ const (
 var (
 	MessageTagDrop = fmt.Sprintf("%U__DROP__", '\\') // U+005C__DROP__
 	MessageTagAll  = fmt.Sprintf("%U__ALL__", '\\')  // U+005C__ALL__
-
-	// the standard resources used by the `init` and `main`containers.
-	standardResources = corev1.ResourceRequirements{
-		Requests: corev1.ResourceList{
-			"cpu":    resource.MustParse("100m"),
-			"memory": resource.MustParse("128Mi"),
-		},
-	}
 )