Chore: tickgen changes to test reduce pipelines (#587)

Signed-off-by: ashwinidulams <ashttk@gmail.com>
numaproj · Mar 7, 2023 · abfdd78 · abfdd78
1 parent d8b1aa0
commit abfdd78
Show file tree

Hide file tree

Showing 16 changed files with 620 additions and 380 deletions.
diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json
@@ -17715,6 +17715,11 @@
         "duration": {
           "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration"
         },
+        "keyCount": {
+          "description": "KeyCount is the number of unique keys in the payload",
+          "format": "int32",
+          "type": "integer"
+        },
         "msgSize": {
           "description": "Size of each generated message",
           "format": "int32",
@@ -17723,6 +17728,11 @@
         "rpu": {
           "format": "int64",
           "type": "integer"
+        },
+        "value": {
+          "description": "Value is an optional uint64 value to be written in to the payload",
+          "format": "int64",
+          "type": "integer"
         }
       },
       "type": "object"

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -17720,6 +17720,11 @@
         "duration": {
           "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration"
         },
+        "keyCount": {
+          "description": "KeyCount is the number of unique keys in the payload",
+          "type": "integer",
+          "format": "int32"
+        },
         "msgSize": {
           "description": "Size of each generated message",
           "type": "integer",
@@ -17728,6 +17733,11 @@
         "rpu": {
           "type": "integer",
           "format": "int64"
+        },
+        "value": {
+          "description": "Value is an optional uint64 value to be written in to the payload",
+          "type": "integer",
+          "format": "int64"
         }
       }
     },

diff --git a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml
@@ -3745,6 +3745,9 @@ spec:
                             duration:
                               default: 1s
                               type: string
+                            keyCount:
+                              format: int32
+                              type: integer
                             msgSize:
                               default: 8
                               format: int32
@@ -3753,6 +3756,9 @@ spec:
                               default: 5
                               format: int64
                               type: integer
+                            value:
+                              format: int64
+                              type: integer
                           type: object
                         http:
                           properties:

diff --git a/config/base/crds/full/numaflow.numaproj.io_vertices.yaml b/config/base/crds/full/numaflow.numaproj.io_vertices.yaml
@@ -2309,6 +2309,9 @@ spec:
                       duration:
                         default: 1s
                         type: string
+                      keyCount:
+                        format: int32
+                        type: integer
                       msgSize:
                         default: 8
                         format: int32
@@ -2317,6 +2320,9 @@ spec:
                         default: 5
                         format: int64
                         type: integer
+                      value:
+                        format: int64
+                        type: integer
                     type: object
                   http:
                     properties:

diff --git a/config/install.yaml b/config/install.yaml
@@ -5999,6 +5999,9 @@ spec:
                             duration:
                               default: 1s
                               type: string
+                            keyCount:
+                              format: int32
+                              type: integer
                             msgSize:
                               default: 8
                               format: int32
@@ -6007,6 +6010,9 @@ spec:
                               default: 5
                               format: int64
                               type: integer
+                            value:
+                              format: int64
+                              type: integer
                           type: object
                         http:
                           properties:
@@ -9767,6 +9773,9 @@ spec:
                       duration:
                         default: 1s
                         type: string
+                      keyCount:
+                        format: int32
+                        type: integer
                       msgSize:
                         default: 8
                         format: int32
@@ -9775,6 +9784,9 @@ spec:
                         default: 5
                         format: int64
                         type: integer
+                      value:
+                        format: int64
+                        type: integer
                     type: object
                   http:
                     properties:

diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml
@@ -5999,6 +5999,9 @@ spec:
                             duration:
                               default: 1s
                               type: string
+                            keyCount:
+                              format: int32
+                              type: integer
                             msgSize:
                               default: 8
                               format: int32
@@ -6007,6 +6010,9 @@ spec:
                               default: 5
                               format: int64
                               type: integer
+                            value:
+                              format: int64
+                              type: integer
                           type: object
                         http:
                           properties:
@@ -9767,6 +9773,9 @@ spec:
                       duration:
                         default: 1s
                         type: string
+                      keyCount:
+                        format: int32
+                        type: integer
                       msgSize:
                         default: 8
                         format: int32
@@ -9775,6 +9784,9 @@ spec:
                         default: 5
                         format: int64
                         type: integer
+                      value:
+                        format: int64
+                        type: integer
                     type: object
                   http:
                     properties:

diff --git a/docs/APIs.md b/docs/APIs.md
@@ -1096,6 +1096,26 @@ Size of each generated message
 </p>
 </td>
 </tr>
+<tr>
+<td>
+<code>keyCount</code></br> <em> int32 </em>
+</td>
+<td>
+<p>
+KeyCount is the number of unique keys in the payload
+</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>value</code></br> <em> uint64 </em>
+</td>
+<td>
+<p>
+Value is an optional uint64 value to be written in to the payload
+</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.GetDaemonDeploymentReq">

diff --git a/examples/1-simple-pipeline-keys.yaml b/examples/1-simple-pipeline-keys.yaml
@@ -0,0 +1,27 @@
+apiVersion: numaflow.numaproj.io/v1alpha1
+kind: Pipeline
+metadata:
+  name: simple-pipeline
+spec:
+  vertices:
+    - name: in
+      source:
+        # A self data generating source
+        generator:
+          rpu: 5
+          duration: 1s
+          keyCount: 5
+          value: 5
+    - name: cat
+      udf:
+        builtin:
+          name: cat # A built-in UDF which simply cats the message
+    - name: out
+      sink:
+        # A simple log printing sink
+        log: {}
+  edges:
+    - from: in
+      to: cat
+    - from: cat
+      to: out