feat(controller): Workflows progress. Closes #2717

api/openapi-spec/swagger.json

@@ -3042,6 +3042,10 @@
           "description": "PodIP captures the IP of the pod for daemoned steps",
           "type": "string"
         },
+        "progress": {
+          "description": "Progress to completion",
+          "type": "string"
+        },
         "resourcesDuration": {
           "description": "ResourcesDuration is indicative, but not accurate, resource duration. This is populated when the nodes completes.",
           "type": "object",
@@ -4602,6 +4606,10 @@
           "description": "Phase a simple, high-level summary of where the workflow is in its lifecycle.",
           "type": "string"
         },
+        "progress": {
+          "description": "Progress to completion",
+          "type": "string"
+        },
         "resourcesDuration": {
           "description": "ResourcesDuration is the total for the workflow",
           "type": "object",

cmd/argo/commands/get.go

@@ -145,6 +145,7 @@ func printWorkflowHelper(wf *wfv1.Workflow, getArgs getFlags) string {
 		if wf.Status.EstimatedDuration > 0 {
 			out += fmt.Sprintf(fmtStr, "EstimatedDuration:", humanize.Duration(wf.Status.EstimatedDuration.ToDuration()))
 		}
+		out += fmt.Sprintf(fmtStr, "Progress:", wf.Status.Progress)
 	}
 	if !wf.Status.ResourcesDuration.IsZero() {
 		out += fmt.Sprintf(fmtStr, "ResourcesDuration:", wf.Status.ResourcesDuration)

cmd/argo/commands/get_test.go

@@ -109,6 +109,16 @@ func TestStatusToNodeFieldSelector(t *testing.T) {
 }
 
 func Test_printWorkflowHelper(t *testing.T) {
+	t.Run("Progress", func(t *testing.T) {
+		var wf wfv1.Workflow
+		testutil.MustUnmarshallYAML(`
+status:
+  phase: Running
+  progress: 1/2
+`, &wf)
+		output := printWorkflowHelper(&wf, getFlags{})
+		assert.Regexp(t, `Progress: *1/2`, output)
+	})
 	t.Run("EstimatedDuration", func(t *testing.T) {
 		var wf wfv1.Workflow
 		testutil.MustUnmarshallYAML(`

docs/fields.md

@@ -318,6 +318,8 @@ WorkflowTemplate is the definition of a workflow template resource
 
 - [`cron-backfill.yaml`](https://github.com/argoproj/argo/blob/master/examples/cron-backfill.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`templates.yaml`](https://github.com/argoproj/argo/blob/master/examples/workflow-template/templates.yaml)
 </details>
 
@@ -529,6 +531,8 @@ WorkflowSpec is the specification of a Workflow.
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)
@@ -672,6 +676,7 @@ WorkflowStatus contains overall status information about a workflow
 |`outputs`|[`Outputs`](#outputs)|Outputs captures output values and artifact locations produced by the workflow via global outputs|
 |`persistentVolumeClaims`|`Array<`[`Volume`](#volume)`>`|PersistentVolumeClaims tracks all PVCs that were created as part of the io.argoproj.workflow.v1alpha1. The contents of this list are drained at the end of the workflow.|
 |`phase`|`string`|Phase a simple, high-level summary of where the workflow is in its lifecycle.|
+|`progress`|`string`|Progress to completion|
 |`resourcesDuration`|`Map< integer , int64 >`|ResourcesDuration is the total for the workflow|
 |`startedAt`|[`Time`](#time)|Time at which this workflow started|
 |`storedTemplates`|[`Template`](#template)|StoredTemplates is a mapping between a template ref and the node's status.|
@@ -878,6 +883,8 @@ CronWorkflowSpec is the specification of a CronWorkflow
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)
@@ -1187,6 +1194,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate.
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)
@@ -1788,6 +1797,8 @@ Template is a reusable and composable unit of execution in a workflow
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)
@@ -1996,6 +2007,7 @@ NodeStatus contains status information about an individual node in the workflow
 |`outputs`|[`Outputs`](#outputs)|Outputs captures output parameter values and artifact locations produced by this template invocation|
 |`phase`|`string`|Phase a simple, high-level summary of where the node is in its lifecycle. Can be used as a state machine.|
 |`podIP`|`string`|PodIP captures the IP of the pod for daemoned steps|
+|`progress`|`string`|Progress to completion|
 |`resourcesDuration`|`Map< integer , int64 >`|ResourcesDuration is indicative, but not accurate, resource duration. This is populated when the nodes completes.|
 |`startedAt`|[`Time`](#time)|Time at which this node started|
 |~`storedTemplateID`~|~`string`~|~StoredTemplateID is the ID of stored template.~ DEPRECATED: This value is not used anymore.|
@@ -2906,6 +2918,8 @@ Pod metdata
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)
@@ -4179,6 +4193,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)
@@ -4703,6 +4719,8 @@ A single application container that you want to run within a pod.
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`resubmit.yaml`](https://github.com/argoproj/argo/blob/master/examples/resubmit.yaml)
 
 - [`retry-backoff.yaml`](https://github.com/argoproj/argo/blob/master/examples/retry-backoff.yaml)
@@ -5361,6 +5379,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)
@@ -6380,6 +6400,8 @@ ListMeta describes metadata that synthetic resources must have, including lists
 
 - [`pod-spec-yaml-patch.yaml`](https://github.com/argoproj/argo/blob/master/examples/pod-spec-yaml-patch.yaml)
 
+- [`progress-workflowtemplate.yaml`](https://github.com/argoproj/argo/blob/master/examples/progress-workflowtemplate.yaml)
+
 - [`recursive-for-loop.yaml`](https://github.com/argoproj/argo/blob/master/examples/recursive-for-loop.yaml)
 
 - [`resource-delete-with-flags.yaml`](https://github.com/argoproj/argo/blob/master/examples/resource-delete-with-flags.yaml)

docs/progress.md

@@ -0,0 +1,29 @@
+# Workflow Progress
+
+![alpha](assets/alpha.svg)
+
+> v2.12 and after
+
+When you run a workflow, the controller will report on its progress.
+
+We define progress as two numbers, `N/M` such that `0 <= N <= M and 0 <= M <= 1`. 
+
+* `N` is the number of completed tasks.
+* `M` is the total number of tasks.
+
+E.g. `0/0`, `0/1` or `50/100`.
+
+Unlike [estimated duration](estimated-duration.md), progress is deterministic. I.e. it will be the same for each workflow, regardless of any problems. 
+
+Progress for each node is calculated as follows:
+
+1. For a pod node, the most recently logged `#argo progress=N/M` value.
+2. For a leaf node either `1/1` in completed or `0/1` otherwise.
+3. For non-leaf nodes, the sum of its children.
+
+For a whole workflow's, progress is the sum of all its leaf nodes.
+
+This allows you to have pods which report their own progress just by logging it. They should always print out `M/M` (i.e `N = M`) as the final line to correctly report progress.
+
+!!! Warning 
+    `M` will increase during workflow run each time a node is added to the graph.

docs/swagger.md

@@ -1261,6 +1261,7 @@ NodeStatus contains status information about an individual node in the workflow
 | outputs | [io.argoproj.workflow.v1alpha1.Outputs](#io.argoproj.workflow.v1alpha1.outputs) | Outputs captures output parameter values and artifact locations produced by this template invocation | No |
 | phase | string | Phase a simple, high-level summary of where the node is in its lifecycle. Can be used as a state machine. | No |
 | podIP | string | PodIP captures the IP of the pod for daemoned steps | No |
+| progress | string | Progress to completion | No |
 | resourcesDuration | object | ResourcesDuration is indicative, but not accurate, resource duration. This is populated when the nodes completes. | No |
 | startedAt | [io.k8s.apimachinery.pkg.apis.meta.v1.Time](#io.k8s.apimachinery.pkg.apis.meta.v1.time) | Time at which this node started | No |
 | storedTemplateID | string | StoredTemplateID is the ID of stored template. DEPRECATED: This value is not used anymore. | No |
@@ -1825,6 +1826,7 @@ WorkflowStatus contains overall status information about a workflow
 | outputs | [io.argoproj.workflow.v1alpha1.Outputs](#io.argoproj.workflow.v1alpha1.outputs) | Outputs captures output values and artifact locations produced by the workflow via global outputs | No |
 | persistentVolumeClaims | [ [io.k8s.api.core.v1.Volume](#io.k8s.api.core.v1.volume) ] | PersistentVolumeClaims tracks all PVCs that were created as part of the io.argoproj.workflow.v1alpha1. The contents of this list are drained at the end of the workflow. | No |
 | phase | string | Phase a simple, high-level summary of where the workflow is in its lifecycle. | No |
+| progress | string | Progress to completion | No |
 | resourcesDuration | object | ResourcesDuration is the total for the workflow | No |
 | startedAt | [io.k8s.apimachinery.pkg.apis.meta.v1.Time](#io.k8s.apimachinery.pkg.apis.meta.v1.time) | Time at which this workflow started | No |
 | storedTemplates | object | StoredTemplates is a mapping between a template ref and the node's status. | No |

examples/progress-workflowtemplate.yaml

@@ -0,0 +1,18 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: progress
+spec:
+  entrypoint: main
+  templates:
+    - name: main
+      container:
+        image: argoproj/argosay:v1
+        command: [ sh, -c ]
+        args:
+          - |
+            for p in $(seq 1 100); do
+              echo "#argo progress=$p/100, message=\"my argo message\""
+              sleep 10s
+            done
+

manifests/base/crds/full/argoproj.io_workflows.yaml

@@ -7416,6 +7416,8 @@ spec:
                     type: string
                   podIP:
                     type: string
+                  progress:
+                    type: string
                   resourcesDuration:
                     additionalProperties:
                       format: int64
@@ -8393,6 +8395,8 @@ spec:
               type: array
             phase:
               type: string
+            progress:
+              type: string
             resourcesDuration:
               additionalProperties:
                 format: int64

mkdocs.yml

@@ -42,6 +42,7 @@ nav:
           - artifact-repository-ref.md
           - resource-duration.md
           - estimated-duration.md
+          - progress.md
           - workflow-creator.md
           - synchronization.md
           - workflow-of-workflows.md