diff --git a/docs/fields.md b/docs/fields.md index 9dc48663f0d0..8e283589bb54 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -90,6 +90,8 @@ Workflow is the definition of a workflow resource - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -389,6 +391,8 @@ WorkflowSpec is the specification of a Workflow. - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -699,6 +703,8 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -974,6 +980,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -1225,6 +1233,8 @@ Arguments to a template - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`influxdb-ci.yaml`](../examples/influxdb-ci.yaml) @@ -1456,6 +1466,8 @@ Template is a reusable and composable unit of execution in a workflow - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -1752,6 +1764,8 @@ Outputs hold parameters, artifacts, and results from a step - [`global-outputs.yaml`](../examples/global-outputs.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`influxdb-ci.yaml`](../examples/influxdb-ci.yaml) @@ -1815,6 +1829,8 @@ Artifact indicates an artifact to place at a specified path - [`global-outputs.yaml`](../examples/global-outputs.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`influxdb-ci.yaml`](../examples/influxdb-ci.yaml) @@ -1910,6 +1926,8 @@ Parameter indicate a passed string parameter to a service template with an optio - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`influxdb-ci.yaml`](../examples/influxdb-ci.yaml) - [`k8s-jobs.yaml`](../examples/k8s-jobs.yaml) @@ -2180,6 +2198,8 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`global-outputs.yaml`](../examples/global-outputs.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`influxdb-ci.yaml`](../examples/influxdb-ci.yaml) @@ -2351,6 +2371,8 @@ Pod metdata - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -2710,6 +2732,8 @@ WorkflowStep is a reference to a template to execute in a series of step - [`global-outputs.yaml`](../examples/global-outputs.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -3072,6 +3096,8 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`global-outputs.yaml`](../examples/global-outputs.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`k8s-jobs.yaml`](../examples/k8s-jobs.yaml) - [`k8s-orchestration.yaml`](../examples/k8s-orchestration.yaml) @@ -3326,6 +3352,8 @@ Sequence expands a workflow step into numeric range - [`cron-backfill.yaml`](../examples/cron-backfill.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`loops-sequence.yaml`](../examples/loops-sequence.yaml) - [`work-avoidance.yaml`](../examples/work-avoidance.yaml) @@ -3465,6 +3493,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -3935,6 +3965,8 @@ A single application container that you want to run within a pod. - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -4547,6 +4579,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) @@ -5108,6 +5142,8 @@ EnvVarSource represents a source for the value of an EnvVar. - [`global-outputs.yaml`](../examples/global-outputs.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`k8s-jobs.yaml`](../examples/k8s-jobs.yaml) - [`k8s-orchestration.yaml`](../examples/k8s-orchestration.yaml) @@ -5515,6 +5551,8 @@ ListMeta describes metadata that synthetic resources must have, including lists - [`global-parameters.yaml`](../examples/global-parameters.yaml) +- [`handle-large-output-results.yaml`](../examples/handle-large-output-results.yaml) + - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml) - [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml) diff --git a/examples/handle-large-output-results.yaml b/examples/handle-large-output-results.yaml new file mode 100644 index 000000000000..ee2bcc2ebf91 --- /dev/null +++ b/examples/handle-large-output-results.yaml @@ -0,0 +1,55 @@ +# Output parameters and 'results' are passed between pods via annotations. Annotations have a size +# limit. If your output is larger than 256 kB, you should use an artifact instead of a parameter. +# +# If you're looping over an output parameter using withParam, and your parameter gets too large, you +# can use an output artifact combined with a 'count' and withSequence loop to work around the size +# limit. See (https://stackoverflow.com/q/62537126/684776) for further explanation. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: handle-large-output-results- +spec: + entrypoint: handle-large-output-results + templates: + - name: handle-large-output-results + steps: + - - name: get-items + template: get-items + - - name: sequence-param + template: echo + arguments: + parameters: + - name: index + value: "{{item}}" + artifacts: + - name: items + from: "{{steps.get-items.outputs.artifacts.items}}" + withSequence: + count: "{{steps.get-items.outputs.parameters.count}}" + - name: get-items + container: + image: alpine:latest + command: ["/bin/sh", "-c"] + # This small array is just for an example. Make sure the count matches the actual size of + # your JSON array. + args: ["echo '[\"a\", \"b\", \"c\"]' > /tmp/items && echo '3' > /tmp/count"] + outputs: + artifacts: + - name: items + path: /tmp/items + parameters: + - name: count + valueFrom: + path: /tmp/count + - name: echo + inputs: + parameters: + - name: index + artifacts: + - name: items + path: /tmp/items + container: + image: stedolan/jq:latest + command: [sh, -c] + # jq is just one way to get the appropriate item. + args: ["cat /tmp/items | jq '.[{{inputs.parameters.index}}]'"] \ No newline at end of file