apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: annotations: workflows.argoproj.io/pod-name-format: v2 creationTimestamp: "2024-04-12T14:33:13Z" deletionGracePeriodSeconds: 0 deletionTimestamp: "2024-04-12T14:35:58Z" finalizers: - workflows.argoproj.io/artifact-gc generateName: workflow- generation: 11 labels: events.argoproj.io/action-timestamp: "1712932393152" events.argoproj.io/sensor: workflow-sensor events.argoproj.io/trigger: workflow-trigger workflows.argoproj.io/completed: "true" workflows.argoproj.io/phase: Failed name: workflow-j7mjs namespace: workflow resourceVersion: "32247481" uid: f3dcd76e-957a-4d6b-969b-7d040692ade8 spec: arguments: parameters: - name: INPUT_BUCKET_NAME value: data-bucket - name: INPUT_PRODUCT_NAME value: path/to/input-file priority: 1 workflowTemplateRef: name: create-data status: artifactGCStatus: podsRecouped: workflow-j7mjs-artgc-wfdel-473657125: true strategiesProcessed: OnWorkflowCompletion: true OnWorkflowDeletion: true artifactRepositoryRef: artifactRepository: s3: accessKeySecret: key: USERNAME name: artifact-repository bucket: argo-artifacts endpoint: s3.storage:9999 insecure: true secretKeySecret: key: PASSWORD name: artifact-repository default: true conditions: - status: "False" type: PodRunning - status: "True" type: Completed - message: 'Artifact Garbage Collection failed for strategy workflow-j7mjs-artgc-wfdel-473657125, pod OnWorkflowDeletion exited with non-zero exit code: check pod logs for more information' status: "True" type: ArtifactGCError finishedAt: "2024-04-12T14:34:57Z" nodes: workflow-j7mjs: children: - workflow-j7mjs-538700425 displayName: workflow-j7mjs finishedAt: "2024-04-12T14:34:57Z" id: workflow-j7mjs inputs: parameters: - name: INPUT_BUCKET_NAME value: data-bucket - name: INPUT_PRODUCT_NAME value: path/to/input-file name: workflow-j7mjs outboundNodes: - workflow-j7mjs-3807881620 - workflow-j7mjs-3284464227 - workflow-j7mjs-2828888727 - workflow-j7mjs-2888026804 phase: Failed progress: 1/5 resourcesDuration: cpu: 62 memory: 653 startedAt: "2024-04-12T14:33:13Z" templateName: create-data templateScope: local/ type: DAG workflow-j7mjs-538700425: boundaryID: workflow-j7mjs children: - workflow-j7mjs-3242448406 - workflow-j7mjs-2762173675 - workflow-j7mjs-879604029 - workflow-j7mjs-2949086152 displayName: download-raw-data finishedAt: "2024-04-12T14:33:16Z" hostNodeName: node-5.company.com id: workflow-j7mjs-538700425 inputs: parameters: - name: BUCKET_NAME value: data-bucket - name: PRODUCT_NAME value: path/to/input-file name: workflow-j7mjs.download-raw-data outputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-j7mjs/workflow-j7mjs-download-raw-data-538700425/RAW_DATA.tgz - name: main-logs s3: key: workflow-j7mjs/workflow-j7mjs-download-raw-data-538700425/main.log exitCode: "0" phase: Succeeded progress: 1/1 resourcesDuration: cpu: 0 memory: 4 startedAt: "2024-04-12T14:33:13Z" templateName: download-raw-data templateScope: local/ type: Pod workflow-j7mjs-879604029: boundaryID: workflow-j7mjs children: - workflow-j7mjs-2828888727 displayName: create-variant-a finishedAt: "2024-04-12T14:34:45Z" hostNodeName: node-6.company.com id: workflow-j7mjs-879604029 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-j7mjs/workflow-j7mjs-download-raw-data-538700425/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-1 - name: MAP_TYPE value: map-type-1 message: Error (exit code 1) name: workflow-j7mjs.create-variant-a outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-j7mjs/workflow-j7mjs-create-data-variant-879604029/FINAL_DATA.tgz - name: main-logs s3: key: workflow-j7mjs/workflow-j7mjs-create-data-variant-879604029/main.log exitCode: "1" phase: Failed progress: 0/1 resourcesDuration: cpu: 15 memory: 161 startedAt: "2024-04-12T14:33:23Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-j7mjs-2762173675: boundaryID: workflow-j7mjs children: - workflow-j7mjs-3284464227 displayName: create-variant-d finishedAt: "2024-04-12T14:34:55Z" hostNodeName: node-5.company.com id: workflow-j7mjs-2762173675 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-j7mjs/workflow-j7mjs-download-raw-data-538700425/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-2 - name: MAP_TYPE value: map-type-2 message: Pod was active on the node longer than the specified deadline name: workflow-j7mjs.create-variant-d outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data - name: main-logs s3: key: workflow-j7mjs/workflow-j7mjs-create-data-variant-2762173675/main.log exitCode: "143" phase: Failed progress: 0/1 resourcesDuration: cpu: 18 memory: 182 startedAt: "2024-04-12T14:33:23Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-j7mjs-2828888727: boundaryID: workflow-j7mjs displayName: upload-variant-a finishedAt: "2024-04-12T14:34:47Z" id: workflow-j7mjs-2828888727 message: 'omitted: depends condition not met' name: workflow-j7mjs.upload-variant-a nodeFlag: {} phase: Omitted startedAt: "2024-04-12T14:34:47Z" templateName: upload-data-variant templateScope: local/ type: Skipped workflow-j7mjs-2888026804: boundaryID: workflow-j7mjs displayName: upload-variant-b finishedAt: "2024-04-12T14:34:57Z" id: workflow-j7mjs-2888026804 message: 'omitted: depends condition not met' name: workflow-j7mjs.upload-variant-b nodeFlag: {} phase: Omitted startedAt: "2024-04-12T14:34:57Z" templateName: upload-data-variant templateScope: local/ type: Skipped workflow-j7mjs-2949086152: boundaryID: workflow-j7mjs children: - workflow-j7mjs-2888026804 displayName: create-variant-b finishedAt: "2024-04-12T14:34:55Z" hostNodeName: node-7.company.com id: workflow-j7mjs-2949086152 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-j7mjs/workflow-j7mjs-download-raw-data-538700425/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-2 - name: MAP_TYPE value: map-type-1 message: Pod was active on the node longer than the specified deadline name: workflow-j7mjs.create-variant-b outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-j7mjs/workflow-j7mjs-create-data-variant-2949086152/FINAL_DATA.tgz - name: main-logs s3: key: workflow-j7mjs/workflow-j7mjs-create-data-variant-2949086152/main.log exitCode: "0" phase: Failed progress: 0/1 resourcesDuration: cpu: 17 memory: 180 startedAt: "2024-04-12T14:33:23Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-j7mjs-3242448406: boundaryID: workflow-j7mjs children: - workflow-j7mjs-3807881620 displayName: create-variant-c finishedAt: "2024-04-12T14:34:27Z" hostNodeName: node-5.company.com id: workflow-j7mjs-3242448406 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-j7mjs/workflow-j7mjs-download-raw-data-538700425/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-1 - name: MAP_TYPE value: map-type-2 message: Error (exit code 1) name: workflow-j7mjs.create-variant-c outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-j7mjs/workflow-j7mjs-create-data-variant-3242448406/FINAL_DATA.tgz - name: main-logs s3: key: workflow-j7mjs/workflow-j7mjs-create-data-variant-3242448406/main.log exitCode: "1" phase: Failed progress: 0/1 resourcesDuration: cpu: 12 memory: 126 startedAt: "2024-04-12T14:33:23Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-j7mjs-3284464227: boundaryID: workflow-j7mjs displayName: upload-variant-d finishedAt: "2024-04-12T14:34:57Z" id: workflow-j7mjs-3284464227 message: 'omitted: depends condition not met' name: workflow-j7mjs.upload-variant-d nodeFlag: {} phase: Omitted startedAt: "2024-04-12T14:34:57Z" templateName: upload-data-variant templateScope: local/ type: Skipped workflow-j7mjs-3807881620: boundaryID: workflow-j7mjs displayName: upload-variant-c finishedAt: "2024-04-12T14:34:37Z" id: workflow-j7mjs-3807881620 message: 'omitted: depends condition not met' name: workflow-j7mjs.upload-variant-c nodeFlag: {} phase: Omitted startedAt: "2024-04-12T14:34:37Z" templateName: upload-data-variant templateScope: local/ type: Skipped phase: Failed progress: 1/5 resourcesDuration: cpu: 62 memory: 653 startedAt: "2024-04-12T14:33:13Z" storedTemplates: namespaced/create-data/create-data: dag: tasks: - arguments: parameters: - name: BUCKET_NAME value: '{{inputs.parameters.INPUT_BUCKET_NAME}}' - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' name: download-raw-data template: download-raw-data - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-a template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-b template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-c template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-d template: create-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-c.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_C dependencies: - create-variant-c name: upload-variant-c template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-d.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_D dependencies: - create-variant-d name: upload-variant-d template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-a.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_A dependencies: - create-variant-a name: upload-variant-a template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-b.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_B dependencies: - create-variant-b name: upload-variant-b template: upload-data-variant inputs: parameters: - name: INPUT_BUCKET_NAME - name: INPUT_PRODUCT_NAME metadata: {} name: create-data outputs: {} serviceAccountName: workflow namespaced/create-data/create-data-variant: activeDeadlineSeconds: 90 container: command: - sh - -c - data-processor env: - name: DATA_PROCESSOR_INPUT_FILE value: /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} - name: DATA_PROCESSOR_OUTPUT_DIR value: /tmp/final-data - name: DATA_PROCESSOR_OUTPUT_FLAT value: "False" - name: DATA_PROCESSOR_OUTPUT_ZIPPED value: "False" - name: DATA_PROCESSOR_REGION value: '{{inputs.parameters.REGION}}' - name: DATA_PROCESSOR_MAP_TYPE value: '{{inputs.parameters.MAP_TYPE}}' - name: DATA_PROCESSOR_PLOT value: "True" - name: DATA_PROCESSOR_DPI_PLOT value: "300" - name: DATA_PROCESSOR_QUICKLOOK value: "True" - name: DATA_PROCESSOR_DPI_QUICKLOOK value: "60" - name: DATA_PROCESSOR_LOG_DIR value: /tmp/log image: company.com/data-processor:latest name: "" resources: {} inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data parameters: - name: PRODUCT_NAME - name: REGION - name: MAP_TYPE metadata: {} name: create-data-variant outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data serviceAccountName: workflow namespaced/create-data/download-raw-data: activeDeadlineSeconds: 60 container: command: - rclone - copyto - --s3-no-check-bucket - --verbose - s3:{{inputs.parameters.BUCKET_NAME}}/{{inputs.parameters.PRODUCT_NAME}} - /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: parameters: - name: BUCKET_NAME - name: PRODUCT_NAME metadata: {} name: download-raw-data outputs: artifacts: - name: RAW_DATA path: /tmp/raw-data serviceAccountName: workflow namespaced/create-data/upload-data-variant: activeDeadlineSeconds: 60 container: args: - copy - --ignore-existing - --no-traverse - --s3-no-check-bucket - --verbose - /tmp/final-data/{{inputs.parameters.PRODUCT_ID}}/ - s3:{{inputs.parameters.BUCKET}}/{{inputs.parameters.PRODUCT_ID}}/ command: - rclone env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: artifacts: - name: FINAL_DATA path: /tmp/final-data parameters: - name: BUCKET - name: PRODUCT_ID metadata: {} name: upload-data-variant outputs: {} serviceAccountName: workflow storedWorkflowTemplateSpec: archiveLogs: true arguments: parameters: - name: INPUT_BUCKET_NAME value: data-bucket - name: INPUT_PRODUCT_NAME value: path/to/input-file artifactGC: serviceAccountName: workflow strategy: OnWorkflowDeletion entrypoint: create-data imagePullSecrets: - name: registry-auth podGC: strategy: OnPodCompletion priority: 1 synchronization: semaphore: configMapKeyRef: key: create-data name: semaphore templates: - dag: tasks: - arguments: parameters: - name: BUCKET_NAME value: '{{inputs.parameters.INPUT_BUCKET_NAME}}' - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' name: download-raw-data template: download-raw-data - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-a template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-b template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-c template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-d template: create-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-c.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_C dependencies: - create-variant-c name: upload-variant-c template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-d.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_D dependencies: - create-variant-d name: upload-variant-d template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-a.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_A dependencies: - create-variant-a name: upload-variant-a template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-b.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_B dependencies: - create-variant-b name: upload-variant-b template: upload-data-variant inputs: parameters: - name: INPUT_BUCKET_NAME - name: INPUT_PRODUCT_NAME metadata: {} name: create-data outputs: {} serviceAccountName: workflow - activeDeadlineSeconds: 60 container: command: - rclone - copyto - --s3-no-check-bucket - --verbose - s3:{{inputs.parameters.BUCKET_NAME}}/{{inputs.parameters.PRODUCT_NAME}} - /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: parameters: - name: BUCKET_NAME - name: PRODUCT_NAME metadata: {} name: download-raw-data outputs: artifacts: - name: RAW_DATA path: /tmp/raw-data serviceAccountName: workflow - activeDeadlineSeconds: 90 container: command: - sh - -c - data-processor env: - name: DATA_PROCESSOR_INPUT_FILE value: /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} - name: DATA_PROCESSOR_OUTPUT_DIR value: /tmp/final-data - name: DATA_PROCESSOR_OUTPUT_FLAT value: "False" - name: DATA_PROCESSOR_OUTPUT_ZIPPED value: "False" - name: DATA_PROCESSOR_REGION value: '{{inputs.parameters.REGION}}' - name: DATA_PROCESSOR_MAP_TYPE value: '{{inputs.parameters.MAP_TYPE}}' - name: DATA_PROCESSOR_PLOT value: "True" - name: DATA_PROCESSOR_DPI_PLOT value: "300" - name: DATA_PROCESSOR_QUICKLOOK value: "True" - name: DATA_PROCESSOR_DPI_QUICKLOOK value: "60" - name: DATA_PROCESSOR_LOG_DIR value: /tmp/log image: company.com/data-processor:latest name: "" resources: {} inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data parameters: - name: PRODUCT_NAME - name: REGION - name: MAP_TYPE metadata: {} name: create-data-variant outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data serviceAccountName: workflow - activeDeadlineSeconds: 60 container: args: - copy - --ignore-existing - --no-traverse - --s3-no-check-bucket - --verbose - /tmp/final-data/{{inputs.parameters.PRODUCT_ID}}/ - s3:{{inputs.parameters.BUCKET}}/{{inputs.parameters.PRODUCT_ID}}/ command: - rclone env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: artifacts: - name: FINAL_DATA path: /tmp/final-data parameters: - name: BUCKET - name: PRODUCT_ID metadata: {} name: upload-data-variant outputs: {} serviceAccountName: workflow ttlStrategy: secondsAfterCompletion: 84600 secondsAfterFailure: 60 secondsAfterSuccess: 0 workflowTemplateRef: name: create-data taskResultsCompletionStatus: workflow-j7mjs-538700425: true workflow-j7mjs-879604029: true workflow-j7mjs-2762173675: true workflow-j7mjs-2949086152: true workflow-j7mjs-3242448406: true