apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: annotations: workflows.argoproj.io/pod-name-format: v2 creationTimestamp: "2024-04-19T07:26:08Z" finalizers: - workflows.argoproj.io/artifact-gc generateName: workflow- generation: 7 labels: events.argoproj.io/action-timestamp: "1713511568287" events.argoproj.io/sensor: workflow-sensor events.argoproj.io/trigger: workflow-trigger workflows.argoproj.io/completed: "true" workflows.argoproj.io/phase: Failed name: workflow-nz98s namespace: workflow resourceVersion: "40343682" uid: cd999df2-9bb8-4be1-b22f-f09b106fa002 spec: arguments: parameters: - name: INPUT_BUCKET_NAME value: data-bucket - name: INPUT_PRODUCT_NAME value: path/to/input-file priority: 1 workflowTemplateRef: name: create-data status: artifactGCStatus: strategiesProcessed: OnWorkflowCompletion: true artifactRepositoryRef: artifactRepository: s3: accessKeySecret: key: USERNAME name: artifact-repository bucket: argo-artifacts endpoint: s3.storage:9999 insecure: true secretKeySecret: key: PASSWORD name: artifact-repository default: true conditions: - status: "False" type: PodRunning - status: "True" type: Completed finishedAt: "2024-04-19T07:26:57Z" nodes: workflow-nz98s: children: - workflow-nz98s-595829946 displayName: workflow-nz98s finishedAt: "2024-04-19T07:26:57Z" id: workflow-nz98s inputs: parameters: - name: INPUT_BUCKET_NAME value: data-bucket - name: INPUT_PRODUCT_NAME value: path/to/input-file name: workflow-nz98s outboundNodes: - workflow-nz98s-887702939 - workflow-nz98s-2781052136 - workflow-nz98s-2568760254 - workflow-nz98s-3817893249 phase: Failed progress: 5/7 resourcesDuration: cpu: 12 memory: 170 startedAt: "2024-04-19T07:26:08Z" templateName: create-data templateScope: local/ type: DAG workflow-nz98s-595829946: boundaryID: workflow-nz98s children: - workflow-nz98s-1887220395 - workflow-nz98s-2146061522 - workflow-nz98s-3695595158 - workflow-nz98s-3215320427 displayName: download-raw-data finishedAt: "2024-04-19T07:26:12Z" hostNodeName: node-6.company.com id: workflow-nz98s-595829946 inputs: parameters: - name: BUCKET_NAME value: data-bucket - name: PRODUCT_NAME value: path/to/input-file name: workflow-nz98s.download-raw-data outputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-nz98s/workflow-nz98s-download-raw-data-595829946/RAW_DATA.tgz - name: main-logs s3: key: workflow-nz98s/workflow-nz98s-download-raw-data-595829946/main.log exitCode: "0" phase: Succeeded progress: 1/1 resourcesDuration: cpu: 0 memory: 4 startedAt: "2024-04-19T07:26:08Z" templateName: download-raw-data templateScope: local/ type: Pod workflow-nz98s-887702939: boundaryID: workflow-nz98s displayName: upload-variant-c finishedAt: "2024-04-19T07:26:47Z" id: workflow-nz98s-887702939 message: 'omitted: depends condition not met' name: workflow-nz98s.upload-variant-c nodeFlag: {} phase: Omitted startedAt: "2024-04-19T07:26:47Z" templateName: upload-data-variant templateScope: local/ type: Skipped workflow-nz98s-1887220395: boundaryID: workflow-nz98s children: - workflow-nz98s-887702939 displayName: create-variant-c finishedAt: "2024-04-19T07:26:37Z" hostNodeName: node-6.company.com id: workflow-nz98s-1887220395 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-nz98s/workflow-nz98s-download-raw-data-595829946/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-1 - name: MAP_TYPE value: map-type-2 message: Error (exit code 1) name: workflow-nz98s.create-variant-c outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-1887220395/FINAL_DATA.tgz - name: main-logs s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-1887220395/main.log exitCode: "1" phase: Failed progress: 0/1 resourcesDuration: cpu: 2 memory: 33 startedAt: "2024-04-19T07:26:18Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-nz98s-2146061522: boundaryID: workflow-nz98s children: - workflow-nz98s-2781052136 displayName: create-variant-d finishedAt: "2024-04-19T07:26:44Z" hostNodeName: node-6.company.com id: workflow-nz98s-2146061522 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-nz98s/workflow-nz98s-download-raw-data-595829946/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-2 - name: MAP_TYPE value: map-type-2 name: workflow-nz98s.create-variant-d outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-2146061522/FINAL_DATA.tgz - name: main-logs s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-2146061522/main.log exitCode: "0" phase: Succeeded progress: 1/1 resourcesDuration: cpu: 4 memory: 47 startedAt: "2024-04-19T07:26:18Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-nz98s-2568760254: boundaryID: workflow-nz98s displayName: upload-variant-a finishedAt: "2024-04-19T07:26:47Z" id: workflow-nz98s-2568760254 message: 'omitted: depends condition not met' name: workflow-nz98s.upload-variant-a nodeFlag: {} phase: Omitted startedAt: "2024-04-19T07:26:47Z" templateName: upload-data-variant templateScope: local/ type: Skipped workflow-nz98s-2781052136: boundaryID: workflow-nz98s displayName: upload-variant-d finishedAt: "2024-04-19T07:26:50Z" hostNodeName: node-6.company.com id: workflow-nz98s-2781052136 inputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-2146061522/FINAL_DATA.tgz parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_D name: workflow-nz98s.upload-variant-d outputs: artifacts: - name: main-logs s3: key: workflow-nz98s/workflow-nz98s-upload-data-variant-2781052136/main.log exitCode: "0" phase: Succeeded progress: 1/1 resourcesDuration: cpu: 0 memory: 3 startedAt: "2024-04-19T07:26:47Z" templateName: upload-data-variant templateScope: local/ type: Pod workflow-nz98s-3215320427: boundaryID: workflow-nz98s children: - workflow-nz98s-3817893249 displayName: create-variant-b finishedAt: "2024-04-19T07:26:44Z" hostNodeName: node-6.company.com id: workflow-nz98s-3215320427 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-nz98s/workflow-nz98s-download-raw-data-595829946/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-2 - name: MAP_TYPE value: map-type-1 name: workflow-nz98s.create-variant-b outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-3215320427/FINAL_DATA.tgz - name: main-logs s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-3215320427/main.log exitCode: "0" phase: Succeeded progress: 1/1 resourcesDuration: cpu: 4 memory: 47 startedAt: "2024-04-19T07:26:18Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-nz98s-3695595158: boundaryID: workflow-nz98s children: - workflow-nz98s-2568760254 displayName: create-variant-a finishedAt: "2024-04-19T07:26:37Z" hostNodeName: node-6.company.com id: workflow-nz98s-3695595158 inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data s3: key: workflow-nz98s/workflow-nz98s-download-raw-data-595829946/RAW_DATA.tgz parameters: - name: PRODUCT_NAME value: path/to/input-file - name: REGION value: region-1 - name: MAP_TYPE value: map-type-1 message: Error (exit code 1) name: workflow-nz98s.create-variant-a outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-3695595158/FINAL_DATA.tgz - name: main-logs s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-3695595158/main.log exitCode: "1" phase: Failed progress: 0/1 resourcesDuration: cpu: 2 memory: 33 startedAt: "2024-04-19T07:26:18Z" templateName: create-data-variant templateScope: local/ type: Pod workflow-nz98s-3817893249: boundaryID: workflow-nz98s displayName: upload-variant-b finishedAt: "2024-04-19T07:26:50Z" hostNodeName: node-6.company.com id: workflow-nz98s-3817893249 inputs: artifacts: - name: FINAL_DATA path: /tmp/final-data s3: key: workflow-nz98s/workflow-nz98s-create-data-variant-3215320427/FINAL_DATA.tgz parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_B name: workflow-nz98s.upload-variant-b outputs: artifacts: - name: main-logs s3: key: workflow-nz98s/workflow-nz98s-upload-data-variant-3817893249/main.log exitCode: "0" phase: Succeeded progress: 1/1 resourcesDuration: cpu: 0 memory: 3 startedAt: "2024-04-19T07:26:47Z" templateName: upload-data-variant templateScope: local/ type: Pod phase: Failed progress: 5/7 resourcesDuration: cpu: 12 memory: 170 startedAt: "2024-04-19T07:26:08Z" storedTemplates: namespaced/create-data/create-data: dag: tasks: - arguments: parameters: - name: BUCKET_NAME value: '{{inputs.parameters.INPUT_BUCKET_NAME}}' - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' name: download-raw-data template: download-raw-data - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-a template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-b template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-c template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-d template: create-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-c.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_C dependencies: - create-variant-c name: upload-variant-c template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-d.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_D dependencies: - create-variant-d name: upload-variant-d template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-a.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_A dependencies: - create-variant-a name: upload-variant-a template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-b.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_B dependencies: - create-variant-b name: upload-variant-b template: upload-data-variant inputs: parameters: - name: INPUT_BUCKET_NAME - name: INPUT_PRODUCT_NAME metadata: {} name: create-data outputs: {} serviceAccountName: workflow namespaced/create-data/create-data-variant: activeDeadlineSeconds: 90 container: command: - sh - -c - data-processor env: - name: DATA_PROCESSOR_INPUT_FILE value: /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} - name: DATA_PROCESSOR_OUTPUT_DIR value: /tmp/final-data - name: DATA_PROCESSOR_OUTPUT_FLAT value: "False" - name: DATA_PROCESSOR_OUTPUT_ZIPPED value: "False" - name: DATA_PROCESSOR_REGION value: '{{inputs.parameters.REGION}}' - name: DATA_PROCESSOR_MAP_TYPE value: '{{inputs.parameters.MAP_TYPE}}' - name: DATA_PROCESSOR_PLOT value: "True" - name: DATA_PROCESSOR_DPI_PLOT value: "300" - name: DATA_PROCESSOR_QUICKLOOK value: "True" - name: DATA_PROCESSOR_DPI_QUICKLOOK value: "60" - name: DATA_PROCESSOR_LOG_DIR value: /tmp/log image: company.com/data-processor:latest name: "" resources: {} inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data parameters: - name: PRODUCT_NAME - name: REGION - name: MAP_TYPE metadata: {} name: create-data-variant outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data serviceAccountName: workflow namespaced/create-data/download-raw-data: activeDeadlineSeconds: 60 container: command: - rclone - copyto - --s3-no-check-bucket - --verbose - s3:{{inputs.parameters.BUCKET_NAME}}/{{inputs.parameters.PRODUCT_NAME}} - /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: parameters: - name: BUCKET_NAME - name: PRODUCT_NAME metadata: {} name: download-raw-data outputs: artifacts: - name: RAW_DATA path: /tmp/raw-data serviceAccountName: workflow namespaced/create-data/upload-data-variant: activeDeadlineSeconds: 60 container: args: - copy - --ignore-existing - --no-traverse - --s3-no-check-bucket - --verbose - /tmp/final-data/{{inputs.parameters.PRODUCT_ID}}/ - s3:{{inputs.parameters.BUCKET}}/{{inputs.parameters.PRODUCT_ID}}/ command: - rclone env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: artifacts: - name: FINAL_DATA path: /tmp/final-data parameters: - name: BUCKET - name: PRODUCT_ID metadata: {} name: upload-data-variant outputs: {} serviceAccountName: workflow storedWorkflowTemplateSpec: archiveLogs: true arguments: parameters: - name: INPUT_BUCKET_NAME value: data-bucket - name: INPUT_PRODUCT_NAME value: path/to/input-file artifactGC: serviceAccountName: workflow strategy: OnWorkflowDeletion entrypoint: create-data imagePullSecrets: - name: registry-auth podGC: strategy: OnPodCompletion priority: 1 synchronization: semaphore: configMapKeyRef: key: create-data name: semaphore templates: - dag: tasks: - arguments: parameters: - name: BUCKET_NAME value: '{{inputs.parameters.INPUT_BUCKET_NAME}}' - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' name: download-raw-data template: download-raw-data - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-a template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-1 dependencies: - download-raw-data name: create-variant-b template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-1 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-c template: create-data-variant - arguments: artifacts: - from: '{{tasks.download-raw-data.outputs.artifacts.RAW_DATA}}' name: RAW_DATA parameters: - name: PRODUCT_NAME value: '{{inputs.parameters.INPUT_PRODUCT_NAME}}' - name: REGION value: region-2 - name: MAP_TYPE value: map-type-2 dependencies: - download-raw-data name: create-variant-d template: create-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-c.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_C dependencies: - create-variant-c name: upload-variant-c template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-d.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_D dependencies: - create-variant-d name: upload-variant-d template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-a.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_A dependencies: - create-variant-a name: upload-variant-a template: upload-data-variant - arguments: artifacts: - from: '{{tasks.create-variant-b.outputs.artifacts.FINAL_DATA}}' name: FINAL_DATA parameters: - name: BUCKET value: data-bucket - name: PRODUCT_ID value: VARIANT_B dependencies: - create-variant-b name: upload-variant-b template: upload-data-variant inputs: parameters: - name: INPUT_BUCKET_NAME - name: INPUT_PRODUCT_NAME metadata: {} name: create-data outputs: {} serviceAccountName: workflow - activeDeadlineSeconds: 60 container: command: - rclone - copyto - --s3-no-check-bucket - --verbose - s3:{{inputs.parameters.BUCKET_NAME}}/{{inputs.parameters.PRODUCT_NAME}} - /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: parameters: - name: BUCKET_NAME - name: PRODUCT_NAME metadata: {} name: download-raw-data outputs: artifacts: - name: RAW_DATA path: /tmp/raw-data serviceAccountName: workflow - activeDeadlineSeconds: 90 container: command: - sh - -c - data-processor env: - name: DATA_PROCESSOR_INPUT_FILE value: /tmp/raw-data/{{inputs.parameters.PRODUCT_NAME}} - name: DATA_PROCESSOR_OUTPUT_DIR value: /tmp/final-data - name: DATA_PROCESSOR_OUTPUT_FLAT value: "False" - name: DATA_PROCESSOR_OUTPUT_ZIPPED value: "False" - name: DATA_PROCESSOR_REGION value: '{{inputs.parameters.REGION}}' - name: DATA_PROCESSOR_MAP_TYPE value: '{{inputs.parameters.MAP_TYPE}}' - name: DATA_PROCESSOR_PLOT value: "True" - name: DATA_PROCESSOR_DPI_PLOT value: "300" - name: DATA_PROCESSOR_QUICKLOOK value: "True" - name: DATA_PROCESSOR_DPI_QUICKLOOK value: "60" - name: DATA_PROCESSOR_LOG_DIR value: /tmp/log image: company.com/data-processor:latest name: "" resources: {} inputs: artifacts: - name: RAW_DATA path: /tmp/raw-data parameters: - name: PRODUCT_NAME - name: REGION - name: MAP_TYPE metadata: {} name: create-data-variant outputs: artifacts: - name: FINAL_DATA path: /tmp/final-data serviceAccountName: workflow - activeDeadlineSeconds: 60 container: args: - copy - --ignore-existing - --no-traverse - --s3-no-check-bucket - --verbose - /tmp/final-data/{{inputs.parameters.PRODUCT_ID}}/ - s3:{{inputs.parameters.BUCKET}}/{{inputs.parameters.PRODUCT_ID}}/ command: - rclone env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: key: USERNAME name: data-repository - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: key: PASSWORD name: data-repository envFrom: - configMapRef: name: data-repository image: company.com/utils/rclone:latest name: "" resources: {} inputs: artifacts: - name: FINAL_DATA path: /tmp/final-data parameters: - name: BUCKET - name: PRODUCT_ID metadata: {} name: upload-data-variant outputs: {} serviceAccountName: workflow ttlStrategy: secondsAfterCompletion: 84600 secondsAfterFailure: 60 secondsAfterSuccess: 0 workflowTemplateRef: name: create-data taskResultsCompletionStatus: workflow-nz98s-595829946: true workflow-nz98s-1887220395: true workflow-nz98s-2146061522: true workflow-nz98s-2781052136: true workflow-nz98s-3215320427: true workflow-nz98s-3695595158: true workflow-nz98s-3817893249: true