From 3306034a00e63591949789df01d18e87c7a6e70f Mon Sep 17 00:00:00 2001 From: John Ibsen Date: Wed, 26 Feb 2025 12:57:28 -0500 Subject: [PATCH 1/3] fix(circleci): Add CircleCI workflow nil check for CreatedDate Adds a nil check during `convertWorkflows` task when CircleCI workflows API returns a weird malformed pipeline data with no created_date. Previously, the entire task fails and panics when trying to get convert a nil in ToTime util. Fixes Issue #8309 --- .../e2e/raw_tables/_raw_circleci_api_workflows.csv | 11 +++++++++++ backend/plugins/circleci/tasks/workflow_converter.go | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv index 43edf350c1f..3f3beb0a2de 100644 --- a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv +++ b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv @@ -98,3 +98,14 @@ id,params,data,url,input,created_at ""created_at"" : ""2023-03-25T18:06:13Z"", ""stopped_at"" : ""2023-03-25T18:06:28Z"" }",https://circleci.com/api/v2/pipeline/afe0cabe-e7ee-4eb7-bf13-bb6170d139f0/workflow,"{""id"": ""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:58.245 +28,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""pipeline_id"" : ""f5a2a6e8-4430-4935-8bec-77ca2b26c61d"", + ""id"" : ""b3b77371-e27a-4e93-a554-c728f757cd24"", + ""name"" : ""workflow"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""pipeline_number"" : 16, + ""created_at"" : """", + ""stopped_at"" : """" + }",https://circleci.com/api/v2/pipeline/f5a2a6e8-4430-4935-8bec-77ca2b26c61d/workflow,"{""id"": ""f5a2a6e8-4430-4935-8bec-77ca2b26c61d"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": """", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:45:58.245 diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go index 7389b22716c..fc678a83552 100644 --- a/backend/plugins/circleci/tasks/workflow_converter.go +++ b/backend/plugins/circleci/tasks/workflow_converter.go @@ -40,6 +40,9 @@ var ConvertWorkflowsMeta = plugin.SubTaskMeta{ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_WORKFLOW_TABLE) + logger := taskCtx.GetLogger() + logger.Info("convert workflows") + db := taskCtx.GetDal() clauses := []dal.Clause{ dal.From(&models.CircleciWorkflow{}), @@ -57,6 +60,15 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { Input: cursor, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { userTool := inputRow.(*models.CircleciWorkflow) + // Check if CreatedDate is null + if userTool.CreatedDate == nil { + if userTool.PipelineId != nil { + logger.Warn("CreatedDate is null in the CircleCI API response for %s", userTool.PipelineId) + } else { + logger.Warn("CreatedDate is null in the CircleCI API response, skipping this entry") + } + return nil, nil + } createdAt := userTool.CreatedDate.ToTime() pipeline := &devops.CICDPipeline{ DomainEntity: domainlayer.DomainEntity{ From b7a57accc1ac865be1f4e7c9548dc64ca7fb078d Mon Sep 17 00:00:00 2001 From: John Ibsen Date: Wed, 26 Feb 2025 18:37:36 +0000 Subject: [PATCH 2/3] Update to fix compile problems --- backend/plugins/circleci/tasks/workflow_converter.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go index fc678a83552..dc13c23af70 100644 --- a/backend/plugins/circleci/tasks/workflow_converter.go +++ b/backend/plugins/circleci/tasks/workflow_converter.go @@ -62,11 +62,7 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { userTool := inputRow.(*models.CircleciWorkflow) // Check if CreatedDate is null if userTool.CreatedDate == nil { - if userTool.PipelineId != nil { - logger.Warn("CreatedDate is null in the CircleCI API response for %s", userTool.PipelineId) - } else { - logger.Warn("CreatedDate is null in the CircleCI API response, skipping this entry") - } + logger.Info("CreatedDate is null in the CircleCI API response for %s", userTool.PipelineId) return nil, nil } createdAt := userTool.CreatedDate.ToTime() From 282fdcab95963cb8370775a91d64764a3a62688e Mon Sep 17 00:00:00 2001 From: John Ibsen Date: Wed, 26 Feb 2025 21:39:52 +0000 Subject: [PATCH 3/3] Tweak the _tool csv to allow for the NULL value entries --- .../e2e/raw_tables/_raw_circleci_api_workflows.csv | 2 +- .../e2e/snapshot_tables/_tool_circleci_workflows.csv | 1 + backend/plugins/circleci/tasks/workflow_converter.go | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv index 3f3beb0a2de..5760d52da61 100644 --- a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv +++ b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv @@ -104,7 +104,7 @@ id,params,data,url,input,created_at ""name"" : ""workflow"", ""project_slug"" : ""gh/coldgust/coldgust.github.io"", ""status"" : ""failed"", - ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""started_by"" : """", ""pipeline_number"" : 16, ""created_at"" : """", ""stopped_at"" : """" diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv index 2ca84d9a0b0..bd6af7cc904 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv @@ -4,6 +4,7 @@ connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status 1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:28.000+00:00,5 1,8971a56b-5547-4824-94dd-07bb467524c5,github/coldgust/coldgust.github.io,87aad008-1ad5-486a-8174-fdeed846561a,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,2,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:23.000+00:00,5 1,8fe60291-68f7-40e2-acec-d99bff4da713,github/coldgust/coldgust.github.io,afef32b3-5ffe-48d2-8d9e-46dcedd82554,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,1,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:22.000+00:00,4 +1,b3b77371-e27a-4e93-a554-c728f757cd24,github/coldgust/coldgust.github.io,f5a2a6e8-4430-4935-8bec-77ca2b26c61d,,workflow,,,failed,,16,,,0 1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,14 1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,5 1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,15 diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go index dc13c23af70..fa5ecf8644b 100644 --- a/backend/plugins/circleci/tasks/workflow_converter.go +++ b/backend/plugins/circleci/tasks/workflow_converter.go @@ -60,10 +60,10 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { Input: cursor, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { userTool := inputRow.(*models.CircleciWorkflow) - // Check if CreatedDate is null - if userTool.CreatedDate == nil { - logger.Info("CreatedDate is null in the CircleCI API response for %s", userTool.PipelineId) - return nil, nil + // Skip if CreatedDate is null or empty string - still enters into the `_tool_circleci_workflows` table with null values + if userTool.CreatedDate.ToNullableTime() == nil { + logger.Info("CreatedDate is null or empty string in the CircleCI API response for %s", userTool.PipelineId) + return []interface{}{}, nil } createdAt := userTool.CreatedDate.ToTime() pipeline := &devops.CICDPipeline{