From ba76b6112bcc5661e0f0b0661a590a135b679219 Mon Sep 17 00:00:00 2001 From: abeizn Date: Mon, 1 Aug 2022 17:35:55 +0800 Subject: [PATCH 1/9] feat: github add cicd feature --- plugins/github/impl/impl.go | 4 + plugins/github/models/job.go | 52 +++++++++ .../20220728_add_github_runs_table.go | 82 ++++++++++++++ .../20220729_add_github_jobs_table.go | 75 +++++++++++++ .../models/migrationscripts/register.go | 2 + plugins/github/models/run.go | 59 ++++++++++ plugins/github/tasks/cicd_job_collector.go | 103 ++++++++++++++++++ plugins/github/tasks/cicd_job_extractor.go | 88 +++++++++++++++ plugins/github/tasks/cicd_run_collector.go | 83 ++++++++++++++ plugins/github/tasks/cicd_run_extractor.go | 96 ++++++++++++++++ 10 files changed, 644 insertions(+) create mode 100644 plugins/github/models/job.go create mode 100644 plugins/github/models/migrationscripts/20220728_add_github_runs_table.go create mode 100644 plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go create mode 100644 plugins/github/models/run.go create mode 100644 plugins/github/tasks/cicd_job_collector.go create mode 100644 plugins/github/tasks/cicd_job_extractor.go create mode 100644 plugins/github/tasks/cicd_run_collector.go create mode 100644 plugins/github/tasks/cicd_run_extractor.go diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go index 56c0a16f7e3..d57345f2de9 100644 --- a/plugins/github/impl/impl.go +++ b/plugins/github/impl/impl.go @@ -78,6 +78,10 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta { tasks.ExtractAccountsMeta, tasks.CollectAccountOrgMeta, tasks.ExtractAccountOrgMeta, + tasks.CollectRunsMeta, + tasks.ExtractRunsMeta, + tasks.CollectJobsMeta, + tasks.ExtractJobsMeta, tasks.EnrichPullRequestIssuesMeta, tasks.ConvertRepoMeta, tasks.ConvertIssuesMeta, diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go new file mode 100644 index 00000000000..e319b33a229 --- /dev/null +++ b/plugins/github/models/job.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/models/common" + "gorm.io/datatypes" +) + +type GithubJob struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` + RunID int `json:"run_id"` + RunURL string `json:"run_url" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + Name string `json:"name" gorm:"type:varchar(255)"` + Steps datatypes.JSON `json:"steps"` + CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` + Labels datatypes.JSON `json:"labels"` + RunnerID int `json:"runner_id"` + RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` + RunnerGroupID int `json:"runner_group_id"` +} + +func (GithubJob) TableName() string { + return "_tool_github_jobs" +} diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go new file mode 100644 index 00000000000..3632e4307d2 --- /dev/null +++ b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "fmt" + "time" + + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "gorm.io/gorm" +) + +type GithubRun20220728 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` +} + +func (GithubRun20220728) TableName() string { + return "_tool_github_runs" +} + +type addGithubRunsTable struct{} + +func (u *addGithubRunsTable) Up(ctx context.Context, db *gorm.DB) error { + // create table + err := db.Migrator().CreateTable(GithubRun20220728{}) + if err != nil { + return fmt.Errorf("create table _tool_github_runs error") + } + return nil + +} + +func (*addGithubRunsTable) Version() uint64 { + return 20220728000001 +} + +func (*addGithubRunsTable) Name() string { + return "Github add github_runs table" +} diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go new file mode 100644 index 00000000000..31cfb210141 --- /dev/null +++ b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "fmt" + "time" + + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "gorm.io/datatypes" + "gorm.io/gorm" +) + +type GithubJob20220729 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` + RunID int `json:"run_id"` + RunURL string `json:"run_url" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + Name string `json:"name" gorm:"type:varchar(255)"` + Steps datatypes.JSON `json:"steps"` + CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` + Labels datatypes.JSON `json:"labels"` + RunnerID int `json:"runner_id"` + RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` + RunnerGroupID int `json:"runner_group_id"` +} + +func (GithubJob20220729) TableName() string { + return "_tool_github_jobs" +} + +type addGithubJobsTable struct{} + +func (u *addGithubJobsTable) Up(ctx context.Context, db *gorm.DB) error { + // create table + err := db.Migrator().CreateTable(GithubJob20220729{}) + if err != nil { + return fmt.Errorf("create table _tool_github_jobs error") + } + return nil + +} + +func (*addGithubJobsTable) Version() uint64 { + return 20220729000001 +} + +func (*addGithubJobsTable) Name() string { + return "Github add github_jobs table" +} diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go index c1365f7d4f2..4f899e789cc 100644 --- a/plugins/github/models/migrationscripts/register.go +++ b/plugins/github/models/migrationscripts/register.go @@ -25,5 +25,7 @@ import ( func All() []migration.Script { return []migration.Script{ new(addInitTables), + new(addGithubRunsTable), + new(addGithubJobsTable), } } diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go new file mode 100644 index 00000000000..fbc923ead35 --- /dev/null +++ b/plugins/github/models/run.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/models/common" +) + +type GithubRun struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` +} + +func (GithubRun) TableName() string { + return "_tool_github_runs" +} diff --git a/plugins/github/tasks/cicd_job_collector.go b/plugins/github/tasks/cicd_job_collector.go new file mode 100644 index 00000000000..d78034adf40 --- /dev/null +++ b/plugins/github/tasks/cicd_job_collector.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/helper" +) + +const RAW_JOB_TABLE = "github_api_jobs" + +var CollectJobsMeta = core.SubTaskMeta{ + Name: "collectJobs", + EntryPoint: CollectJobs, + EnabledByDefault: true, + Description: "Collect Jobs data from Github action api", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func CollectJobs(taskCtx core.SubTaskContext) error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubTaskData) + cursor, err := db.Cursor( + dal.Select("id"), + dal.From(models.GithubRun{}.TableName()), + ) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGithubRun{})) + if err != nil { + return err + } + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_JOB_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 100, + Input: iterator, + Incremental: false, + UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/actions/runs/{{ .Input.ID }}/jobs", + Query: func(reqData *helper.RequestData) (url.Values, error) { + query := url.Values{} + query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) + query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { + body := &GithubRawJobsResult{} + err := helper.UnmarshalResponse(res, body) + if err != nil { + return nil, err + } + return body.GithubWorkflowJobs, nil + }, + }) + + if err != nil { + return err + } + return collector.Execute() +} + +type SimpleGithubRun struct { + ID int64 +} + +type GithubRawJobsResult struct { + TotalCount int64 `json:"total_count"` + GithubWorkflowJobs []json.RawMessage `json:"jobs"` +} diff --git a/plugins/github/tasks/cicd_job_extractor.go b/plugins/github/tasks/cicd_job_extractor.go new file mode 100644 index 00000000000..9cbf58dbd1c --- /dev/null +++ b/plugins/github/tasks/cicd_job_extractor.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/helper" +) + +var ExtractJobsMeta = core.SubTaskMeta{ + Name: "extractJobs", + EntryPoint: ExtractJobs, + EnabledByDefault: true, + Description: "Extract raw run data into tool layer table github_jobs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func ExtractJobs(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*GithubTaskData) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_JOB_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, error) { + githubJob := &models.GithubJob{} + err := json.Unmarshal(row.Data, githubJob) + if err != nil { + return nil, err + } + + results := make([]interface{}, 0, 1) + githubJobResult := &models.GithubJob{ + ConnectionId: data.Options.ConnectionId, + ID: githubJob.ID, + RunID: githubJob.RunID, + RunURL: githubJob.RunURL, + NodeID: githubJob.NodeID, + HeadSha: githubJob.HeadSha, + URL: githubJob.URL, + HTMLURL: githubJob.HTMLURL, + Status: githubJob.Status, + Conclusion: githubJob.Conclusion, + StartedAt: githubJob.StartedAt, + CompletedAt: githubJob.CompletedAt, + Name: githubJob.Name, + Steps: githubJob.Steps, + CheckRunURL: githubJob.CheckRunURL, + Labels: githubJob.Labels, + RunnerID: githubJob.RunID, + RunnerName: githubJob.RunnerName, + RunnerGroupID: githubJob.RunnerGroupID, + } + results = append(results, githubJobResult) + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/plugins/github/tasks/cicd_run_collector.go b/plugins/github/tasks/cicd_run_collector.go new file mode 100644 index 00000000000..91068da35bd --- /dev/null +++ b/plugins/github/tasks/cicd_run_collector.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" +) + +const RAW_RUN_TABLE = "github_api_runs" + +var CollectRunsMeta = core.SubTaskMeta{ + Name: "collectRuns", + EntryPoint: CollectRuns, + EnabledByDefault: true, + Description: "Collect Runs data from Github action api", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func CollectRuns(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*GithubTaskData) + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_RUN_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 100, + Incremental: false, + UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/actions/runs", + Query: func(reqData *helper.RequestData) (url.Values, error) { + query := url.Values{} + query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) + query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { + body := &GithubRawRunsResult{} + err := helper.UnmarshalResponse(res, body) + if err != nil { + return nil, err + } + return body.GithubWorkflowRuns, nil + }, + }) + + if err != nil { + return err + } + return collector.Execute() +} + +type GithubRawRunsResult struct { + TotalCount int64 `json:"total_count"` + GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"` +} diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go new file mode 100644 index 00000000000..fa64c71f8cf --- /dev/null +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -0,0 +1,96 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/helper" +) + +var ExtractRunsMeta = core.SubTaskMeta{ + Name: "extractRuns", + EntryPoint: ExtractRuns, + EnabledByDefault: true, + Description: "Extract raw run data into tool layer table github_runs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func ExtractRuns(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*GithubTaskData) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_RUN_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, error) { + githubRun := &models.GithubRun{} + err := json.Unmarshal(row.Data, githubRun) + if err != nil { + return nil, err + } + + results := make([]interface{}, 0, 1) + githubRunResult := &models.GithubRun{ + ConnectionId: data.Options.ConnectionId, + ID: githubRun.ID, + Name: githubRun.Name, + NodeID: githubRun.NodeID, + HeadBranch: githubRun.HeadBranch, + HeadSha: githubRun.HeadSha, + Path: githubRun.Path, + RunNumber: githubRun.RunNumber, + Event: githubRun.Event, + Status: githubRun.Status, + Conclusion: githubRun.Conclusion, + WorkflowID: githubRun.WorkflowID, + CheckSuiteID: githubRun.CheckSuiteID, + CheckSuiteNodeID: githubRun.CheckSuiteNodeID, + URL: githubRun.URL, + HTMLURL: githubRun.HTMLURL, + CreatedAt: githubRun.CreatedAt, + UpdatedAt: githubRun.UpdatedAt, + RunAttempt: githubRun.RunAttempt, + RunStartedAt: githubRun.RunStartedAt, + JobsURL: githubRun.JobsURL, + LogsURL: githubRun.LogsURL, + CheckSuiteURL: githubRun.CheckSuiteURL, + ArtifactsURL: githubRun.ArtifactsURL, + CancelURL: githubRun.CancelURL, + RerunURL: githubRun.RerunURL, + WorkflowURL: githubRun.WorkflowURL, + } + results = append(results, githubRunResult) + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} From 2bdbaac04af40f5214ffda3d81292d4a3895582e Mon Sep 17 00:00:00 2001 From: abeizn Date: Tue, 2 Aug 2022 16:33:24 +0800 Subject: [PATCH 2/9] feat: github add cicd feature and update fields --- plugins/github/models/job.go | 4 +- .../20220728_add_github_runs_table.go | 55 ++++++++++--------- .../20220729_add_github_jobs_table.go | 4 +- plugins/github/models/run.go | 55 ++++++++++--------- plugins/github/tasks/cicd_run_extractor.go | 6 +- 5 files changed, 64 insertions(+), 60 deletions(-) diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go index e319b33a229..15acaa6b258 100644 --- a/plugins/github/models/job.go +++ b/plugins/github/models/job.go @@ -36,8 +36,8 @@ type GithubJob struct { HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` Status string `json:"status" gorm:"type:varchar(255)"` Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - StartedAt time.Time `json:"started_at"` - CompletedAt time.Time `json:"completed_at"` + StartedAt *time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at"` Name string `json:"name" gorm:"type:varchar(255)"` Steps datatypes.JSON `json:"steps"` CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go index 3632e4307d2..bc5b630713d 100644 --- a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go +++ b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go @@ -28,33 +28,34 @@ import ( type GithubRun20220728 struct { archived.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` - Name string `json:"name" gorm:"type:varchar(255)"` - NodeID string `json:"node_id" gorm:"type:varchar(255)"` - HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` - HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` - Path string `json:"path" gorm:"type:varchar(255)"` - RunNumber int `json:"run_number"` - Event string `json:"event" gorm:"type:varchar(255)"` - Status string `json:"status" gorm:"type:varchar(255)"` - Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - WorkflowID int `json:"workflow_id"` - CheckSuiteID int64 `json:"check_suite_id"` - CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` - URL string `json:"url" gorm:"type:varchar(255)"` - HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - RunAttempt int `json:"run_attempt"` - RunStartedAt time.Time `json:"run_started_at"` - JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` - LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` - CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` - ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` - CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` - RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` - WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + ConnectionId uint64 `gorm:"primaryKey"` + GithubId int `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + GithubCreatedAt *time.Time `json:"created_at"` + GithubUpdatedAt *time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt *time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` } func (GithubRun20220728) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go index 31cfb210141..981addd4b77 100644 --- a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go +++ b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go @@ -39,8 +39,8 @@ type GithubJob20220729 struct { HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` Status string `json:"status" gorm:"type:varchar(255)"` Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - StartedAt time.Time `json:"started_at"` - CompletedAt time.Time `json:"completed_at"` + StartedAt *time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at"` Name string `json:"name" gorm:"type:varchar(255)"` Steps datatypes.JSON `json:"steps"` CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go index fbc923ead35..d749f3a62d4 100644 --- a/plugins/github/models/run.go +++ b/plugins/github/models/run.go @@ -25,33 +25,34 @@ import ( type GithubRun struct { common.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` - Name string `json:"name" gorm:"type:varchar(255)"` - NodeID string `json:"node_id" gorm:"type:varchar(255)"` - HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` - HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` - Path string `json:"path" gorm:"type:varchar(255)"` - RunNumber int `json:"run_number"` - Event string `json:"event" gorm:"type:varchar(255)"` - Status string `json:"status" gorm:"type:varchar(255)"` - Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - WorkflowID int `json:"workflow_id"` - CheckSuiteID int64 `json:"check_suite_id"` - CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` - URL string `json:"url" gorm:"type:varchar(255)"` - HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - RunAttempt int `json:"run_attempt"` - RunStartedAt time.Time `json:"run_started_at"` - JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` - LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` - CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` - ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` - CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` - RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` - WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + ConnectionId uint64 `gorm:"primaryKey"` + GithubId int `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + GithubCreatedAt *time.Time `json:"created_at"` + GithubUpdatedAt *time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt *time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` } func (GithubRun) TableName() string { diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go index fa64c71f8cf..87d07b728ea 100644 --- a/plugins/github/tasks/cicd_run_extractor.go +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -35,6 +35,7 @@ var ExtractRunsMeta = core.SubTaskMeta{ func ExtractRuns(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -56,6 +57,7 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubRunResult := &models.GithubRun{ ConnectionId: data.Options.ConnectionId, + GithubId: repoId, ID: githubRun.ID, Name: githubRun.Name, NodeID: githubRun.NodeID, @@ -71,8 +73,8 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { CheckSuiteNodeID: githubRun.CheckSuiteNodeID, URL: githubRun.URL, HTMLURL: githubRun.HTMLURL, - CreatedAt: githubRun.CreatedAt, - UpdatedAt: githubRun.UpdatedAt, + GithubCreatedAt: githubRun.GithubCreatedAt, + GithubUpdatedAt: githubRun.GithubUpdatedAt, RunAttempt: githubRun.RunAttempt, RunStartedAt: githubRun.RunStartedAt, JobsURL: githubRun.JobsURL, From 04108dda740c81b951db04db890e1dbe53e0b2f9 Mon Sep 17 00:00:00 2001 From: abeizn Date: Thu, 4 Aug 2022 18:14:48 +0800 Subject: [PATCH 3/9] feat: github cicd enricher from run to pipeline --- plugins/github/impl/impl.go | 1 + plugins/github/models/job.go | 1 + ...ns_table.go => 20220728_add_runs_table.go} | 2 +- ...bs_table.go => 20220729_add_jobs_table.go} | 1 + .../20220802_add_pipeline_table.go | 64 +++++++++++++ .../models/migrationscripts/register.go | 1 + plugins/github/models/pipeline.go | 41 ++++++++ plugins/github/models/run.go | 2 +- plugins/github/tasks/cicd_job_extractor.go | 2 + plugins/github/tasks/cicd_run_enricher.go | 94 +++++++++++++++++++ plugins/github/tasks/cicd_run_extractor.go | 4 +- 11 files changed, 209 insertions(+), 4 deletions(-) rename plugins/github/models/migrationscripts/{20220728_add_github_runs_table.go => 20220728_add_runs_table.go} (97%) rename plugins/github/models/migrationscripts/{20220729_add_github_jobs_table.go => 20220729_add_jobs_table.go} (97%) create mode 100644 plugins/github/models/migrationscripts/20220802_add_pipeline_table.go create mode 100644 plugins/github/models/pipeline.go create mode 100644 plugins/github/tasks/cicd_run_enricher.go diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go index d57345f2de9..bdb992234a2 100644 --- a/plugins/github/impl/impl.go +++ b/plugins/github/impl/impl.go @@ -82,6 +82,7 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta { tasks.ExtractRunsMeta, tasks.CollectJobsMeta, tasks.ExtractJobsMeta, + tasks.EnrichPipelinesMeta, tasks.EnrichPullRequestIssuesMeta, tasks.ConvertRepoMeta, tasks.ConvertIssuesMeta, diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go index 15acaa6b258..59e35284af7 100644 --- a/plugins/github/models/job.go +++ b/plugins/github/models/job.go @@ -45,6 +45,7 @@ type GithubJob struct { RunnerID int `json:"runner_id"` RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` RunnerGroupID int `json:"runner_group_id"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubJob) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_runs_table.go similarity index 97% rename from plugins/github/models/migrationscripts/20220728_add_github_runs_table.go rename to plugins/github/models/migrationscripts/20220728_add_runs_table.go index bc5b630713d..0a996885bfe 100644 --- a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go +++ b/plugins/github/models/migrationscripts/20220728_add_runs_table.go @@ -29,7 +29,6 @@ import ( type GithubRun20220728 struct { archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` - GithubId int `gorm:"primaryKey"` ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` Name string `json:"name" gorm:"type:varchar(255)"` NodeID string `json:"node_id" gorm:"type:varchar(255)"` @@ -56,6 +55,7 @@ type GithubRun20220728 struct { CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubRun20220728) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go similarity index 97% rename from plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go rename to plugins/github/models/migrationscripts/20220729_add_jobs_table.go index 981addd4b77..a0cb978bebb 100644 --- a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go +++ b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go @@ -48,6 +48,7 @@ type GithubJob20220729 struct { RunnerID int `json:"runner_id"` RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` RunnerGroupID int `json:"runner_group_id"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubJob20220729) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go new file mode 100644 index 00000000000..48e11f61f12 --- /dev/null +++ b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "fmt" + "time" + + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "gorm.io/gorm" +) + +type GithubPipeline20220803 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"` + Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"` + StartedDate *time.Time `json:"started_time"` + FinishedDate *time.Time `json:"finished_time"` + Duration float64 `json:"duration"` + Status string `json:"status" gorm:"type:varchar(255)"` + Results string `json:"results" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` +} + +func (GithubPipeline20220803) TableName() string { + return "_tool_github_pipelines" +} + +type addGithubPipelineTable struct{} + +func (u *addGithubPipelineTable) Up(ctx context.Context, db *gorm.DB) error { + // create table + err := db.Migrator().CreateTable(GithubPipeline20220803{}) + if err != nil { + return fmt.Errorf("create table _tool_github_pipelines error") + } + return nil + +} + +func (*addGithubPipelineTable) Version() uint64 { + return 20220803000001 +} + +func (*addGithubPipelineTable) Name() string { + return "Github add github_pipelines table" +} diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go index 4f899e789cc..439d77d48b6 100644 --- a/plugins/github/models/migrationscripts/register.go +++ b/plugins/github/models/migrationscripts/register.go @@ -27,5 +27,6 @@ func All() []migration.Script { new(addInitTables), new(addGithubRunsTable), new(addGithubJobsTable), + new(addGithubPipelineTable), } } diff --git a/plugins/github/models/pipeline.go b/plugins/github/models/pipeline.go new file mode 100644 index 00000000000..2f5ad6ecfe7 --- /dev/null +++ b/plugins/github/models/pipeline.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/models/common" +) + +type GithubPipeline struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"` + Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"` + StartedDate *time.Time `json:"started_time"` + FinishedDate *time.Time `json:"finished_time"` + Duration float64 `json:"duration"` + Status string `json:"status" gorm:"type:varchar(255)"` + Results string `json:"results" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` +} + +func (GithubPipeline) TableName() string { + return "_tool_github_pipelines" +} diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go index d749f3a62d4..e3c7642e3bf 100644 --- a/plugins/github/models/run.go +++ b/plugins/github/models/run.go @@ -26,7 +26,6 @@ import ( type GithubRun struct { common.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` - GithubId int `gorm:"primaryKey"` ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` Name string `json:"name" gorm:"type:varchar(255)"` NodeID string `json:"node_id" gorm:"type:varchar(255)"` @@ -53,6 +52,7 @@ type GithubRun struct { CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubRun) TableName() string { diff --git a/plugins/github/tasks/cicd_job_extractor.go b/plugins/github/tasks/cicd_job_extractor.go index 9cbf58dbd1c..cc2e4960c33 100644 --- a/plugins/github/tasks/cicd_job_extractor.go +++ b/plugins/github/tasks/cicd_job_extractor.go @@ -74,6 +74,8 @@ func ExtractJobs(taskCtx core.SubTaskContext) error { RunnerID: githubJob.RunID, RunnerName: githubJob.RunnerName, RunnerGroupID: githubJob.RunnerGroupID, + // TODO + //Type: "CI/CD", } results = append(results, githubJobResult) return results, nil diff --git a/plugins/github/tasks/cicd_run_enricher.go b/plugins/github/tasks/cicd_run_enricher.go new file mode 100644 index 00000000000..31a93145cb6 --- /dev/null +++ b/plugins/github/tasks/cicd_run_enricher.go @@ -0,0 +1,94 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" + githubModels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var EnrichPipelinesMeta = core.SubTaskMeta{ + Name: "enrichPipelines", + EntryPoint: EnrichPipelines, + EnabledByDefault: true, + Description: "Create tool layer table github_pipelines from github_runs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubTaskData) + + entity := &githubModels.GithubPipeline{} + cursor, err := db.Cursor( + dal.Select("head_sha, head_branch, status, conclusion, github_created_at, github_updated_at, run_attempt, run_started_at"), + dal.From(&githubModels.GithubRun{}), + dal.Orderby("head_sha"), + ) + if err != nil { + return err + } + defer cursor.Close() + + for cursor.Next() { + var item githubModels.GithubRun + err = db.Fetch(cursor, &item) + if err != nil { + return err + } + + if item.HeadSha != entity.Commit { + entity.ConnectionId = data.Options.ConnectionId + entity.Commit = item.HeadSha + entity.Branch = item.HeadBranch // to do + entity.StartedDate = item.GithubCreatedAt + entity.FinishedDate = item.GithubUpdatedAt + entity.Status = item.Status + if entity.Status == "completed" { + entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) + } + entity.Results = item.Conclusion + // TODO + entity.Type = "CI/CD" + } else { + if item.GithubCreatedAt.Before(*entity.StartedDate) { + entity.StartedDate = item.GithubCreatedAt + } + if item.GithubUpdatedAt.After(*entity.FinishedDate) { + entity.FinishedDate = item.GithubCreatedAt + } + if item.Status != "completed" { + entity.Status = item.Status + } else { + entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) + } + if item.Conclusion != "success" { + entity.Results = item.Conclusion + } + + } + err := db.CreateOrUpdate(entity) + if err != nil { + return err + } + } + + return err + +} diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go index 87d07b728ea..eedaa8f8461 100644 --- a/plugins/github/tasks/cicd_run_extractor.go +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -35,7 +35,6 @@ var ExtractRunsMeta = core.SubTaskMeta{ func ExtractRuns(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) - repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -57,7 +56,6 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubRunResult := &models.GithubRun{ ConnectionId: data.Options.ConnectionId, - GithubId: repoId, ID: githubRun.ID, Name: githubRun.Name, NodeID: githubRun.NodeID, @@ -84,6 +82,8 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { CancelURL: githubRun.CancelURL, RerunURL: githubRun.RerunURL, WorkflowURL: githubRun.WorkflowURL, + // TODO + //Type: "CI/CD", } results = append(results, githubRunResult) return results, nil From de920c0a6e369169e3e2ad3e470e1bd251e54cbf Mon Sep 17 00:00:00 2001 From: abeizn Date: Mon, 1 Aug 2022 17:35:55 +0800 Subject: [PATCH 4/9] feat: github add cicd feature --- .../models/migrationscripts/20220728_add_github_runs_table.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go index bc5b630713d..8c13e10d9c3 100644 --- a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go +++ b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go @@ -29,7 +29,6 @@ import ( type GithubRun20220728 struct { archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` - GithubId int `gorm:"primaryKey"` ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` Name string `json:"name" gorm:"type:varchar(255)"` NodeID string `json:"node_id" gorm:"type:varchar(255)"` From 8e7a3011a45abba5fc488dbc9eb8f85cf33ac99b Mon Sep 17 00:00:00 2001 From: abeizn Date: Tue, 2 Aug 2022 16:33:24 +0800 Subject: [PATCH 5/9] feat: github add cicd feature and update fields --- plugins/github/tasks/cicd_run_extractor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go index eedaa8f8461..9b727ac17c0 100644 --- a/plugins/github/tasks/cicd_run_extractor.go +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -35,6 +35,7 @@ var ExtractRunsMeta = core.SubTaskMeta{ func ExtractRuns(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -56,6 +57,7 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubRunResult := &models.GithubRun{ ConnectionId: data.Options.ConnectionId, + GithubId: repoId, ID: githubRun.ID, Name: githubRun.Name, NodeID: githubRun.NodeID, From 068d77e2d9f86502439636125f73ab522e06b371 Mon Sep 17 00:00:00 2001 From: abeizn Date: Thu, 4 Aug 2022 18:14:48 +0800 Subject: [PATCH 6/9] feat: github cicd enricher from run to pipeline --- .../20220728_add_github_runs_table.go | 82 ------------------- .../20220729_add_github_jobs_table.go | 75 ----------------- plugins/github/tasks/cicd_run_extractor.go | 2 - 3 files changed, 159 deletions(-) delete mode 100644 plugins/github/models/migrationscripts/20220728_add_github_runs_table.go delete mode 100644 plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go deleted file mode 100644 index 8c13e10d9c3..00000000000 --- a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package migrationscripts - -import ( - "context" - "fmt" - "time" - - "github.com/apache/incubator-devlake/models/migrationscripts/archived" - "gorm.io/gorm" -) - -type GithubRun20220728 struct { - archived.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` - Name string `json:"name" gorm:"type:varchar(255)"` - NodeID string `json:"node_id" gorm:"type:varchar(255)"` - HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` - HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` - Path string `json:"path" gorm:"type:varchar(255)"` - RunNumber int `json:"run_number"` - Event string `json:"event" gorm:"type:varchar(255)"` - Status string `json:"status" gorm:"type:varchar(255)"` - Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - WorkflowID int `json:"workflow_id"` - CheckSuiteID int64 `json:"check_suite_id"` - CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` - URL string `json:"url" gorm:"type:varchar(255)"` - HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` - GithubCreatedAt *time.Time `json:"created_at"` - GithubUpdatedAt *time.Time `json:"updated_at"` - RunAttempt int `json:"run_attempt"` - RunStartedAt *time.Time `json:"run_started_at"` - JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` - LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` - CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` - ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` - CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` - RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` - WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` -} - -func (GithubRun20220728) TableName() string { - return "_tool_github_runs" -} - -type addGithubRunsTable struct{} - -func (u *addGithubRunsTable) Up(ctx context.Context, db *gorm.DB) error { - // create table - err := db.Migrator().CreateTable(GithubRun20220728{}) - if err != nil { - return fmt.Errorf("create table _tool_github_runs error") - } - return nil - -} - -func (*addGithubRunsTable) Version() uint64 { - return 20220728000001 -} - -func (*addGithubRunsTable) Name() string { - return "Github add github_runs table" -} diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go deleted file mode 100644 index 981addd4b77..00000000000 --- a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package migrationscripts - -import ( - "context" - "fmt" - "time" - - "github.com/apache/incubator-devlake/models/migrationscripts/archived" - "gorm.io/datatypes" - "gorm.io/gorm" -) - -type GithubJob20220729 struct { - archived.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` - RunID int `json:"run_id"` - RunURL string `json:"run_url" gorm:"type:varchar(255)"` - NodeID string `json:"node_id" gorm:"type:varchar(255)"` - HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` - URL string `json:"url" gorm:"type:varchar(255)"` - HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` - Status string `json:"status" gorm:"type:varchar(255)"` - Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - StartedAt *time.Time `json:"started_at"` - CompletedAt *time.Time `json:"completed_at"` - Name string `json:"name" gorm:"type:varchar(255)"` - Steps datatypes.JSON `json:"steps"` - CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` - Labels datatypes.JSON `json:"labels"` - RunnerID int `json:"runner_id"` - RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` - RunnerGroupID int `json:"runner_group_id"` -} - -func (GithubJob20220729) TableName() string { - return "_tool_github_jobs" -} - -type addGithubJobsTable struct{} - -func (u *addGithubJobsTable) Up(ctx context.Context, db *gorm.DB) error { - // create table - err := db.Migrator().CreateTable(GithubJob20220729{}) - if err != nil { - return fmt.Errorf("create table _tool_github_jobs error") - } - return nil - -} - -func (*addGithubJobsTable) Version() uint64 { - return 20220729000001 -} - -func (*addGithubJobsTable) Name() string { - return "Github add github_jobs table" -} diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go index 9b727ac17c0..eedaa8f8461 100644 --- a/plugins/github/tasks/cicd_run_extractor.go +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -35,7 +35,6 @@ var ExtractRunsMeta = core.SubTaskMeta{ func ExtractRuns(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) - repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -57,7 +56,6 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubRunResult := &models.GithubRun{ ConnectionId: data.Options.ConnectionId, - GithubId: repoId, ID: githubRun.ID, Name: githubRun.Name, NodeID: githubRun.NodeID, From 5ebf11d05ec2a1e4d100207fd1840437480ab92e Mon Sep 17 00:00:00 2001 From: abeizn Date: Fri, 5 Aug 2022 09:01:25 +0800 Subject: [PATCH 7/9] feat: github cicd enricher from run to pipeline --- plugins/github/models/job.go | 1 + .../migrationscripts/20220728_add_runs_table.go | 1 + .../migrationscripts/20220729_add_jobs_table.go | 1 + .../20220802_add_pipeline_table.go | 3 ++- plugins/github/models/pipeline.go | 3 ++- plugins/github/models/run.go | 1 + plugins/github/tasks/cicd_job_extractor.go | 2 ++ plugins/github/tasks/cicd_run_enricher.go | 14 +++++++------- plugins/github/tasks/cicd_run_extractor.go | 2 ++ 9 files changed, 19 insertions(+), 9 deletions(-) diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go index 59e35284af7..31a95430b75 100644 --- a/plugins/github/models/job.go +++ b/plugins/github/models/job.go @@ -27,6 +27,7 @@ import ( type GithubJob struct { common.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` RunID int `json:"run_id"` RunURL string `json:"run_url" gorm:"type:varchar(255)"` diff --git a/plugins/github/models/migrationscripts/20220728_add_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_runs_table.go index 0a996885bfe..aee05c57eae 100644 --- a/plugins/github/models/migrationscripts/20220728_add_runs_table.go +++ b/plugins/github/models/migrationscripts/20220728_add_runs_table.go @@ -29,6 +29,7 @@ import ( type GithubRun20220728 struct { archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` Name string `json:"name" gorm:"type:varchar(255)"` NodeID string `json:"node_id" gorm:"type:varchar(255)"` diff --git a/plugins/github/models/migrationscripts/20220729_add_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go index a0cb978bebb..9631dcebcd9 100644 --- a/plugins/github/models/migrationscripts/20220729_add_jobs_table.go +++ b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go @@ -30,6 +30,7 @@ import ( type GithubJob20220729 struct { archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` RunID int `json:"run_id"` RunURL string `json:"run_url" gorm:"type:varchar(255)"` diff --git a/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go index 48e11f61f12..84e1c87aa56 100644 --- a/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go +++ b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go @@ -29,13 +29,14 @@ import ( type GithubPipeline20220803 struct { archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"` Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"` StartedDate *time.Time `json:"started_time"` FinishedDate *time.Time `json:"finished_time"` Duration float64 `json:"duration"` Status string `json:"status" gorm:"type:varchar(255)"` - Results string `json:"results" gorm:"type:varchar(255)"` + Result string `json:"results" gorm:"type:varchar(255)"` Type string `json:"type" gorm:"type:varchar(255)"` } diff --git a/plugins/github/models/pipeline.go b/plugins/github/models/pipeline.go index 2f5ad6ecfe7..0b18e6b5022 100644 --- a/plugins/github/models/pipeline.go +++ b/plugins/github/models/pipeline.go @@ -26,13 +26,14 @@ import ( type GithubPipeline struct { common.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"` Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"` StartedDate *time.Time `json:"started_time"` FinishedDate *time.Time `json:"finished_time"` Duration float64 `json:"duration"` Status string `json:"status" gorm:"type:varchar(255)"` - Results string `json:"results" gorm:"type:varchar(255)"` + Result string `json:"results" gorm:"type:varchar(255)"` Type string `json:"type" gorm:"type:varchar(255)"` } diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go index e3c7642e3bf..bd2ec06c5fe 100644 --- a/plugins/github/models/run.go +++ b/plugins/github/models/run.go @@ -26,6 +26,7 @@ import ( type GithubRun struct { common.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` Name string `json:"name" gorm:"type:varchar(255)"` NodeID string `json:"node_id" gorm:"type:varchar(255)"` diff --git a/plugins/github/tasks/cicd_job_extractor.go b/plugins/github/tasks/cicd_job_extractor.go index cc2e4960c33..b8e8fcae1d6 100644 --- a/plugins/github/tasks/cicd_job_extractor.go +++ b/plugins/github/tasks/cicd_job_extractor.go @@ -35,6 +35,7 @@ var ExtractJobsMeta = core.SubTaskMeta{ func ExtractJobs(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -56,6 +57,7 @@ func ExtractJobs(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubJobResult := &models.GithubJob{ ConnectionId: data.Options.ConnectionId, + RepoId: repoId, ID: githubJob.ID, RunID: githubJob.RunID, RunURL: githubJob.RunURL, diff --git a/plugins/github/tasks/cicd_run_enricher.go b/plugins/github/tasks/cicd_run_enricher.go index 31a93145cb6..30c47fce216 100644 --- a/plugins/github/tasks/cicd_run_enricher.go +++ b/plugins/github/tasks/cicd_run_enricher.go @@ -35,11 +35,10 @@ func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { db := taskCtx.GetDal() data := taskCtx.GetData().(*GithubTaskData) - entity := &githubModels.GithubPipeline{} cursor, err := db.Cursor( dal.Select("head_sha, head_branch, status, conclusion, github_created_at, github_updated_at, run_attempt, run_started_at"), dal.From(&githubModels.GithubRun{}), - dal.Orderby("head_sha"), + dal.Orderby("head_sha, github_created_at"), ) if err != nil { return err @@ -47,6 +46,7 @@ func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { defer cursor.Close() for cursor.Next() { + entity := &githubModels.GithubPipeline{} var item githubModels.GithubRun err = db.Fetch(cursor, &item) if err != nil { @@ -55,15 +55,16 @@ func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { if item.HeadSha != entity.Commit { entity.ConnectionId = data.Options.ConnectionId + entity.RepoId = item.RepoId entity.Commit = item.HeadSha - entity.Branch = item.HeadBranch // to do + entity.Branch = item.HeadBranch entity.StartedDate = item.GithubCreatedAt entity.FinishedDate = item.GithubUpdatedAt entity.Status = item.Status if entity.Status == "completed" { entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) } - entity.Results = item.Conclusion + entity.Result = item.Conclusion // TODO entity.Type = "CI/CD" } else { @@ -72,14 +73,13 @@ func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { } if item.GithubUpdatedAt.After(*entity.FinishedDate) { entity.FinishedDate = item.GithubCreatedAt + entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) } if item.Status != "completed" { entity.Status = item.Status - } else { - entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) } if item.Conclusion != "success" { - entity.Results = item.Conclusion + entity.Result = item.Conclusion } } diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go index eedaa8f8461..703bc652915 100644 --- a/plugins/github/tasks/cicd_run_extractor.go +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -35,6 +35,7 @@ var ExtractRunsMeta = core.SubTaskMeta{ func ExtractRuns(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -57,6 +58,7 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { githubRunResult := &models.GithubRun{ ConnectionId: data.Options.ConnectionId, ID: githubRun.ID, + RepoId: repoId, Name: githubRun.Name, NodeID: githubRun.NodeID, HeadBranch: githubRun.HeadBranch, From 87ac9032156d47c855d9fda73beec595bac9777e Mon Sep 17 00:00:00 2001 From: abeizn Date: Fri, 5 Aug 2022 09:19:23 +0800 Subject: [PATCH 8/9] feat: github cicd enricher from run to pipeline --- plugins/github/tasks/cicd_run_enricher.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/github/tasks/cicd_run_enricher.go b/plugins/github/tasks/cicd_run_enricher.go index 30c47fce216..834fb4d532f 100644 --- a/plugins/github/tasks/cicd_run_enricher.go +++ b/plugins/github/tasks/cicd_run_enricher.go @@ -34,6 +34,7 @@ var EnrichPipelinesMeta = core.SubTaskMeta{ func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { db := taskCtx.GetDal() data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId cursor, err := db.Cursor( dal.Select("head_sha, head_branch, status, conclusion, github_created_at, github_updated_at, run_attempt, run_started_at"), @@ -55,7 +56,7 @@ func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { if item.HeadSha != entity.Commit { entity.ConnectionId = data.Options.ConnectionId - entity.RepoId = item.RepoId + entity.RepoId = repoId entity.Commit = item.HeadSha entity.Branch = item.HeadBranch entity.StartedDate = item.GithubCreatedAt From fac101153499406838379aac39299dd976c85a61 Mon Sep 17 00:00:00 2001 From: abeizn Date: Fri, 5 Aug 2022 09:41:57 +0800 Subject: [PATCH 9/9] feat: github cicd enricher from run to pipeline --- plugins/github/tasks/cicd_run_enricher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/github/tasks/cicd_run_enricher.go b/plugins/github/tasks/cicd_run_enricher.go index 834fb4d532f..ee90e7d64a8 100644 --- a/plugins/github/tasks/cicd_run_enricher.go +++ b/plugins/github/tasks/cicd_run_enricher.go @@ -72,7 +72,7 @@ func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { if item.GithubCreatedAt.Before(*entity.StartedDate) { entity.StartedDate = item.GithubCreatedAt } - if item.GithubUpdatedAt.After(*entity.FinishedDate) { + if item.GithubUpdatedAt.After(*entity.FinishedDate) && item.Status == "completed" { entity.FinishedDate = item.GithubCreatedAt entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) }