From d32168a3e3576e82d98968ad7ca97c6848e85390 Mon Sep 17 00:00:00 2001 From: abeizn Date: Mon, 1 Aug 2022 17:35:55 +0800 Subject: [PATCH 1/2] feat: github add cicd feature --- plugins/github/impl/impl.go | 4 + plugins/github/models/job.go | 52 +++++++++ .../20220728_add_github_runs_table.go | 82 ++++++++++++++ .../20220729_add_github_jobs_table.go | 75 +++++++++++++ .../models/migrationscripts/register.go | 2 + plugins/github/models/run.go | 59 ++++++++++ plugins/github/tasks/cicd_job_collector.go | 103 ++++++++++++++++++ plugins/github/tasks/cicd_job_extractor.go | 88 +++++++++++++++ plugins/github/tasks/cicd_run_collector.go | 83 ++++++++++++++ plugins/github/tasks/cicd_run_extractor.go | 96 ++++++++++++++++ 10 files changed, 644 insertions(+) create mode 100644 plugins/github/models/job.go create mode 100644 plugins/github/models/migrationscripts/20220728_add_github_runs_table.go create mode 100644 plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go create mode 100644 plugins/github/models/run.go create mode 100644 plugins/github/tasks/cicd_job_collector.go create mode 100644 plugins/github/tasks/cicd_job_extractor.go create mode 100644 plugins/github/tasks/cicd_run_collector.go create mode 100644 plugins/github/tasks/cicd_run_extractor.go diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go index 56c0a16f7e3..d57345f2de9 100644 --- a/plugins/github/impl/impl.go +++ b/plugins/github/impl/impl.go @@ -78,6 +78,10 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta { tasks.ExtractAccountsMeta, tasks.CollectAccountOrgMeta, tasks.ExtractAccountOrgMeta, + tasks.CollectRunsMeta, + tasks.ExtractRunsMeta, + tasks.CollectJobsMeta, + tasks.ExtractJobsMeta, tasks.EnrichPullRequestIssuesMeta, tasks.ConvertRepoMeta, tasks.ConvertIssuesMeta, diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go new file mode 100644 index 00000000000..e319b33a229 --- /dev/null +++ b/plugins/github/models/job.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/models/common" + "gorm.io/datatypes" +) + +type GithubJob struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` + RunID int `json:"run_id"` + RunURL string `json:"run_url" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + Name string `json:"name" gorm:"type:varchar(255)"` + Steps datatypes.JSON `json:"steps"` + CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` + Labels datatypes.JSON `json:"labels"` + RunnerID int `json:"runner_id"` + RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` + RunnerGroupID int `json:"runner_group_id"` +} + +func (GithubJob) TableName() string { + return "_tool_github_jobs" +} diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go new file mode 100644 index 00000000000..3632e4307d2 --- /dev/null +++ b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "fmt" + "time" + + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "gorm.io/gorm" +) + +type GithubRun20220728 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` +} + +func (GithubRun20220728) TableName() string { + return "_tool_github_runs" +} + +type addGithubRunsTable struct{} + +func (u *addGithubRunsTable) Up(ctx context.Context, db *gorm.DB) error { + // create table + err := db.Migrator().CreateTable(GithubRun20220728{}) + if err != nil { + return fmt.Errorf("create table _tool_github_runs error") + } + return nil + +} + +func (*addGithubRunsTable) Version() uint64 { + return 20220728000001 +} + +func (*addGithubRunsTable) Name() string { + return "Github add github_runs table" +} diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go new file mode 100644 index 00000000000..31cfb210141 --- /dev/null +++ b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "fmt" + "time" + + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "gorm.io/datatypes" + "gorm.io/gorm" +) + +type GithubJob20220729 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` + RunID int `json:"run_id"` + RunURL string `json:"run_url" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + Name string `json:"name" gorm:"type:varchar(255)"` + Steps datatypes.JSON `json:"steps"` + CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` + Labels datatypes.JSON `json:"labels"` + RunnerID int `json:"runner_id"` + RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` + RunnerGroupID int `json:"runner_group_id"` +} + +func (GithubJob20220729) TableName() string { + return "_tool_github_jobs" +} + +type addGithubJobsTable struct{} + +func (u *addGithubJobsTable) Up(ctx context.Context, db *gorm.DB) error { + // create table + err := db.Migrator().CreateTable(GithubJob20220729{}) + if err != nil { + return fmt.Errorf("create table _tool_github_jobs error") + } + return nil + +} + +func (*addGithubJobsTable) Version() uint64 { + return 20220729000001 +} + +func (*addGithubJobsTable) Name() string { + return "Github add github_jobs table" +} diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go index c1365f7d4f2..4f899e789cc 100644 --- a/plugins/github/models/migrationscripts/register.go +++ b/plugins/github/models/migrationscripts/register.go @@ -25,5 +25,7 @@ import ( func All() []migration.Script { return []migration.Script{ new(addInitTables), + new(addGithubRunsTable), + new(addGithubJobsTable), } } diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go new file mode 100644 index 00000000000..fbc923ead35 --- /dev/null +++ b/plugins/github/models/run.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/models/common" +) + +type GithubRun struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` +} + +func (GithubRun) TableName() string { + return "_tool_github_runs" +} diff --git a/plugins/github/tasks/cicd_job_collector.go b/plugins/github/tasks/cicd_job_collector.go new file mode 100644 index 00000000000..d78034adf40 --- /dev/null +++ b/plugins/github/tasks/cicd_job_collector.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/helper" +) + +const RAW_JOB_TABLE = "github_api_jobs" + +var CollectJobsMeta = core.SubTaskMeta{ + Name: "collectJobs", + EntryPoint: CollectJobs, + EnabledByDefault: true, + Description: "Collect Jobs data from Github action api", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func CollectJobs(taskCtx core.SubTaskContext) error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubTaskData) + cursor, err := db.Cursor( + dal.Select("id"), + dal.From(models.GithubRun{}.TableName()), + ) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGithubRun{})) + if err != nil { + return err + } + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_JOB_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 100, + Input: iterator, + Incremental: false, + UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/actions/runs/{{ .Input.ID }}/jobs", + Query: func(reqData *helper.RequestData) (url.Values, error) { + query := url.Values{} + query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) + query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { + body := &GithubRawJobsResult{} + err := helper.UnmarshalResponse(res, body) + if err != nil { + return nil, err + } + return body.GithubWorkflowJobs, nil + }, + }) + + if err != nil { + return err + } + return collector.Execute() +} + +type SimpleGithubRun struct { + ID int64 +} + +type GithubRawJobsResult struct { + TotalCount int64 `json:"total_count"` + GithubWorkflowJobs []json.RawMessage `json:"jobs"` +} diff --git a/plugins/github/tasks/cicd_job_extractor.go b/plugins/github/tasks/cicd_job_extractor.go new file mode 100644 index 00000000000..9cbf58dbd1c --- /dev/null +++ b/plugins/github/tasks/cicd_job_extractor.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/helper" +) + +var ExtractJobsMeta = core.SubTaskMeta{ + Name: "extractJobs", + EntryPoint: ExtractJobs, + EnabledByDefault: true, + Description: "Extract raw run data into tool layer table github_jobs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func ExtractJobs(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*GithubTaskData) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_JOB_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, error) { + githubJob := &models.GithubJob{} + err := json.Unmarshal(row.Data, githubJob) + if err != nil { + return nil, err + } + + results := make([]interface{}, 0, 1) + githubJobResult := &models.GithubJob{ + ConnectionId: data.Options.ConnectionId, + ID: githubJob.ID, + RunID: githubJob.RunID, + RunURL: githubJob.RunURL, + NodeID: githubJob.NodeID, + HeadSha: githubJob.HeadSha, + URL: githubJob.URL, + HTMLURL: githubJob.HTMLURL, + Status: githubJob.Status, + Conclusion: githubJob.Conclusion, + StartedAt: githubJob.StartedAt, + CompletedAt: githubJob.CompletedAt, + Name: githubJob.Name, + Steps: githubJob.Steps, + CheckRunURL: githubJob.CheckRunURL, + Labels: githubJob.Labels, + RunnerID: githubJob.RunID, + RunnerName: githubJob.RunnerName, + RunnerGroupID: githubJob.RunnerGroupID, + } + results = append(results, githubJobResult) + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/plugins/github/tasks/cicd_run_collector.go b/plugins/github/tasks/cicd_run_collector.go new file mode 100644 index 00000000000..91068da35bd --- /dev/null +++ b/plugins/github/tasks/cicd_run_collector.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" +) + +const RAW_RUN_TABLE = "github_api_runs" + +var CollectRunsMeta = core.SubTaskMeta{ + Name: "collectRuns", + EntryPoint: CollectRuns, + EnabledByDefault: true, + Description: "Collect Runs data from Github action api", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func CollectRuns(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*GithubTaskData) + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_RUN_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 100, + Incremental: false, + UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/actions/runs", + Query: func(reqData *helper.RequestData) (url.Values, error) { + query := url.Values{} + query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) + query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { + body := &GithubRawRunsResult{} + err := helper.UnmarshalResponse(res, body) + if err != nil { + return nil, err + } + return body.GithubWorkflowRuns, nil + }, + }) + + if err != nil { + return err + } + return collector.Execute() +} + +type GithubRawRunsResult struct { + TotalCount int64 `json:"total_count"` + GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"` +} diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go new file mode 100644 index 00000000000..fa64c71f8cf --- /dev/null +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -0,0 +1,96 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/helper" +) + +var ExtractRunsMeta = core.SubTaskMeta{ + Name: "extractRuns", + EntryPoint: ExtractRuns, + EnabledByDefault: true, + Description: "Extract raw run data into tool layer table github_runs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func ExtractRuns(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*GithubTaskData) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Owner: data.Options.Owner, + Repo: data.Options.Repo, + }, + Table: RAW_RUN_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, error) { + githubRun := &models.GithubRun{} + err := json.Unmarshal(row.Data, githubRun) + if err != nil { + return nil, err + } + + results := make([]interface{}, 0, 1) + githubRunResult := &models.GithubRun{ + ConnectionId: data.Options.ConnectionId, + ID: githubRun.ID, + Name: githubRun.Name, + NodeID: githubRun.NodeID, + HeadBranch: githubRun.HeadBranch, + HeadSha: githubRun.HeadSha, + Path: githubRun.Path, + RunNumber: githubRun.RunNumber, + Event: githubRun.Event, + Status: githubRun.Status, + Conclusion: githubRun.Conclusion, + WorkflowID: githubRun.WorkflowID, + CheckSuiteID: githubRun.CheckSuiteID, + CheckSuiteNodeID: githubRun.CheckSuiteNodeID, + URL: githubRun.URL, + HTMLURL: githubRun.HTMLURL, + CreatedAt: githubRun.CreatedAt, + UpdatedAt: githubRun.UpdatedAt, + RunAttempt: githubRun.RunAttempt, + RunStartedAt: githubRun.RunStartedAt, + JobsURL: githubRun.JobsURL, + LogsURL: githubRun.LogsURL, + CheckSuiteURL: githubRun.CheckSuiteURL, + ArtifactsURL: githubRun.ArtifactsURL, + CancelURL: githubRun.CancelURL, + RerunURL: githubRun.RerunURL, + WorkflowURL: githubRun.WorkflowURL, + } + results = append(results, githubRunResult) + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} From 4b70dbff74463d4064c8a17e1e0fc15889f8b208 Mon Sep 17 00:00:00 2001 From: abeizn Date: Tue, 2 Aug 2022 16:33:24 +0800 Subject: [PATCH 2/2] feat: github add cicd feature and update fields --- plugins/github/models/job.go | 4 +- .../20220728_add_github_runs_table.go | 55 ++++++++++--------- .../20220729_add_github_jobs_table.go | 4 +- plugins/github/models/run.go | 55 ++++++++++--------- plugins/github/tasks/cicd_run_extractor.go | 6 +- 5 files changed, 64 insertions(+), 60 deletions(-) diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go index e319b33a229..15acaa6b258 100644 --- a/plugins/github/models/job.go +++ b/plugins/github/models/job.go @@ -36,8 +36,8 @@ type GithubJob struct { HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` Status string `json:"status" gorm:"type:varchar(255)"` Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - StartedAt time.Time `json:"started_at"` - CompletedAt time.Time `json:"completed_at"` + StartedAt *time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at"` Name string `json:"name" gorm:"type:varchar(255)"` Steps datatypes.JSON `json:"steps"` CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go index 3632e4307d2..bc5b630713d 100644 --- a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go +++ b/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go @@ -28,33 +28,34 @@ import ( type GithubRun20220728 struct { archived.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` - Name string `json:"name" gorm:"type:varchar(255)"` - NodeID string `json:"node_id" gorm:"type:varchar(255)"` - HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` - HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` - Path string `json:"path" gorm:"type:varchar(255)"` - RunNumber int `json:"run_number"` - Event string `json:"event" gorm:"type:varchar(255)"` - Status string `json:"status" gorm:"type:varchar(255)"` - Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - WorkflowID int `json:"workflow_id"` - CheckSuiteID int64 `json:"check_suite_id"` - CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` - URL string `json:"url" gorm:"type:varchar(255)"` - HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - RunAttempt int `json:"run_attempt"` - RunStartedAt time.Time `json:"run_started_at"` - JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` - LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` - CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` - ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` - CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` - RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` - WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + ConnectionId uint64 `gorm:"primaryKey"` + GithubId int `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + GithubCreatedAt *time.Time `json:"created_at"` + GithubUpdatedAt *time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt *time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` } func (GithubRun20220728) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go index 31cfb210141..981addd4b77 100644 --- a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go +++ b/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go @@ -39,8 +39,8 @@ type GithubJob20220729 struct { HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` Status string `json:"status" gorm:"type:varchar(255)"` Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - StartedAt time.Time `json:"started_at"` - CompletedAt time.Time `json:"completed_at"` + StartedAt *time.Time `json:"started_at"` + CompletedAt *time.Time `json:"completed_at"` Name string `json:"name" gorm:"type:varchar(255)"` Steps datatypes.JSON `json:"steps"` CheckRunURL string `json:"check_run_url" gorm:"type:varchar(255)"` diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go index fbc923ead35..d749f3a62d4 100644 --- a/plugins/github/models/run.go +++ b/plugins/github/models/run.go @@ -25,33 +25,34 @@ import ( type GithubRun struct { common.NoPKModel - ConnectionId uint64 `gorm:"primaryKey"` - ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` - Name string `json:"name" gorm:"type:varchar(255)"` - NodeID string `json:"node_id" gorm:"type:varchar(255)"` - HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` - HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` - Path string `json:"path" gorm:"type:varchar(255)"` - RunNumber int `json:"run_number"` - Event string `json:"event" gorm:"type:varchar(255)"` - Status string `json:"status" gorm:"type:varchar(255)"` - Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` - WorkflowID int `json:"workflow_id"` - CheckSuiteID int64 `json:"check_suite_id"` - CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` - URL string `json:"url" gorm:"type:varchar(255)"` - HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - RunAttempt int `json:"run_attempt"` - RunStartedAt time.Time `json:"run_started_at"` - JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` - LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` - CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` - ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` - CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` - RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` - WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + ConnectionId uint64 `gorm:"primaryKey"` + GithubId int `gorm:"primaryKey"` + ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` + Name string `json:"name" gorm:"type:varchar(255)"` + NodeID string `json:"node_id" gorm:"type:varchar(255)"` + HeadBranch string `json:"head_branch" gorm:"type:varchar(255)"` + HeadSha string `json:"head_sha" gorm:"type:varchar(255)"` + Path string `json:"path" gorm:"type:varchar(255)"` + RunNumber int `json:"run_number"` + Event string `json:"event" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + Conclusion string `json:"conclusion" gorm:"type:varchar(255)"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id" gorm:"type:varchar(255)"` + URL string `json:"url" gorm:"type:varchar(255)"` + HTMLURL string `json:"html_url" gorm:"type:varchar(255)"` + GithubCreatedAt *time.Time `json:"created_at"` + GithubUpdatedAt *time.Time `json:"updated_at"` + RunAttempt int `json:"run_attempt"` + RunStartedAt *time.Time `json:"run_started_at"` + JobsURL string `json:"jobs_url" gorm:"type:varchar(255)"` + LogsURL string `json:"logs_url" gorm:"type:varchar(255)"` + CheckSuiteURL string `json:"check_suite_url" gorm:"type:varchar(255)"` + ArtifactsURL string `json:"artifacts_url" gorm:"type:varchar(255)"` + CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` + RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` + WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` } func (GithubRun) TableName() string { diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go index fa64c71f8cf..87d07b728ea 100644 --- a/plugins/github/tasks/cicd_run_extractor.go +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -35,6 +35,7 @@ var ExtractRunsMeta = core.SubTaskMeta{ func ExtractRuns(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -56,6 +57,7 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubRunResult := &models.GithubRun{ ConnectionId: data.Options.ConnectionId, + GithubId: repoId, ID: githubRun.ID, Name: githubRun.Name, NodeID: githubRun.NodeID, @@ -71,8 +73,8 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { CheckSuiteNodeID: githubRun.CheckSuiteNodeID, URL: githubRun.URL, HTMLURL: githubRun.HTMLURL, - CreatedAt: githubRun.CreatedAt, - UpdatedAt: githubRun.UpdatedAt, + GithubCreatedAt: githubRun.GithubCreatedAt, + GithubUpdatedAt: githubRun.GithubUpdatedAt, RunAttempt: githubRun.RunAttempt, RunStartedAt: githubRun.RunStartedAt, JobsURL: githubRun.JobsURL,