diff --git a/plugins/github/impl/impl.go b/plugins/github/impl/impl.go index d57345f2de9..bdb992234a2 100644 --- a/plugins/github/impl/impl.go +++ b/plugins/github/impl/impl.go @@ -82,6 +82,7 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta { tasks.ExtractRunsMeta, tasks.CollectJobsMeta, tasks.ExtractJobsMeta, + tasks.EnrichPipelinesMeta, tasks.EnrichPullRequestIssuesMeta, tasks.ConvertRepoMeta, tasks.ConvertIssuesMeta, diff --git a/plugins/github/models/job.go b/plugins/github/models/job.go index 15acaa6b258..31a95430b75 100644 --- a/plugins/github/models/job.go +++ b/plugins/github/models/job.go @@ -27,6 +27,7 @@ import ( type GithubJob struct { common.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` RunID int `json:"run_id"` RunURL string `json:"run_url" gorm:"type:varchar(255)"` @@ -45,6 +46,7 @@ type GithubJob struct { RunnerID int `json:"runner_id"` RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` RunnerGroupID int `json:"runner_group_id"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubJob) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go b/plugins/github/models/migrationscripts/20220728_add_runs_table.go similarity index 96% rename from plugins/github/models/migrationscripts/20220728_add_github_runs_table.go rename to plugins/github/models/migrationscripts/20220728_add_runs_table.go index bc5b630713d..aee05c57eae 100644 --- a/plugins/github/models/migrationscripts/20220728_add_github_runs_table.go +++ b/plugins/github/models/migrationscripts/20220728_add_runs_table.go @@ -29,7 +29,7 @@ import ( type GithubRun20220728 struct { archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` - GithubId int `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` Name string `json:"name" gorm:"type:varchar(255)"` NodeID string `json:"node_id" gorm:"type:varchar(255)"` @@ -56,6 +56,7 @@ type GithubRun20220728 struct { CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubRun20220728) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go similarity index 95% rename from plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go rename to plugins/github/models/migrationscripts/20220729_add_jobs_table.go index 981addd4b77..9631dcebcd9 100644 --- a/plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go +++ b/plugins/github/models/migrationscripts/20220729_add_jobs_table.go @@ -30,6 +30,7 @@ import ( type GithubJob20220729 struct { archived.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int `json:"id" gorm:"primaryKey;autoIncrement:false"` RunID int `json:"run_id"` RunURL string `json:"run_url" gorm:"type:varchar(255)"` @@ -48,6 +49,7 @@ type GithubJob20220729 struct { RunnerID int `json:"runner_id"` RunnerName string `json:"runner_name" gorm:"type:varchar(255)"` RunnerGroupID int `json:"runner_group_id"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubJob20220729) TableName() string { diff --git a/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go new file mode 100644 index 00000000000..84e1c87aa56 --- /dev/null +++ b/plugins/github/models/migrationscripts/20220802_add_pipeline_table.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "fmt" + "time" + + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "gorm.io/gorm" +) + +type GithubPipeline20220803 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` + Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"` + Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"` + StartedDate *time.Time `json:"started_time"` + FinishedDate *time.Time `json:"finished_time"` + Duration float64 `json:"duration"` + Status string `json:"status" gorm:"type:varchar(255)"` + Result string `json:"results" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` +} + +func (GithubPipeline20220803) TableName() string { + return "_tool_github_pipelines" +} + +type addGithubPipelineTable struct{} + +func (u *addGithubPipelineTable) Up(ctx context.Context, db *gorm.DB) error { + // create table + err := db.Migrator().CreateTable(GithubPipeline20220803{}) + if err != nil { + return fmt.Errorf("create table _tool_github_pipelines error") + } + return nil + +} + +func (*addGithubPipelineTable) Version() uint64 { + return 20220803000001 +} + +func (*addGithubPipelineTable) Name() string { + return "Github add github_pipelines table" +} diff --git a/plugins/github/models/migrationscripts/register.go b/plugins/github/models/migrationscripts/register.go index 4f899e789cc..439d77d48b6 100644 --- a/plugins/github/models/migrationscripts/register.go +++ b/plugins/github/models/migrationscripts/register.go @@ -27,5 +27,6 @@ func All() []migration.Script { new(addInitTables), new(addGithubRunsTable), new(addGithubJobsTable), + new(addGithubPipelineTable), } } diff --git a/plugins/github/models/pipeline.go b/plugins/github/models/pipeline.go new file mode 100644 index 00000000000..0b18e6b5022 --- /dev/null +++ b/plugins/github/models/pipeline.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/models/common" +) + +type GithubPipeline struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` + Branch string `json:"branch" gorm:"primaryKey;type:varchar(255)"` + Commit string `json:"commit" gorm:"primaryKey;type:varchar(255)"` + StartedDate *time.Time `json:"started_time"` + FinishedDate *time.Time `json:"finished_time"` + Duration float64 `json:"duration"` + Status string `json:"status" gorm:"type:varchar(255)"` + Result string `json:"results" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` +} + +func (GithubPipeline) TableName() string { + return "_tool_github_pipelines" +} diff --git a/plugins/github/models/run.go b/plugins/github/models/run.go index d749f3a62d4..bd2ec06c5fe 100644 --- a/plugins/github/models/run.go +++ b/plugins/github/models/run.go @@ -26,7 +26,7 @@ import ( type GithubRun struct { common.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` - GithubId int `gorm:"primaryKey"` + RepoId int `gorm:"primaryKey"` ID int64 `json:"id" gorm:"primaryKey;autoIncrement:false"` Name string `json:"name" gorm:"type:varchar(255)"` NodeID string `json:"node_id" gorm:"type:varchar(255)"` @@ -53,6 +53,7 @@ type GithubRun struct { CancelURL string `json:"cancel_url" gorm:"type:varchar(255)"` RerunURL string `json:"rerun_url" gorm:"type:varchar(255)"` WorkflowURL string `json:"workflow_url" gorm:"type:varchar(255)"` + Type string `json:"type" gorm:"type:varchar(255)"` } func (GithubRun) TableName() string { diff --git a/plugins/github/tasks/cicd_job_extractor.go b/plugins/github/tasks/cicd_job_extractor.go index 9cbf58dbd1c..b8e8fcae1d6 100644 --- a/plugins/github/tasks/cicd_job_extractor.go +++ b/plugins/github/tasks/cicd_job_extractor.go @@ -35,6 +35,7 @@ var ExtractJobsMeta = core.SubTaskMeta{ func ExtractJobs(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -56,6 +57,7 @@ func ExtractJobs(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubJobResult := &models.GithubJob{ ConnectionId: data.Options.ConnectionId, + RepoId: repoId, ID: githubJob.ID, RunID: githubJob.RunID, RunURL: githubJob.RunURL, @@ -74,6 +76,8 @@ func ExtractJobs(taskCtx core.SubTaskContext) error { RunnerID: githubJob.RunID, RunnerName: githubJob.RunnerName, RunnerGroupID: githubJob.RunnerGroupID, + // TODO + //Type: "CI/CD", } results = append(results, githubJobResult) return results, nil diff --git a/plugins/github/tasks/cicd_run_enricher.go b/plugins/github/tasks/cicd_run_enricher.go new file mode 100644 index 00000000000..ee90e7d64a8 --- /dev/null +++ b/plugins/github/tasks/cicd_run_enricher.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" + githubModels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var EnrichPipelinesMeta = core.SubTaskMeta{ + Name: "enrichPipelines", + EntryPoint: EnrichPipelines, + EnabledByDefault: true, + Description: "Create tool layer table github_pipelines from github_runs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func EnrichPipelines(taskCtx core.SubTaskContext) (err error) { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubTaskData) + repoId := data.Repo.GithubId + + cursor, err := db.Cursor( + dal.Select("head_sha, head_branch, status, conclusion, github_created_at, github_updated_at, run_attempt, run_started_at"), + dal.From(&githubModels.GithubRun{}), + dal.Orderby("head_sha, github_created_at"), + ) + if err != nil { + return err + } + defer cursor.Close() + + for cursor.Next() { + entity := &githubModels.GithubPipeline{} + var item githubModels.GithubRun + err = db.Fetch(cursor, &item) + if err != nil { + return err + } + + if item.HeadSha != entity.Commit { + entity.ConnectionId = data.Options.ConnectionId + entity.RepoId = repoId + entity.Commit = item.HeadSha + entity.Branch = item.HeadBranch + entity.StartedDate = item.GithubCreatedAt + entity.FinishedDate = item.GithubUpdatedAt + entity.Status = item.Status + if entity.Status == "completed" { + entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) + } + entity.Result = item.Conclusion + // TODO + entity.Type = "CI/CD" + } else { + if item.GithubCreatedAt.Before(*entity.StartedDate) { + entity.StartedDate = item.GithubCreatedAt + } + if item.GithubUpdatedAt.After(*entity.FinishedDate) && item.Status == "completed" { + entity.FinishedDate = item.GithubCreatedAt + entity.Duration = float64(item.GithubUpdatedAt.Sub(*item.GithubCreatedAt).Seconds()) + } + if item.Status != "completed" { + entity.Status = item.Status + } + if item.Conclusion != "success" { + entity.Result = item.Conclusion + } + + } + err := db.CreateOrUpdate(entity) + if err != nil { + return err + } + } + + return err + +} diff --git a/plugins/github/tasks/cicd_run_extractor.go b/plugins/github/tasks/cicd_run_extractor.go index 87d07b728ea..703bc652915 100644 --- a/plugins/github/tasks/cicd_run_extractor.go +++ b/plugins/github/tasks/cicd_run_extractor.go @@ -57,8 +57,8 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) githubRunResult := &models.GithubRun{ ConnectionId: data.Options.ConnectionId, - GithubId: repoId, ID: githubRun.ID, + RepoId: repoId, Name: githubRun.Name, NodeID: githubRun.NodeID, HeadBranch: githubRun.HeadBranch, @@ -84,6 +84,8 @@ func ExtractRuns(taskCtx core.SubTaskContext) error { CancelURL: githubRun.CancelURL, RerunURL: githubRun.RerunURL, WorkflowURL: githubRun.WorkflowURL, + // TODO + //Type: "CI/CD", } results = append(results, githubRunResult) return results, nil