feat: github add cicd feature

plugins/github/impl/impl.go

@@ -78,6 +78,10 @@ func (plugin Github) SubTaskMetas() []core.SubTaskMeta {
 		tasks.ExtractAccountsMeta,
 		tasks.CollectAccountOrgMeta,
 		tasks.ExtractAccountOrgMeta,
+		tasks.CollectRunsMeta,
+		tasks.ExtractRunsMeta,
+		tasks.CollectJobsMeta,
+		tasks.ExtractJobsMeta,
 		tasks.EnrichPullRequestIssuesMeta,
 		tasks.ConvertRepoMeta,
 		tasks.ConvertIssuesMeta,

plugins/github/models/job.go

@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+	"time"
+
+	"github.com/apache/incubator-devlake/models/common"
+	"gorm.io/datatypes"
+)
+
+type GithubJob struct {
+	common.NoPKModel
+	ConnectionId  uint64         `gorm:"primaryKey"`
+	ID            int            `json:"id" gorm:"primaryKey;autoIncrement:false"`
+	RunID         int            `json:"run_id"`
+	RunURL        string         `json:"run_url" gorm:"type:varchar(255)"`
+	NodeID        string         `json:"node_id" gorm:"type:varchar(255)"`
+	HeadSha       string         `json:"head_sha" gorm:"type:varchar(255)"`
+	URL           string         `json:"url" gorm:"type:varchar(255)"`
+	HTMLURL       string         `json:"html_url" gorm:"type:varchar(255)"`
+	Status        string         `json:"status" gorm:"type:varchar(255)"`
+	Conclusion    string         `json:"conclusion" gorm:"type:varchar(255)"`
+	StartedAt     *time.Time     `json:"started_at"`
+	CompletedAt   *time.Time     `json:"completed_at"`
+	Name          string         `json:"name" gorm:"type:varchar(255)"`
+	Steps         datatypes.JSON `json:"steps"`
+	CheckRunURL   string         `json:"check_run_url" gorm:"type:varchar(255)"`
+	Labels        datatypes.JSON `json:"labels"`
+	RunnerID      int            `json:"runner_id"`
+	RunnerName    string         `json:"runner_name" gorm:"type:varchar(255)"`
+	RunnerGroupID int            `json:"runner_group_id"`
+}
+
+func (GithubJob) TableName() string {
+	return "_tool_github_jobs"
+}

plugins/github/models/migrationscripts/20220728_add_github_runs_table.go

@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+	"gorm.io/gorm"
+)
+
+type GithubRun20220728 struct {
+	archived.NoPKModel
+	ConnectionId     uint64     `gorm:"primaryKey"`
+	GithubId         int        `gorm:"primaryKey"`
+	ID               int64      `json:"id" gorm:"primaryKey;autoIncrement:false"`
+	Name             string     `json:"name" gorm:"type:varchar(255)"`
+	NodeID           string     `json:"node_id" gorm:"type:varchar(255)"`
+	HeadBranch       string     `json:"head_branch" gorm:"type:varchar(255)"`
+	HeadSha          string     `json:"head_sha" gorm:"type:varchar(255)"`
+	Path             string     `json:"path" gorm:"type:varchar(255)"`
+	RunNumber        int        `json:"run_number"`
+	Event            string     `json:"event" gorm:"type:varchar(255)"`
+	Status           string     `json:"status" gorm:"type:varchar(255)"`
+	Conclusion       string     `json:"conclusion" gorm:"type:varchar(255)"`
+	WorkflowID       int        `json:"workflow_id"`
+	CheckSuiteID     int64      `json:"check_suite_id"`
+	CheckSuiteNodeID string     `json:"check_suite_node_id" gorm:"type:varchar(255)"`
+	URL              string     `json:"url" gorm:"type:varchar(255)"`
+	HTMLURL          string     `json:"html_url" gorm:"type:varchar(255)"`
+	GithubCreatedAt  *time.Time `json:"created_at"`
+	GithubUpdatedAt  *time.Time `json:"updated_at"`
+	RunAttempt       int        `json:"run_attempt"`
+	RunStartedAt     *time.Time `json:"run_started_at"`
+	JobsURL          string     `json:"jobs_url" gorm:"type:varchar(255)"`
+	LogsURL          string     `json:"logs_url" gorm:"type:varchar(255)"`
+	CheckSuiteURL    string     `json:"check_suite_url" gorm:"type:varchar(255)"`
+	ArtifactsURL     string     `json:"artifacts_url" gorm:"type:varchar(255)"`
+	CancelURL        string     `json:"cancel_url" gorm:"type:varchar(255)"`
+	RerunURL         string     `json:"rerun_url" gorm:"type:varchar(255)"`
+	WorkflowURL      string     `json:"workflow_url" gorm:"type:varchar(255)"`
+}
+
+func (GithubRun20220728) TableName() string {
+	return "_tool_github_runs"
+}
+
+type addGithubRunsTable struct{}
+
+func (u *addGithubRunsTable) Up(ctx context.Context, db *gorm.DB) error {
+	// create table
+	err := db.Migrator().CreateTable(GithubRun20220728{})
+	if err != nil {
+		return fmt.Errorf("create table _tool_github_runs error")
+	}
+	return nil
+
+}
+
+func (*addGithubRunsTable) Version() uint64 {
+	return 20220728000001
+}
+
+func (*addGithubRunsTable) Name() string {
+	return "Github add github_runs table"
+}

plugins/github/models/migrationscripts/20220729_add_github_jobs_table.go

@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/apache/incubator-devlake/models/migrationscripts/archived"
+	"gorm.io/datatypes"
+	"gorm.io/gorm"
+)
+
+type GithubJob20220729 struct {
+	archived.NoPKModel
+	ConnectionId  uint64         `gorm:"primaryKey"`
+	ID            int            `json:"id" gorm:"primaryKey;autoIncrement:false"`
+	RunID         int            `json:"run_id"`
+	RunURL        string         `json:"run_url" gorm:"type:varchar(255)"`
+	NodeID        string         `json:"node_id" gorm:"type:varchar(255)"`
+	HeadSha       string         `json:"head_sha" gorm:"type:varchar(255)"`
+	URL           string         `json:"url" gorm:"type:varchar(255)"`
+	HTMLURL       string         `json:"html_url" gorm:"type:varchar(255)"`
+	Status        string         `json:"status" gorm:"type:varchar(255)"`
+	Conclusion    string         `json:"conclusion" gorm:"type:varchar(255)"`
+	StartedAt     *time.Time     `json:"started_at"`
+	CompletedAt   *time.Time     `json:"completed_at"`
+	Name          string         `json:"name" gorm:"type:varchar(255)"`
+	Steps         datatypes.JSON `json:"steps"`
+	CheckRunURL   string         `json:"check_run_url" gorm:"type:varchar(255)"`
+	Labels        datatypes.JSON `json:"labels"`
+	RunnerID      int            `json:"runner_id"`
+	RunnerName    string         `json:"runner_name" gorm:"type:varchar(255)"`
+	RunnerGroupID int            `json:"runner_group_id"`
+}
+
+func (GithubJob20220729) TableName() string {
+	return "_tool_github_jobs"
+}
+
+type addGithubJobsTable struct{}
+
+func (u *addGithubJobsTable) Up(ctx context.Context, db *gorm.DB) error {
+	// create table
+	err := db.Migrator().CreateTable(GithubJob20220729{})
+	if err != nil {
+		return fmt.Errorf("create table _tool_github_jobs error")
+	}
+	return nil
+
+}
+
+func (*addGithubJobsTable) Version() uint64 {
+	return 20220729000001
+}
+
+func (*addGithubJobsTable) Name() string {
+	return "Github add github_jobs table"
+}

plugins/github/models/migrationscripts/register.go

@@ -25,5 +25,7 @@ import (
 func All() []migration.Script {
 	return []migration.Script{
 		new(addInitTables),
+		new(addGithubRunsTable),
+		new(addGithubJobsTable),
 	}
 }

plugins/github/models/run.go

@@ -0,0 +1,60 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+	"time"
+
+	"github.com/apache/incubator-devlake/models/common"
+)
+
+type GithubRun struct {
+	common.NoPKModel
+	ConnectionId     uint64     `gorm:"primaryKey"`
+	GithubId         int        `gorm:"primaryKey"`
+	ID               int64      `json:"id" gorm:"primaryKey;autoIncrement:false"`
+	Name             string     `json:"name" gorm:"type:varchar(255)"`
+	NodeID           string     `json:"node_id" gorm:"type:varchar(255)"`
+	HeadBranch       string     `json:"head_branch" gorm:"type:varchar(255)"`
+	HeadSha          string     `json:"head_sha" gorm:"type:varchar(255)"`
+	Path             string     `json:"path" gorm:"type:varchar(255)"`
+	RunNumber        int        `json:"run_number"`
+	Event            string     `json:"event" gorm:"type:varchar(255)"`
+	Status           string     `json:"status" gorm:"type:varchar(255)"`
+	Conclusion       string     `json:"conclusion" gorm:"type:varchar(255)"`
+	WorkflowID       int        `json:"workflow_id"`
+	CheckSuiteID     int64      `json:"check_suite_id"`
+	CheckSuiteNodeID string     `json:"check_suite_node_id" gorm:"type:varchar(255)"`
+	URL              string     `json:"url" gorm:"type:varchar(255)"`
+	HTMLURL          string     `json:"html_url" gorm:"type:varchar(255)"`
+	GithubCreatedAt  *time.Time `json:"created_at"`
+	GithubUpdatedAt  *time.Time `json:"updated_at"`
+	RunAttempt       int        `json:"run_attempt"`
+	RunStartedAt     *time.Time `json:"run_started_at"`
+	JobsURL          string     `json:"jobs_url" gorm:"type:varchar(255)"`
+	LogsURL          string     `json:"logs_url" gorm:"type:varchar(255)"`
+	CheckSuiteURL    string     `json:"check_suite_url" gorm:"type:varchar(255)"`
+	ArtifactsURL     string     `json:"artifacts_url" gorm:"type:varchar(255)"`
+	CancelURL        string     `json:"cancel_url" gorm:"type:varchar(255)"`
+	RerunURL         string     `json:"rerun_url" gorm:"type:varchar(255)"`
+	WorkflowURL      string     `json:"workflow_url" gorm:"type:varchar(255)"`
+}
+
+func (GithubRun) TableName() string {
+	return "_tool_github_runs"
+}

plugins/github/tasks/cicd_job_collector.go

@@ -0,0 +1,103 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"reflect"
+
+	"github.com/apache/incubator-devlake/plugins/core"
+	"github.com/apache/incubator-devlake/plugins/core/dal"
+	"github.com/apache/incubator-devlake/plugins/github/models"
+	"github.com/apache/incubator-devlake/plugins/helper"
+)
+
+const RAW_JOB_TABLE = "github_api_jobs"
+
+var CollectJobsMeta = core.SubTaskMeta{
+	Name:             "collectJobs",
+	EntryPoint:       CollectJobs,
+	EnabledByDefault: true,
+	Description:      "Collect Jobs data from Github action api",
+	DomainTypes:      []string{core.DOMAIN_TYPE_CICD},
+}
+
+func CollectJobs(taskCtx core.SubTaskContext) error {
+	db := taskCtx.GetDal()
+	data := taskCtx.GetData().(*GithubTaskData)
+	cursor, err := db.Cursor(
+		dal.Select("id"),
+		dal.From(models.GithubRun{}.TableName()),
+	)
+	if err != nil {
+		return err
+	}
+	iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGithubRun{}))
+	if err != nil {
+		return err
+	}
+
+	collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+			Ctx: taskCtx,
+			Params: GithubApiParams{
+				ConnectionId: data.Options.ConnectionId,
+				Owner:        data.Options.Owner,
+				Repo:         data.Options.Repo,
+			},
+			Table: RAW_JOB_TABLE,
+		},
+		ApiClient:   data.ApiClient,
+		PageSize:    100,
+		Input:       iterator,
+		Incremental: false,
+		UrlTemplate: "repos/{{ .Params.Owner }}/{{ .Params.Repo }}/actions/runs/{{ .Input.ID }}/jobs",
+		Query: func(reqData *helper.RequestData) (url.Values, error) {
+			query := url.Values{}
+			query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
+			query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
+			return query, nil
+		},
+		GetTotalPages: GetTotalPagesFromResponse,
+		ResponseParser: func(res *http.Response) ([]json.RawMessage, error) {
+			body := &GithubRawJobsResult{}
+			err := helper.UnmarshalResponse(res, body)
+			if err != nil {
+				return nil, err
+			}
+			return body.GithubWorkflowJobs, nil
+		},
+	})
+
+	if err != nil {
+		return err
+	}
+	return collector.Execute()
+}
+
+type SimpleGithubRun struct {
+	ID int64
+}
+
+type GithubRawJobsResult struct {
+	TotalCount         int64             `json:"total_count"`
+	GithubWorkflowJobs []json.RawMessage `json:"jobs"`
+}