refactor(gitextractor): move "clone a repo" from "PrePareTaskData" to…

… a new subtask called "CloneGitRepo" (#5992)

backend/plugins/gitextractor/impl/impl.go

@@ -18,18 +18,12 @@ limitations under the License.
 package impl
 
 import (
-	"context"
-	"fmt"
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/core/log"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"github.com/apache/incubator-devlake/plugins/gitextractor/models"
 	"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
-	"github.com/apache/incubator-devlake/plugins/gitextractor/store"
 	"github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
-	"strings"
 )
 
 var _ interface {
@@ -55,14 +49,15 @@ func (p GitExtractor) Name() string {
 // return all available subtasks, framework will run them for you in order
 func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta {
 	return []plugin.SubTaskMeta{
+		tasks.CloneGitRepoMeta,
 		tasks.CollectGitCommitMeta,
 		tasks.CollectGitBranchMeta,
 		tasks.CollectGitTagMeta,
 		tasks.CollectGitDiffLineMeta,
 	}
 }
 
-// based on task context and user input options, return data that shared among all subtasks
+// PrepareTaskData based on task context and user input options, return data that shared among all subtasks
 func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
 	var op tasks.GitExtractorOptions
 	if err := helper.Decode(options, &op, nil); err != nil {
@@ -71,12 +66,10 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st
 	if err := op.Valid(); err != nil {
 		return nil, err
 	}
-	storage := store.NewDatabase(taskCtx, op.RepoId)
-	repo, err := NewGitRepo(taskCtx.GetContext(), taskCtx.GetLogger(), storage, op)
-	if err != nil {
-		return nil, err
+	taskData := &tasks.GitExtractorTaskData{
+		Options: &op,
 	}
-	return repo, nil
+	return taskData, nil
 }
 
 func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
@@ -91,20 +84,3 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
 func (p GitExtractor) RootPkgPath() string {
 	return "github.com/apache/incubator-devlake/plugins/gitextractor"
 }
-
-// NewGitRepo create and return a new parser git repo
-func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store, op tasks.GitExtractorOptions) (*parser.GitRepo, errors.Error) {
-	var err errors.Error
-	var repo *parser.GitRepo
-	p := parser.NewGitRepoCreator(storage, logger)
-	if strings.HasPrefix(op.Url, "http") {
-		repo, err = p.CloneOverHTTP(ctx, op.RepoId, op.Url, op.User, op.Password, op.Proxy)
-	} else if url := strings.TrimPrefix(op.Url, "ssh://"); strings.HasPrefix(url, "git@") {
-		repo, err = p.CloneOverSSH(ctx, op.RepoId, url, op.PrivateKey, op.Passphrase)
-	} else if strings.HasPrefix(op.Url, "/") {
-		repo, err = p.LocalRepo(op.Url, op.RepoId)
-	} else {
-		return nil, errors.BadInput.New(fmt.Sprintf("unsupported url [%s]", op.Url))
-	}
-	return repo, err
-}

backend/plugins/gitextractor/main.go

@@ -78,7 +78,7 @@ func main() {
 		"git extractor",
 		nil,
 	)
-	repo, err := impl.NewGitRepo(subTaskCtx.GetContext(), logger, storage, tasks.GitExtractorOptions{
+	repo, err := tasks.NewGitRepo(subTaskCtx.GetContext(), logger, storage, &tasks.GitExtractorOptions{
 		RepoId:   *id,
 		Url:      *url,
 		User:     *user,

backend/plugins/gitextractor/tasks/clone.go

@@ -0,0 +1,72 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"context"
+	"fmt"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/log"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/plugins/gitextractor/models"
+	"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
+	"github.com/apache/incubator-devlake/plugins/gitextractor/store"
+	"strings"
+)
+
+var CloneGitRepoMeta = plugin.SubTaskMeta{
+	Name:             "cloneGitRepo",
+	EntryPoint:       CloneGitRepo,
+	EnabledByDefault: true,
+	Required:         true,
+	Description:      "clone a git repo, make it available to later tasks",
+	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE},
+}
+
+func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
+	taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
+	if !ok {
+		panic("git repo reference not found on context")
+	}
+	op := taskData.Options
+	storage := store.NewDatabase(subTaskCtx, op.RepoId)
+	repo, err := NewGitRepo(subTaskCtx.GetContext(), subTaskCtx.GetLogger(), storage, op)
+	if err != nil {
+		return err
+	}
+	taskData.GitRepo = repo
+	subTaskCtx.TaskContext().SetData(taskData)
+	return nil
+}
+
+// NewGitRepo create and return a new parser git repo
+func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store, op *GitExtractorOptions) (*parser.GitRepo, errors.Error) {
+	var err errors.Error
+	var repo *parser.GitRepo
+	p := parser.NewGitRepoCreator(storage, logger)
+	if strings.HasPrefix(op.Url, "http") {
+		repo, err = p.CloneOverHTTP(ctx, op.RepoId, op.Url, op.User, op.Password, op.Proxy)
+	} else if url := strings.TrimPrefix(op.Url, "ssh://"); strings.HasPrefix(url, "git@") {
+		repo, err = p.CloneOverSSH(ctx, op.RepoId, url, op.PrivateKey, op.Passphrase)
+	} else if strings.HasPrefix(op.Url, "/") {
+		repo, err = p.LocalRepo(op.Url, op.RepoId)
+	} else {
+		return nil, errors.BadInput.New(fmt.Sprintf("unsupported url [%s]", op.Url))
+	}
+	return repo, err
+}

backend/plugins/gitextractor/tasks/git_repo_collector.go

@@ -25,9 +25,14 @@ import (
 	"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
 )
 
+type GitExtractorTaskData struct {
+	Options *GitExtractorOptions
+	GitRepo *parser.GitRepo
+}
+
 type GitExtractorOptions struct {
 	RepoId     string `json:"repoId"`
-	Name       string `jsno:"name"`
+	Name       string `json:"name"`
 	Url        string `json:"url"`
 	User       string `json:"user"`
 	Password   string `json:"password"`
@@ -99,11 +104,14 @@ func CollectGitDiffLines(subTaskCtx plugin.SubTaskContext) errors.Error {
 }
 
 func getGitRepo(subTaskCtx plugin.SubTaskContext) *parser.GitRepo {
-	repo, ok := subTaskCtx.GetData().(*parser.GitRepo)
+	taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
 	if !ok {
 		panic("git repo reference not found on context")
 	}
-	return repo
+	if taskData.GitRepo == nil {
+		panic("git repo is empty, please check subtask: clone repo")
+	}
+	return taskData.GitRepo
 }
 
 var CollectGitCommitMeta = plugin.SubTaskMeta{
@@ -112,6 +120,7 @@ var CollectGitCommitMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "collect git commits into Domain Layer Tables",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CROSS},
+	Dependencies:     []*plugin.SubTaskMeta{&CloneGitRepoMeta},
 }
 
 var CollectGitBranchMeta = plugin.SubTaskMeta{
@@ -120,6 +129,7 @@ var CollectGitBranchMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "collect git branch into Domain Layer Tables",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE},
+	Dependencies:     []*plugin.SubTaskMeta{&CloneGitRepoMeta},
 }
 
 var CollectGitTagMeta = plugin.SubTaskMeta{
@@ -128,6 +138,7 @@ var CollectGitTagMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "collect git tag into Domain Layer Tables",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE},
+	Dependencies:     []*plugin.SubTaskMeta{&CloneGitRepoMeta},
 }
 
 var CollectGitDiffLineMeta = plugin.SubTaskMeta{
@@ -136,4 +147,5 @@ var CollectGitDiffLineMeta = plugin.SubTaskMeta{
 	EnabledByDefault: false,
 	Description:      "collect git commit diff line into Domain Layer Tables",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE},
+	Dependencies:     []*plugin.SubTaskMeta{&CloneGitRepoMeta},
 }