Issues/2038 (apache#2039)

* add some log and ignore case of log_level

* add some log and ignore case of log_level

Co-authored-by: linyh <yanghui@meri.co>

logger/log.go → logger/init.go

@@ -21,10 +21,10 @@ import (
 	"fmt"
 	"github.com/apache/incubator-devlake/config"
 	"github.com/apache/incubator-devlake/plugins/core"
-	"github.com/apache/incubator-devlake/plugins/helper"
 	"github.com/sirupsen/logrus"
 	prefixed "github.com/x-cray/logrus-prefixed-formatter"
 	"os"
+	"strings"
 )
 
 var inner *logrus.Logger
@@ -33,14 +33,14 @@ var Global core.Logger
 func init() {
 	inner = logrus.New()
 	logLevel := logrus.InfoLevel
-	switch config.GetConfig().GetString("LOGGING_LEVEL") {
-	case "Debug":
+	switch strings.ToLower(config.GetConfig().GetString("LOGGING_LEVEL")) {
+	case "debug":
 		logLevel = logrus.DebugLevel
-	case "Info":
+	case "info":
 		logLevel = logrus.InfoLevel
-	case "Warn":
+	case "warn":
 		logLevel = logrus.WarnLevel
-	case "Error":
+	case "error":
 		logLevel = logrus.ErrorLevel
 	}
 	inner.SetLevel(logLevel)
@@ -53,5 +53,5 @@ func init() {
 		inner.Info(fmt.Sprintf("failed to create dir logs: %s", err))
 	}
 	loggerPool := make(map[string]*logrus.Logger)
-	Global = helper.NewDefaultLogger(inner, "", loggerPool)
+	Global = NewDefaultLogger(inner, "", loggerPool)
 }

plugins/helper/default_logger.go → logger/logger.go

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package helper
+package logger
 
 import (
 	"fmt"

plugins/gitextractor/main.go

@@ -42,6 +42,7 @@ func main() {
 	output := flag.String("output", "", "-output")
 	db := flag.String("db", "", "-db")
 	flag.Parse()
+	log := logger.Global.Nested("git extractor")
 	var storage models.Store
 	var err error
 	if *url == "" {
@@ -60,13 +61,12 @@ func main() {
 		if err != nil {
 			panic(err)
 		}
-		storage = store.NewDatabase(database)
+		storage = store.NewDatabase(database, log)
 	} else {
 		panic("either specify `-output` or `-db` argument as destination")
 	}
 	defer storage.Close()
 	ctx := context.Background()
-	log := logger.Global.Nested("git extractor")
 	subTaskCtx := helper.NewStandaloneSubTaskContext(
 		config.GetConfig(),
 		log,

plugins/gitextractor/store/database.go

@@ -19,6 +19,7 @@ package store
 
 import (
 	"github.com/apache/incubator-devlake/models/domainlayer/code"
+	"github.com/apache/incubator-devlake/plugins/core"
 	"github.com/apache/incubator-devlake/plugins/helper"
 	"gorm.io/gorm"
 	"reflect"
@@ -29,40 +30,42 @@ const BathSize = 100
 type Database struct {
 	//db     *gorm.DB
 	driver *helper.BatchSaveDivider
+	log    core.Logger
 }
 
-func NewDatabase(db *gorm.DB) *Database {
+func NewDatabase(db *gorm.DB, log core.Logger) *Database {
 	database := new(Database)
 	database.driver = helper.NewBatchSaveDivider(db, BathSize)
+	database.log = log
 	return database
 }
 
 func (d *Database) RepoCommits(repoCommit *code.RepoCommit) error {
-	batch, err := d.driver.ForType(reflect.TypeOf(repoCommit))
+	batch, err := d.driver.ForType(reflect.TypeOf(repoCommit), d.log)
 	if err != nil {
 		return err
 	}
 	return batch.Add(repoCommit)
 }
 
 func (d *Database) Commits(commit *code.Commit) error {
-	batch, err := d.driver.ForType(reflect.TypeOf(commit))
+	batch, err := d.driver.ForType(reflect.TypeOf(commit), d.log)
 	if err != nil {
 		return err
 	}
 	return batch.Add(commit)
 }
 
 func (d *Database) Refs(ref *code.Ref) error {
-	batch, err := d.driver.ForType(reflect.TypeOf(ref))
+	batch, err := d.driver.ForType(reflect.TypeOf(ref), d.log)
 	if err != nil {
 		return err
 	}
 	return batch.Add(ref)
 }
 
 func (d *Database) CommitFiles(file *code.CommitFile) error {
-	batch, err := d.driver.ForType(reflect.TypeOf(file))
+	batch, err := d.driver.ForType(reflect.TypeOf(file), d.log)
 	if err != nil {
 		return err
 	}
@@ -73,7 +76,7 @@ func (d *Database) CommitParents(pp []*code.CommitParent) error {
 	if len(pp) == 0 {
 		return nil
 	}
-	batch, err := d.driver.ForType(reflect.TypeOf(pp[0]))
+	batch, err := d.driver.ForType(reflect.TypeOf(pp[0]), d.log)
 	if err != nil {
 		return err
 	}

plugins/gitextractor/tasks/git_repo_collector.go

@@ -56,7 +56,7 @@ func (o GitExtractorOptions) Valid() error {
 func CollectGitRepo(subTaskCtx core.SubTaskContext) error {
 	var err error
 	db := subTaskCtx.GetDb()
-	storage := store.NewDatabase(db)
+	storage := store.NewDatabase(db, subTaskCtx.GetLogger())
 	op := subTaskCtx.GetData().(GitExtractorOptions)
 	p := parser.NewLibGit2(storage, subTaskCtx)
 	if strings.HasPrefix(op.Url, "http") {

plugins/helper/api_async_client_test.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/agiledragon/gomonkey/v2"
+	"github.com/apache/incubator-devlake/logger"
 	"github.com/apache/incubator-devlake/plugins/core"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
@@ -103,7 +104,7 @@ func CreateTestAsyncApiClientWithRateLimitAndCtx(t *testing.T, rateLimiter *ApiR
 		return &DefaultTaskContext{
 			&defaultExecContext{
 				cfg:      cfg,
-				logger:   &DefaultLogger{},
+				logger:   &logger.DefaultLogger{},
 				db:       db,
 				ctx:      ctx,
 				name:     "Test",
@@ -146,7 +147,7 @@ func TestWaitAsync_EmptyWork(t *testing.T) {
 func TestWaitAsync_WithWork(t *testing.T) {
 	asyncApiClient, _ := CreateTestAsyncApiClient(t)
 
-	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) {
 	})
 	defer gm_info.Reset()
 
@@ -187,7 +188,7 @@ func TestWaitAsync_WithWork(t *testing.T) {
 func TestWaitAsync_MutiWork(t *testing.T) {
 	asyncApiClient, _ := CreateTestAsyncApiClient(t)
 
-	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) {
 	})
 	defer gm_info.Reset()
 
@@ -231,7 +232,7 @@ func TestWaitAsync_MutiWork(t *testing.T) {
 // go test -gcflags=all=-l -run ^TestDoAsync_OnceSuceess
 func TestDoAsync_OnceSuceess(t *testing.T) {
 	asyncApiClient, _ := CreateTestAsyncApiClient(t)
-	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) {
 	})
 	defer gm_info.Reset()
 
@@ -261,7 +262,7 @@ func TestDoAsync_OnceSuceess(t *testing.T) {
 // go test -gcflags=all=-l -run ^TestDoAsync_TryAndFail
 func TestDoAsync_TryAndFail(t *testing.T) {
 	asyncApiClient, _ := CreateTestAsyncApiClient(t)
-	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) {
 	})
 	defer gm_info.Reset()
 
@@ -294,7 +295,7 @@ func TestDoAsync_TryAndFail(t *testing.T) {
 // go test -gcflags=all=-l -run ^TestDoAsync_TryAndSuceess
 func TestDoAsync_TryAndSuceess(t *testing.T) {
 	asyncApiClient, _ := CreateTestAsyncApiClient(t)
-	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+	gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) {
 	})
 	defer gm_info.Reset()
 

plugins/helper/api_client.go

@@ -217,7 +217,7 @@ func (apiClient *ApiClient) Do(
 		return nil, err
 	}
 
-	// after recieve
+	// after receive
 	if apiClient.afterReponse != nil {
 		err = apiClient.afterReponse(res)
 		if err != nil {

plugins/helper/api_collector_test.go

@@ -33,6 +33,7 @@ import (
 	"time"
 
 	"github.com/agiledragon/gomonkey/v2"
+	"github.com/apache/incubator-devlake/logger"
 	"github.com/apache/incubator-devlake/models/common"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -176,45 +177,41 @@ func MockDB(t *testing.T) {
 	gt = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Table", func(db *gorm.DB, name string, args ...interface{}) *gorm.DB {
 		assert.Equal(t, name, TestTableData.TableName())
 		return db
-	},
-	)
+	})
 
 	gc = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Create", func(db *gorm.DB, value interface{}) *gorm.DB {
 		assert.Equal(t, TestTableData, value.(*TestTable))
 		return db
-	},
-	)
+	})
 
 	gd = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Delete", func(db *gorm.DB, value interface{}, conds ...interface{}) (tx *gorm.DB) {
 		return db
-	},
-	)
+	})
 
 	ga = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "AutoMigrate", func(db *gorm.DB, dst ...interface{}) error {
 		return nil
-	},
-	)
+	})
 
 	god = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Order", func(db *gorm.DB, value interface{}) (tx *gorm.DB) {
 		return db
-	},
-	)
+	})
 
 	gw = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Where", func(db *gorm.DB, query interface{}, args ...interface{}) (tx *gorm.DB) {
 		return db
-	},
-	)
+	})
+
+	gr = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Count", func(db *gorm.DB, count *int64) (tx *gorm.DB) {
+		return db
+	})
 
 	gr = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Rows", func(db *gorm.DB) (*sql.Rows, error) {
 		return &sql.Rows{}, nil
-	},
-	)
+	})
 
 	gs = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "ScanRows", func(db *gorm.DB, rows *sql.Rows, dest interface{}) error {
 		dest = TestRawMessage
 		return nil
-	},
-	)
+	})
 
 }
 
@@ -268,7 +265,7 @@ func CreateTestApiCollector() (*ApiCollector, error) {
 	return NewApiCollector(ApiCollectorArgs{
 		RawDataSubTaskArgs: RawDataSubTaskArgs{
 			Ctx: &DefaultSubTaskContext{
-				defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), db, ctx, "Test", nil, nil),
+				defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), db, ctx, "Test", nil, nil),
 			},
 			Table: TestTable{}.TableName(),
 			Params: &TestParam{
@@ -1096,7 +1093,7 @@ func TestExecute_Total(t *testing.T) {
 	})
 	defer gs.Reset()
 
-	gin := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) {
+	gin := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) {
 	})
 	defer gin.Reset()
 

plugins/helper/api_extractor.go

@@ -64,10 +64,18 @@ func NewApiExtractor(args ApiExtractorArgs) (*ApiExtractor, error) {
 func (extractor *ApiExtractor) Execute() error {
 	// load data from database
 	db := extractor.args.Ctx.GetDb()
-	cursor, err := db.Table(extractor.table).Order("id ASC").Where("params = ?", extractor.params).Rows()
+	log := extractor.args.Ctx.GetLogger()
+	count := int64(0)
+	cursor, err := db.
+		Table(extractor.table).
+		Order("id ASC").
+		Where("params = ?", extractor.params).
+		Count(&count).
+		Rows()
 	if err != nil {
 		return err
 	}
+	log.Info("get data from %s where params=%s and got %d", extractor.table, extractor.params, count)
 	defer cursor.Close()
 	row := &RawData{}
 
@@ -113,7 +121,7 @@ func (extractor *ApiExtractor) Execute() error {
 
 		for _, result := range results {
 			// get the batch operator for the specific type
-			batch, err := divider.ForType(reflect.TypeOf(result))
+			batch, err := divider.ForType(reflect.TypeOf(result), log)
 			if err != nil {
 				return err
 			}

plugins/helper/api_extractor_test.go

@@ -21,6 +21,8 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"github.com/apache/incubator-devlake/logger"
+	"github.com/apache/incubator-devlake/plugins/core"
 	"reflect"
 	"testing"
 	"time"
@@ -48,7 +50,7 @@ func CreateTestApiExtractor(t *testing.T) (*ApiExtractor, error) {
 	return NewApiExtractor(ApiExtractorArgs{
 		RawDataSubTaskArgs: RawDataSubTaskArgs{
 			Ctx: &DefaultSubTaskContext{
-				defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), &gorm.DB{}, ctx, "Test", nil, nil),
+				defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), &gorm.DB{}, ctx, "Test", nil, nil),
 			},
 			Table: TestTable{}.TableName(),
 			Params: &TestParam{
@@ -107,7 +109,7 @@ func TestApiExtractorExecute(t *testing.T) {
 	)
 
 	fortypeTimes := 0
-	gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) {
+	gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger) (*BatchSave, error) {
 		fortypeTimes++
 		assert.Equal(t, rowType, reflect.TypeOf(TestTableData))
 		err := d.onNewBatchSave(rowType)
@@ -168,7 +170,7 @@ func TestApiExtractorExecute_Cancel(t *testing.T) {
 	)
 
 	fortypeTimes := 0
-	gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) {
+	gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger) (*BatchSave, error) {
 		fortypeTimes++
 		assert.Equal(t, rowType, reflect.TypeOf(TestTableData))
 		err := d.onNewBatchSave(rowType)