From f2dd0d33103c49acc591e879cedf2e2e5382bb0e Mon Sep 17 00:00:00 2001 From: likyh Date: Mon, 6 Jun 2022 21:53:16 +0800 Subject: [PATCH] Issues/2038 (#2039) * add some log and ignore case of log_level * add some log and ignore case of log_level Co-authored-by: linyh --- logger/{log.go => init.go} | 14 ++++---- .../default_logger.go => logger/logger.go | 2 +- plugins/gitextractor/main.go | 4 +-- plugins/gitextractor/store/database.go | 15 +++++---- .../gitextractor/tasks/git_repo_collector.go | 2 +- plugins/helper/api_async_client_test.go | 13 ++++---- plugins/helper/api_client.go | 2 +- plugins/helper/api_collector_test.go | 33 +++++++++---------- plugins/helper/api_extractor.go | 12 +++++-- plugins/helper/api_extractor_test.go | 8 +++-- plugins/helper/batch_save.go | 20 +++++++---- plugins/helper/batch_save_divider.go | 5 +-- plugins/helper/batch_save_divider_test.go | 7 ++-- plugins/helper/batch_save_test.go | 3 +- plugins/helper/data_convertor.go | 2 +- plugins/helper/data_convertor_test.go | 8 +++-- plugins/helper/default_task_context.go | 13 ++++++-- plugins/helper/default_task_logger.go | 3 +- runner/db.go | 8 ++--- 19 files changed, 104 insertions(+), 70 deletions(-) rename logger/{log.go => init.go} (87%) rename plugins/helper/default_logger.go => logger/logger.go (99%) diff --git a/logger/log.go b/logger/init.go similarity index 87% rename from logger/log.go rename to logger/init.go index 7fc466f8695..a5ff6616910 100644 --- a/logger/log.go +++ b/logger/init.go @@ -21,10 +21,10 @@ import ( "fmt" "github.com/apache/incubator-devlake/config" "github.com/apache/incubator-devlake/plugins/core" - "github.com/apache/incubator-devlake/plugins/helper" "github.com/sirupsen/logrus" prefixed "github.com/x-cray/logrus-prefixed-formatter" "os" + "strings" ) var inner *logrus.Logger @@ -33,14 +33,14 @@ var Global core.Logger func init() { inner = logrus.New() logLevel := logrus.InfoLevel - switch config.GetConfig().GetString("LOGGING_LEVEL") { - case "Debug": + switch strings.ToLower(config.GetConfig().GetString("LOGGING_LEVEL")) { + case "debug": logLevel = logrus.DebugLevel - case "Info": + case "info": logLevel = logrus.InfoLevel - case "Warn": + case "warn": logLevel = logrus.WarnLevel - case "Error": + case "error": logLevel = logrus.ErrorLevel } inner.SetLevel(logLevel) @@ -53,5 +53,5 @@ func init() { inner.Info(fmt.Sprintf("failed to create dir logs: %s", err)) } loggerPool := make(map[string]*logrus.Logger) - Global = helper.NewDefaultLogger(inner, "", loggerPool) + Global = NewDefaultLogger(inner, "", loggerPool) } diff --git a/plugins/helper/default_logger.go b/logger/logger.go similarity index 99% rename from plugins/helper/default_logger.go rename to logger/logger.go index c6afb66f5bf..110affe1d98 100644 --- a/plugins/helper/default_logger.go +++ b/logger/logger.go @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helper +package logger import ( "fmt" diff --git a/plugins/gitextractor/main.go b/plugins/gitextractor/main.go index 7da1f30a750..300ae5b0df6 100644 --- a/plugins/gitextractor/main.go +++ b/plugins/gitextractor/main.go @@ -42,6 +42,7 @@ func main() { output := flag.String("output", "", "-output") db := flag.String("db", "", "-db") flag.Parse() + log := logger.Global.Nested("git extractor") var storage models.Store var err error if *url == "" { @@ -60,13 +61,12 @@ func main() { if err != nil { panic(err) } - storage = store.NewDatabase(database) + storage = store.NewDatabase(database, log) } else { panic("either specify `-output` or `-db` argument as destination") } defer storage.Close() ctx := context.Background() - log := logger.Global.Nested("git extractor") subTaskCtx := helper.NewStandaloneSubTaskContext( config.GetConfig(), log, diff --git a/plugins/gitextractor/store/database.go b/plugins/gitextractor/store/database.go index 7c872c8d867..4e264fc6aea 100644 --- a/plugins/gitextractor/store/database.go +++ b/plugins/gitextractor/store/database.go @@ -19,6 +19,7 @@ package store import ( "github.com/apache/incubator-devlake/models/domainlayer/code" + "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/helper" "gorm.io/gorm" "reflect" @@ -29,16 +30,18 @@ const BathSize = 100 type Database struct { //db *gorm.DB driver *helper.BatchSaveDivider + log core.Logger } -func NewDatabase(db *gorm.DB) *Database { +func NewDatabase(db *gorm.DB, log core.Logger) *Database { database := new(Database) database.driver = helper.NewBatchSaveDivider(db, BathSize) + database.log = log return database } func (d *Database) RepoCommits(repoCommit *code.RepoCommit) error { - batch, err := d.driver.ForType(reflect.TypeOf(repoCommit)) + batch, err := d.driver.ForType(reflect.TypeOf(repoCommit), d.log) if err != nil { return err } @@ -46,7 +49,7 @@ func (d *Database) RepoCommits(repoCommit *code.RepoCommit) error { } func (d *Database) Commits(commit *code.Commit) error { - batch, err := d.driver.ForType(reflect.TypeOf(commit)) + batch, err := d.driver.ForType(reflect.TypeOf(commit), d.log) if err != nil { return err } @@ -54,7 +57,7 @@ func (d *Database) Commits(commit *code.Commit) error { } func (d *Database) Refs(ref *code.Ref) error { - batch, err := d.driver.ForType(reflect.TypeOf(ref)) + batch, err := d.driver.ForType(reflect.TypeOf(ref), d.log) if err != nil { return err } @@ -62,7 +65,7 @@ func (d *Database) Refs(ref *code.Ref) error { } func (d *Database) CommitFiles(file *code.CommitFile) error { - batch, err := d.driver.ForType(reflect.TypeOf(file)) + batch, err := d.driver.ForType(reflect.TypeOf(file), d.log) if err != nil { return err } @@ -73,7 +76,7 @@ func (d *Database) CommitParents(pp []*code.CommitParent) error { if len(pp) == 0 { return nil } - batch, err := d.driver.ForType(reflect.TypeOf(pp[0])) + batch, err := d.driver.ForType(reflect.TypeOf(pp[0]), d.log) if err != nil { return err } diff --git a/plugins/gitextractor/tasks/git_repo_collector.go b/plugins/gitextractor/tasks/git_repo_collector.go index 0ee6fece2b0..4dcb8cd24c2 100644 --- a/plugins/gitextractor/tasks/git_repo_collector.go +++ b/plugins/gitextractor/tasks/git_repo_collector.go @@ -56,7 +56,7 @@ func (o GitExtractorOptions) Valid() error { func CollectGitRepo(subTaskCtx core.SubTaskContext) error { var err error db := subTaskCtx.GetDb() - storage := store.NewDatabase(db) + storage := store.NewDatabase(db, subTaskCtx.GetLogger()) op := subTaskCtx.GetData().(GitExtractorOptions) p := parser.NewLibGit2(storage, subTaskCtx) if strings.HasPrefix(op.Url, "http") { diff --git a/plugins/helper/api_async_client_test.go b/plugins/helper/api_async_client_test.go index dff9c5f8b91..266c22b8aa8 100644 --- a/plugins/helper/api_async_client_test.go +++ b/plugins/helper/api_async_client_test.go @@ -31,6 +31,7 @@ import ( "time" "github.com/agiledragon/gomonkey/v2" + "github.com/apache/incubator-devlake/logger" "github.com/apache/incubator-devlake/plugins/core" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -103,7 +104,7 @@ func CreateTestAsyncApiClientWithRateLimitAndCtx(t *testing.T, rateLimiter *ApiR return &DefaultTaskContext{ &defaultExecContext{ cfg: cfg, - logger: &DefaultLogger{}, + logger: &logger.DefaultLogger{}, db: db, ctx: ctx, name: "Test", @@ -146,7 +147,7 @@ func TestWaitAsync_EmptyWork(t *testing.T) { func TestWaitAsync_WithWork(t *testing.T) { asyncApiClient, _ := CreateTestAsyncApiClient(t) - gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) { + gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) { }) defer gm_info.Reset() @@ -187,7 +188,7 @@ func TestWaitAsync_WithWork(t *testing.T) { func TestWaitAsync_MutiWork(t *testing.T) { asyncApiClient, _ := CreateTestAsyncApiClient(t) - gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) { + gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) { }) defer gm_info.Reset() @@ -231,7 +232,7 @@ func TestWaitAsync_MutiWork(t *testing.T) { // go test -gcflags=all=-l -run ^TestDoAsync_OnceSuceess func TestDoAsync_OnceSuceess(t *testing.T) { asyncApiClient, _ := CreateTestAsyncApiClient(t) - gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) { + gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) { }) defer gm_info.Reset() @@ -261,7 +262,7 @@ func TestDoAsync_OnceSuceess(t *testing.T) { // go test -gcflags=all=-l -run ^TestDoAsync_TryAndFail func TestDoAsync_TryAndFail(t *testing.T) { asyncApiClient, _ := CreateTestAsyncApiClient(t) - gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) { + gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) { }) defer gm_info.Reset() @@ -294,7 +295,7 @@ func TestDoAsync_TryAndFail(t *testing.T) { // go test -gcflags=all=-l -run ^TestDoAsync_TryAndSuceess func TestDoAsync_TryAndSuceess(t *testing.T) { asyncApiClient, _ := CreateTestAsyncApiClient(t) - gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) { + gm_info := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) { }) defer gm_info.Reset() diff --git a/plugins/helper/api_client.go b/plugins/helper/api_client.go index 33b7ef7a397..001b4c66f88 100644 --- a/plugins/helper/api_client.go +++ b/plugins/helper/api_client.go @@ -217,7 +217,7 @@ func (apiClient *ApiClient) Do( return nil, err } - // after recieve + // after receive if apiClient.afterReponse != nil { err = apiClient.afterReponse(res) if err != nil { diff --git a/plugins/helper/api_collector_test.go b/plugins/helper/api_collector_test.go index 532b22936b4..c8f89286b91 100644 --- a/plugins/helper/api_collector_test.go +++ b/plugins/helper/api_collector_test.go @@ -33,6 +33,7 @@ import ( "time" "github.com/agiledragon/gomonkey/v2" + "github.com/apache/incubator-devlake/logger" "github.com/apache/incubator-devlake/models/common" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -176,45 +177,41 @@ func MockDB(t *testing.T) { gt = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Table", func(db *gorm.DB, name string, args ...interface{}) *gorm.DB { assert.Equal(t, name, TestTableData.TableName()) return db - }, - ) + }) gc = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Create", func(db *gorm.DB, value interface{}) *gorm.DB { assert.Equal(t, TestTableData, value.(*TestTable)) return db - }, - ) + }) gd = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Delete", func(db *gorm.DB, value interface{}, conds ...interface{}) (tx *gorm.DB) { return db - }, - ) + }) ga = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "AutoMigrate", func(db *gorm.DB, dst ...interface{}) error { return nil - }, - ) + }) god = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Order", func(db *gorm.DB, value interface{}) (tx *gorm.DB) { return db - }, - ) + }) gw = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Where", func(db *gorm.DB, query interface{}, args ...interface{}) (tx *gorm.DB) { return db - }, - ) + }) + + gr = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Count", func(db *gorm.DB, count *int64) (tx *gorm.DB) { + return db + }) gr = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "Rows", func(db *gorm.DB) (*sql.Rows, error) { return &sql.Rows{}, nil - }, - ) + }) gs = gomonkey.ApplyMethod(reflect.TypeOf(&gorm.DB{}), "ScanRows", func(db *gorm.DB, rows *sql.Rows, dest interface{}) error { dest = TestRawMessage return nil - }, - ) + }) } @@ -268,7 +265,7 @@ func CreateTestApiCollector() (*ApiCollector, error) { return NewApiCollector(ApiCollectorArgs{ RawDataSubTaskArgs: RawDataSubTaskArgs{ Ctx: &DefaultSubTaskContext{ - defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), db, ctx, "Test", nil, nil), + defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), db, ctx, "Test", nil, nil), }, Table: TestTable{}.TableName(), Params: &TestParam{ @@ -1096,7 +1093,7 @@ func TestExecute_Total(t *testing.T) { }) defer gs.Reset() - gin := gomonkey.ApplyMethod(reflect.TypeOf(&DefaultLogger{}), "Info", func(_ *DefaultLogger, _ string, _ ...interface{}) { + gin := gomonkey.ApplyMethod(reflect.TypeOf(&logger.DefaultLogger{}), "Info", func(_ *logger.DefaultLogger, _ string, _ ...interface{}) { }) defer gin.Reset() diff --git a/plugins/helper/api_extractor.go b/plugins/helper/api_extractor.go index 8f6d6b6afe9..c96e33ed278 100644 --- a/plugins/helper/api_extractor.go +++ b/plugins/helper/api_extractor.go @@ -64,10 +64,18 @@ func NewApiExtractor(args ApiExtractorArgs) (*ApiExtractor, error) { func (extractor *ApiExtractor) Execute() error { // load data from database db := extractor.args.Ctx.GetDb() - cursor, err := db.Table(extractor.table).Order("id ASC").Where("params = ?", extractor.params).Rows() + log := extractor.args.Ctx.GetLogger() + count := int64(0) + cursor, err := db. + Table(extractor.table). + Order("id ASC"). + Where("params = ?", extractor.params). + Count(&count). + Rows() if err != nil { return err } + log.Info("get data from %s where params=%s and got %d", extractor.table, extractor.params, count) defer cursor.Close() row := &RawData{} @@ -113,7 +121,7 @@ func (extractor *ApiExtractor) Execute() error { for _, result := range results { // get the batch operator for the specific type - batch, err := divider.ForType(reflect.TypeOf(result)) + batch, err := divider.ForType(reflect.TypeOf(result), log) if err != nil { return err } diff --git a/plugins/helper/api_extractor_test.go b/plugins/helper/api_extractor_test.go index b895d2dff0a..d225e71da57 100644 --- a/plugins/helper/api_extractor_test.go +++ b/plugins/helper/api_extractor_test.go @@ -21,6 +21,8 @@ import ( "context" "database/sql" "fmt" + "github.com/apache/incubator-devlake/logger" + "github.com/apache/incubator-devlake/plugins/core" "reflect" "testing" "time" @@ -48,7 +50,7 @@ func CreateTestApiExtractor(t *testing.T) (*ApiExtractor, error) { return NewApiExtractor(ApiExtractorArgs{ RawDataSubTaskArgs: RawDataSubTaskArgs{ Ctx: &DefaultSubTaskContext{ - defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), &gorm.DB{}, ctx, "Test", nil, nil), + defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), &gorm.DB{}, ctx, "Test", nil, nil), }, Table: TestTable{}.TableName(), Params: &TestParam{ @@ -107,7 +109,7 @@ func TestApiExtractorExecute(t *testing.T) { ) fortypeTimes := 0 - gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) { + gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger) (*BatchSave, error) { fortypeTimes++ assert.Equal(t, rowType, reflect.TypeOf(TestTableData)) err := d.onNewBatchSave(rowType) @@ -168,7 +170,7 @@ func TestApiExtractorExecute_Cancel(t *testing.T) { ) fortypeTimes := 0 - gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) { + gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger) (*BatchSave, error) { fortypeTimes++ assert.Equal(t, rowType, reflect.TypeOf(TestTableData)) err := d.onNewBatchSave(rowType) diff --git a/plugins/helper/batch_save.go b/plugins/helper/batch_save.go index 2f9d20b7ac5..a5d66a7f6f7 100644 --- a/plugins/helper/batch_save.go +++ b/plugins/helper/batch_save.go @@ -19,11 +19,11 @@ package helper import ( "fmt" - "reflect" - "strings" - + "github.com/apache/incubator-devlake/plugins/core" "gorm.io/gorm" "gorm.io/gorm/clause" + "reflect" + "strings" ) // Insert data by batch can increase database performance drastically, this class aim to make batch-save easier, @@ -35,20 +35,24 @@ type BatchSave struct { // I'm guessing the reason is the type information lost when converted to interface{} slots reflect.Value db *gorm.DB + logger core.Logger current int size int valueIndex map[string]int } -func NewBatchSave(db *gorm.DB, slotType reflect.Type, size int) (*BatchSave, error) { +func NewBatchSave(db *gorm.DB, log core.Logger, slotType reflect.Type, size int) (*BatchSave, error) { if slotType.Kind() != reflect.Ptr { return nil, fmt.Errorf("slotType must be a pointer") } if !hasPrimaryKey(slotType) { - return nil, fmt.Errorf("no primary key") + return nil, fmt.Errorf("%s no primary key", slotType.String()) } + log = log.Nested(slotType.String()) + log.Info("create batch save success") return &BatchSave{ slotType: slotType, + logger: log, slots: reflect.MakeSlice(reflect.SliceOf(slotType), size, size), db: db, size: size, @@ -80,15 +84,19 @@ func (c *BatchSave) Add(slot interface{}) error { // flush out into database if max outed if c.current == c.size { return c.Flush() + } else if c.current%100 == 0 { + c.logger.Debug("batch save current: %d", c.current) } return nil } func (c *BatchSave) Flush() error { - err := c.db.Clauses(clause.OnConflict{UpdateAll: true}).Create(c.slots.Slice(0, c.current).Interface()).Error + result := c.db.Clauses(clause.OnConflict{UpdateAll: true}).Create(c.slots.Slice(0, c.current).Interface()) + err := result.Error if err != nil { return err } + c.logger.Info("batch save flush %d and %d success", c.slots.Slice(0, c.current).Len(), result.RowsAffected) c.current = 0 c.valueIndex = make(map[string]int) return nil diff --git a/plugins/helper/batch_save_divider.go b/plugins/helper/batch_save_divider.go index 9a20eb42a74..63003e98fef 100644 --- a/plugins/helper/batch_save_divider.go +++ b/plugins/helper/batch_save_divider.go @@ -18,6 +18,7 @@ limitations under the License. package helper import ( + "github.com/apache/incubator-devlake/plugins/core" "reflect" "gorm.io/gorm" @@ -47,13 +48,13 @@ func (d *BatchSaveDivider) OnNewBatchSave(cb OnNewBatchSave) { } // return *BatchSave for specified type -func (d *BatchSaveDivider) ForType(rowType reflect.Type) (*BatchSave, error) { +func (d *BatchSaveDivider) ForType(rowType reflect.Type, log core.Logger) (*BatchSave, error) { // get the cache for the specific type batch := d.batches[rowType] var err error // create one if not exists if batch == nil { - batch, err = NewBatchSave(d.db, rowType, d.batchSize) + batch, err = NewBatchSave(d.db, log, rowType, d.batchSize) if err != nil { return nil, err } diff --git a/plugins/helper/batch_save_divider_test.go b/plugins/helper/batch_save_divider_test.go index 4d7a8181ca8..42991a33a36 100644 --- a/plugins/helper/batch_save_divider_test.go +++ b/plugins/helper/batch_save_divider_test.go @@ -18,6 +18,7 @@ limitations under the License. package helper import ( + "github.com/apache/incubator-devlake/logger" "reflect" "testing" @@ -50,13 +51,13 @@ func TestBatchSaveDivider(t *testing.T) { var b3 *BatchSave // test if it saved and only saved once for one Type - b1, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData)) + b1, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData), logger.Global) assert.Equal(t, initTimes, 1) assert.Equal(t, err, nil) - b2, err = batchSaveDivider.ForType(reflect.TypeOf(&TestTable2{})) + b2, err = batchSaveDivider.ForType(reflect.TypeOf(&TestTable2{}), logger.Global) assert.Equal(t, initTimes, 2) assert.Equal(t, err, nil) - b3, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData)) + b3, err = batchSaveDivider.ForType(reflect.TypeOf(TestTableData), logger.Global) assert.Equal(t, initTimes, 2) assert.Equal(t, err, nil) diff --git a/plugins/helper/batch_save_test.go b/plugins/helper/batch_save_test.go index fd6bdfadcf5..464be8ee846 100644 --- a/plugins/helper/batch_save_test.go +++ b/plugins/helper/batch_save_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/agiledragon/gomonkey/v2" + "github.com/apache/incubator-devlake/logger" "github.com/apache/incubator-devlake/models/domainlayer" "github.com/apache/incubator-devlake/models/domainlayer/ticket" "github.com/stretchr/testify/assert" @@ -157,7 +158,7 @@ func TestBatchSave(t *testing.T) { TestBatchSize = 1 rowType := reflect.TypeOf(TestTableData) - batch, err := NewBatchSave(db, rowType, TestBatchSize) + batch, err := NewBatchSave(db, logger.Global, rowType, TestBatchSize) // test diff type assert.Equal(t, err, nil) diff --git a/plugins/helper/data_convertor.go b/plugins/helper/data_convertor.go index 29dc66a53b9..44a9affb511 100644 --- a/plugins/helper/data_convertor.go +++ b/plugins/helper/data_convertor.go @@ -115,7 +115,7 @@ func (converter *DataConverter) Execute() error { for _, result := range results { // get the batch operator for the specific type - batch, err := divider.ForType(reflect.TypeOf(result)) + batch, err := divider.ForType(reflect.TypeOf(result), converter.args.Ctx.GetLogger()) if err != nil { return err } diff --git a/plugins/helper/data_convertor_test.go b/plugins/helper/data_convertor_test.go index 3a4d84f7176..5791c54d498 100644 --- a/plugins/helper/data_convertor_test.go +++ b/plugins/helper/data_convertor_test.go @@ -26,6 +26,8 @@ import ( "time" "github.com/agiledragon/gomonkey/v2" + "github.com/apache/incubator-devlake/logger" + "github.com/apache/incubator-devlake/plugins/core" "github.com/magiconair/properties/assert" "github.com/sirupsen/logrus" "gorm.io/gorm" @@ -38,7 +40,7 @@ func CreateTestDataConverter(t *testing.T) (*DataConverter, error) { return NewDataConverter(DataConverterArgs{ RawDataSubTaskArgs: RawDataSubTaskArgs{ Ctx: &DefaultSubTaskContext{ - defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), &gorm.DB{}, ctx, "Test", nil, nil), + defaultExecContext: newDefaultExecContext(GetConfigForTest("../../"), logger.NewDefaultLogger(logrus.New(), "Test", make(map[string]*logrus.Logger)), &gorm.DB{}, ctx, "Test", nil, nil), }, Table: TestTable{}.TableName(), Params: &TestParam{ @@ -99,7 +101,7 @@ func TestDataConvertorExecute(t *testing.T) { ) fortypeTimes := 0 - gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) { + gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger) (*BatchSave, error) { fortypeTimes++ assert.Equal(t, rowType, reflect.TypeOf(TestTableData)) err := d.onNewBatchSave(rowType) @@ -160,7 +162,7 @@ func TestDataConvertorExecute_Cancel(t *testing.T) { ) fortypeTimes := 0 - gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type) (*BatchSave, error) { + gf := gomonkey.ApplyMethod(reflect.TypeOf(&BatchSaveDivider{}), "ForType", func(d *BatchSaveDivider, rowType reflect.Type, log core.Logger) (*BatchSave, error) { fortypeTimes++ assert.Equal(t, rowType, reflect.TypeOf(TestTableData)) err := d.onNewBatchSave(rowType) diff --git a/plugins/helper/default_task_context.go b/plugins/helper/default_task_context.go index f49e25347ac..3b14b0d08e0 100644 --- a/plugins/helper/default_task_context.go +++ b/plugins/helper/default_task_context.go @@ -22,6 +22,7 @@ import ( "fmt" "sync" "sync/atomic" + "time" "github.com/apache/incubator-devlake/plugins/core" "github.com/spf13/viper" @@ -152,7 +153,8 @@ func (c *DefaultTaskContext) IncProgress(quantity int) { // SubTaskContext default implementation type DefaultSubTaskContext struct { *defaultExecContext - taskCtx *DefaultTaskContext + taskCtx *DefaultTaskContext + LastProgressTime time.Time } func (c *DefaultSubTaskContext) SetProgress(current int, total int) { @@ -164,7 +166,12 @@ func (c *DefaultSubTaskContext) SetProgress(current int, total int) { func (c *DefaultSubTaskContext) IncProgress(quantity int) { c.defaultExecContext.IncProgress(core.SubTaskIncProgress, quantity) - c.logger.Info("finished records: %d", c.current) + if c.LastProgressTime.IsZero() || c.LastProgressTime.Add(3*time.Second).Before(time.Now()) || c.current%1000 == 0 { + c.LastProgressTime = time.Now() + c.logger.Info("finished records: %d", c.current) + } else { + c.logger.Debug("finished records: %d", c.current) + } } func NewDefaultTaskContext( @@ -193,6 +200,7 @@ func (c *DefaultTaskContext) SubTaskContext(subtask string) (core.SubTaskContext c.subtaskCtxs[subtask] = &DefaultSubTaskContext{ c.defaultExecContext.fork(subtask), c, + time.Time{}, } } c.defaultExecContext.mu.Unlock() @@ -220,6 +228,7 @@ func NewStandaloneSubTaskContext( return &DefaultSubTaskContext{ newDefaultExecContext(cfg, logger, db, ctx, name, data, nil), nil, + time.Time{}, } } diff --git a/plugins/helper/default_task_logger.go b/plugins/helper/default_task_logger.go index 01e72c5494b..2b4b20a37ca 100644 --- a/plugins/helper/default_task_logger.go +++ b/plugins/helper/default_task_logger.go @@ -18,6 +18,7 @@ limitations under the License. package helper import ( + "github.com/apache/incubator-devlake/logger" "github.com/apache/incubator-devlake/plugins/core" "github.com/sirupsen/logrus" ) @@ -26,5 +27,5 @@ import ( // TODO: implement another TaskLogger for distributed runner/worker func NewDefaultTaskLogger(log *logrus.Logger, prefix string, loggerPool map[string]*logrus.Logger) core.Logger { - return NewDefaultLogger(log, prefix, loggerPool) + return logger.NewDefaultLogger(log, prefix, loggerPool) } diff --git a/runner/db.go b/runner/db.go index ecf9b3798d5..a4d3a93b0b0 100644 --- a/runner/db.go +++ b/runner/db.go @@ -33,12 +33,12 @@ import ( func NewGormDb(config *viper.Viper, logger core.Logger) (*gorm.DB, error) { dbLoggingLevel := gormLogger.Error - switch config.GetString("DB_LOGGING_LEVEL") { - case "Silent": + switch strings.ToLower(config.GetString("DB_LOGGING_LEVEL")) { + case "silent": dbLoggingLevel = gormLogger.Silent - case "Warn": + case "warn": dbLoggingLevel = gormLogger.Warn - case "Info": + case "info": dbLoggingLevel = gormLogger.Info }