let d18n be importable by other project (#3)

* add make default .phony

* feature: preview support --verbose mode

* detect remove common.cfg

* detect package support multi goroutine callback

* lint package support multi goroutine callback

* remove package global variables

* update go.mod make d18n importable

* doc: mv logo into doc/images

Co-authored-by: zouyong007 <zouyong007@ke.com>

Makefile

@@ -1,3 +1,5 @@
+default: build
+
 include test/Makefile.mysql
 include test/Makefile.postgres
 include test/Makefile.oracle

README.md

@@ -2,7 +2,7 @@
 
 [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/)
 
-![logo](./logo_64x64.png)
+![logo](./doc/images/logo_64x64.png)
 
 `d18n` is a [numeronym](https://en.wikipedia.org/wiki/Numeronym) short for "data-desensitization", sounds like "d-eighteen-n".
 
@@ -23,7 +23,7 @@ For more details and latest updates, see [doc](./doc/toc.md) and [release](https
 d18n develop with [Golang](https://golang.org/) 1.16+, please install first.
 
 ```bash
-git clone github.com/LianjiaTech/d18n
+git clone https://github.com/LianjiaTech/d18n
 cd d18n
 
 # Mac or Linux

cmd/d18n/d18n.go

@@ -14,61 +14,66 @@
 package main
 
 import (
-	"d18n/common"
-	"d18n/detect"
-	"d18n/emport"
-	"d18n/lint"
-	"d18n/mask"
-	"d18n/preview"
-	"d18n/save"
+	"github.com/LianjiaTech/d18n/common"
+	"github.com/LianjiaTech/d18n/detect"
+	"github.com/LianjiaTech/d18n/emport"
+	"github.com/LianjiaTech/d18n/lint"
+	"github.com/LianjiaTech/d18n/mask"
+	"github.com/LianjiaTech/d18n/preview"
+	"github.com/LianjiaTech/d18n/save"
 )
 
+var c common.Config
+
 func main() {
+	var err error
+
 	// limit cpu 1 core, memory 2GB
 	common.PanicIfError(common.ResourceLimit(1, 2*1024*1024*1024))
 
 	// parse config
 	// common.PanicIfError(common.ParseFlag())
-	common.PanicIfError(common.ParseFlags())
+	c, err = common.ParseFlags()
+	common.PanicIfError(err)
 
 	// parse cipher config
-	common.PanicIfError(mask.ParseCipherConfig(common.Cfg.Cipher))
+	common.PanicIfError(mask.ParseCipherConfig(c.Cipher))
 
 	// print cipher
-	if common.Cfg.PrintCipher {
+	if c.PrintCipher {
 		mask.PrintCipher()
 		return
 	}
 
 	// print config
-	if common.Cfg.PrintConfig {
-		common.PrintConfig()
+	if c.PrintConfig {
+		common.PrintConfig(c)
 		return
 	}
 
 	// preview file
-	if common.Cfg.Preview > 0 {
+	if c.Preview > 0 {
 		common.PanicIfError(previewFile())
 		return
 	}
 
 	// lint file
-	if common.Cfg.Lint {
+	if c.Lint {
 		common.PanicIfError(lintFile())
 		return
 	}
 
 	// detect sensitive info
-	if common.Cfg.Detect {
+	if c.Detect {
 		common.PanicIfError(detectRows())
 		return
 	}
 
 	// init mask corpus
-	common.PanicIfError(mask.InitMaskCorpus(common.Cfg.RandSeed))
+	common.PanicIfError(mask.InitMaskCorpus(c.RandSeed))
 
 	// import file
-	if common.Cfg.Import {
+	if c.Import {
 		common.PanicIfError(emportFile())
 		return
 	}
@@ -77,7 +82,7 @@ func main() {
 }
 
 func previewFile() error {
-	p, err := preview.NewPreviewStruct(common.Cfg)
+	p, err := preview.NewPreviewStruct(c)
 	if err != nil {
 		return err
 	}
@@ -86,46 +91,52 @@ func previewFile() error {
 
 func saveRows() error {
 	// new save struct
-	s, err := save.NewSaveStruct(common.Cfg)
+	s, err := save.NewSaveStruct(c)
 	if err != nil {
 		return err
 	}
 
 	// query and save result
 	common.PanicIfError(s.Save())
 
-	// check save status
-	return s.CheckStatus()
+	// show save status
+	return s.ShowStatus()
 }
 
 func lintFile() error {
+	l, err := lint.NewLintStruct(c)
+	if err != nil {
+		return err
+	}
+
 	// check file format
-	common.PanicIfError(lint.Lint())
+	common.PanicIfError(l.Lint())
 
-	// check lint status
-	return lint.CheckStatus()
+	// show lint status
+	return l.ShowStatus()
 }
 
 func emportFile() error {
-	e, err := emport.NewEmportStruct(common.Cfg)
+	e, err := emport.NewEmportStruct(c)
 	if err != nil {
 		return err
 	}
 	// import file into database
 	common.PanicIfError(e.Emport())
 
-	// check emport status
-	return e.CheckStatus()
+	// show emport status
+	return e.ShowStatus()
 }
 
 func detectRows() error {
-	err := detect.ParseSensitiveConfig()
+	d, err := detect.NewDetectStruct(c)
 	if err != nil {
 		return err
 	}
 
-	common.PanicIfError(detect.Detect())
+	// detect sensitive data
+	common.PanicIfError(d.Detect())
 
-	// check detect status
-	return detect.CheckStatus()
+	// show detect status
+	return d.ShowStatus()
 }

cmd/d18n/d18n_test.go

@@ -17,28 +17,28 @@ import (
 	"os"
 	"testing"
 
-	"d18n/common"
+	"github.com/LianjiaTech/d18n/common"
 )
 
 func init() {
 	common.InitTestEnv()
 }
 
 func TestMainSave(t *testing.T) {
-	orgCfg := common.Cfg
+	orgCfg := common.TestConfig
 	orgArgs := os.Args
 	args := []string{
 		"--defaults-extra-file", common.TestPath + "/test/my.cnf",
-		"--query", common.Cfg.Query,
+		"--query", common.TestConfig.Query,
 	}
 	os.Args = append(os.Args[:1], args...)
 	main()
 	os.Args = orgArgs
-	common.Cfg = orgCfg
+	common.TestConfig = orgCfg
 }
 
 func TestMainEmport(t *testing.T) {
-	orgCfg := common.Cfg
+	orgCfg := common.TestConfig
 	orgArgs := os.Args
 	args := []string{
 		"--file", common.TestPath + "/test/actor.csv", "--import",
@@ -52,61 +52,62 @@ func TestMainEmport(t *testing.T) {
 	os.Args = append(os.Args[:1], args...)
 	main()
 	os.Args = orgArgs
-	common.Cfg = orgCfg
+	common.TestConfig = orgCfg
 }
 
 func Example_lint() {
-	orgCfg := common.Cfg
+	orgCfg := common.TestConfig
 	orgArgs := os.Args
 	args := []string{
 		"--file", common.TestPath + "/test/actor.csv", "--lint",
 	}
 	os.Args = append(os.Args[:1], args...)
 	main()
 	os.Args = orgArgs
-	common.Cfg = orgCfg
+	common.TestConfig = orgCfg
 	// Output:
 	// ok
 }
 
 func TestMainDetect(t *testing.T) {
-	orgCfg := common.Cfg
+	orgCfg := common.TestConfig
 	orgArgs := os.Args
 	args := []string{
+		"--defaults-extra-file", common.TestPath + "/test/my.cnf",
 		"--query", `select * from sakila.actor limit 10`, "--detect",
 	}
 	os.Args = append(os.Args[:1], args...)
 	main()
 	os.Args = orgArgs
-	common.Cfg = orgCfg
+	common.TestConfig = orgCfg
 }
 
 func TestMainPrintConfig(t *testing.T) {
-	orgCfg := common.Cfg
+	orgCfg := common.TestConfig
 	orgArgs := os.Args
 	args := []string{
 		"--print-config",
 	}
 	os.Args = append(os.Args[:1], args...)
 	main()
 	os.Args = orgArgs
-	common.Cfg = orgCfg
+	common.TestConfig = orgCfg
 }
 
 func TestMainPrintCipher(t *testing.T) {
-	orgCfg := common.Cfg
+	orgCfg := common.TestConfig
 	orgArgs := os.Args
 	args := []string{
 		"--print-cipher",
 	}
 	os.Args = append(os.Args[:1], args...)
 	main()
 	os.Args = orgArgs
-	common.Cfg = orgCfg
+	common.TestConfig = orgCfg
 }
 
 func Example_preview() {
-	orgCfg := common.Cfg
+	orgCfg := common.TestConfig
 	orgArgs := os.Args
 	args := []string{
 		"--preview", "2",
@@ -115,7 +116,7 @@ func Example_preview() {
 	os.Args = append(os.Args[:1], args...)
 	main()
 	os.Args = orgArgs
-	common.Cfg = orgCfg
+	common.TestConfig = orgCfg
 	// Output:
 	// actor_id,first_name,last_name,last_update
 	// 1,PENELOPE,GUINESS,2006-02-15 04:34:33

common/common.go

@@ -27,7 +27,7 @@ func InitTestEnv() {
 	_, filename, _, _ := runtime.Caller(0)
 	TestPath = filepath.Dir(filepath.Dir(filename))
 
-	Cfg = Config{
+	TestConfig = Config{
 		Server:   "mysql",
 		User:     "root",
 		Password: "",

common/config.go

@@ -90,9 +90,6 @@ type Config struct {
 	IgnoreColumns  []string // ignore column list
 }
 
-// Cfg global config
-var Cfg Config
-
 func parseCommaFlag(update string) []string {
 	var primary []string
 	update = strings.TrimSpace(update)
@@ -107,44 +104,44 @@ func parseCommaFlag(update string) []string {
 }
 
 // parseDefaultsExtraFile parse --defaults-extra-file file
-func parseDefaultsExtraFile(file string) error {
-	c, err := ini.Load(file)
+func parseDefaultsExtraFile(file string, c *Config) error {
+	config, err := ini.Load(file)
 	if err != nil {
 		return err
 	}
 
 	// get config from [client] section
-	Cfg.User = c.Section("client").Key("user").String()
-	Cfg.Password = c.Section("client").Key("password").String()
-	Cfg.Database = c.Section("client").Key("database").String()
-	Cfg.Host = c.Section("client").Key("host").String()
-	Cfg.Port = c.Section("client").Key("port").String()
-	Cfg.Charset = c.Section("client").Key("default-character-set").String()
+	c.User = config.Section("client").Key("user").String()
+	c.Password = config.Section("client").Key("password").String()
+	c.Database = config.Section("client").Key("database").String()
+	c.Host = config.Section("client").Key("host").String()
+	c.Port = config.Section("client").Key("port").String()
+	c.Charset = config.Section("client").Key("default-character-set").String()
 
 	return err
 }
 
-func ParseSchema() (header []HeaderColumn, err error) {
-	if Cfg.Schema != "" {
+func (c Config) ParseSchema() (header []HeaderColumn, err error) {
+	if c.Schema != "" {
 		// 1. TableTemplate
-		header, err = TableTemplate()
+		header, err = c.TableTemplate()
 		if err != nil {
 			return
 		}
 	} else {
 		// 2. GetColumnTypes -> DBParserColumnNames
 		var columns []*sql.ColumnType
-		columns, err = GetColumnTypes()
+		columns, err = c.GetColumnTypes()
 		if err != nil {
 			return
 		}
-		header = DBParseColumnTypes(columns)
+		header = c.DBParseColumnTypes(columns)
 	}
 	return
 }
 
-func PrintConfig() {
-	buf, err := yaml.Marshal(Cfg)
+func PrintConfig(c Config) {
+	buf, err := yaml.Marshal(c)
 	if err != nil {
 		println(err.Error())
 	}