Merge branch 'martianzhang-main' into main

Makefile

@@ -1,6 +1,8 @@
 default: build
 
 include test/Makefile.mysql
+include test/Makefile.csvq
+include test/Makefile.sqlite
 include test/Makefile.postgres
 include test/Makefile.oracle
 include test/Makefile.mssql
@@ -109,7 +111,7 @@ cover: test
 			{print "$(CYELLOW)"$$0"%$(CEND)"}}'
 
 .PHONY: ci
-ci: cover test-mysql check-diff
+ci: cover test-mysql test-sqlite check-diff
 
 .PHONY: mask-typo
 mask-typo:

common/config.go

@@ -45,7 +45,7 @@ type Config struct {
 	Cipher                  string // cipher config file
 	Mask                    string // mask config file, csv format
 	Sensitive               string // sensitive data detection config
-	Verbose                 bool   // verbose mod
+	Verbose                 []bool // verbose mod
 	CheckEmpty              bool   // check empty result, if empty raise error
 	BOM                     bool   // add BOM ahead of plain text file, windows unicode chaos
 	NULLString              string // NULL value write in file, e.g., NULL, None, nil, ""

common/const.go

@@ -29,7 +29,7 @@ const (
 	WrongArgsCount   = `arguments count mismatch`
 	WrongMaskFunc    = `wrong mask function`
 	WrongArgValue    = `wrong arguments`
-	WrongQuotesValue = `ANSI_QUOTES  mode values not support double quotes`
+	WrongQuotesValue = `ANSI_QUOTES mode values not support double quotes`
 	WrongColumnsCnt  = `columns count mismatch`
 	WrongLargeThan0  = `n should large than 0`
 )

common/db.go

@@ -89,7 +89,7 @@ func (c Config) NewConnection() (*sql.DB, error) {
 		dsn = c.dsnMySQL()
 	case "postgres":
 		dsn = c.dsnPostgres()
-	case "sqlite", "csvq":
+	case "sqlite", "sqlite3", "csvq", "csv":
 		dsn = c.dsnFile()
 	case "oracle":
 		dsn = c.dsnOracle()
@@ -104,6 +104,13 @@ func (c Config) NewConnection() (*sql.DB, error) {
 	if c.DSN != "" {
 		dsn = strings.TrimSpace(c.DSN)
 	}
+
+	switch c.Server {
+	case "sqlite", "sqlite3":
+		c.Server = "sqlite"
+	case "csvq", "csv":
+		c.Server = "csvq"
+	}
 	return sql.Open(c.Server, dsn)
 }
 
@@ -112,11 +119,11 @@ func (c Config) SetForeignKeyChecks(enable bool, conn *sql.DB, args ...string) e
 	var err error
 	var sql string
 	switch c.Server {
-	case "sqlite":
+	case "sqlite", "sqlite3":
 		sql = fmt.Sprintf("pragma foreign_keys %v;", enable)
 	case "mysql":
 		sql = fmt.Sprintf("SET FOREIGN_KEY_CHECKS = %v;", enable)
-	case "csvq", "clickhouse", "presto":
+	case "csvq", "csv", "clickhouse", "presto":
 		return fmt.Errorf("not support foreign key")
 	case "postgres":
 		if enable {
@@ -229,9 +236,13 @@ func (c Config) DBParseHeaderColumn(header []HeaderColumn) []string {
 func (c Config) DBParseColumnTypes(header []*sql.ColumnType) []HeaderColumn {
 	var headerColumns []HeaderColumn
 	for _, h := range header {
+		var scanType string
+		if h.ScanType() != nil { // some database drive will not set ScanType, eg. sqlite
+			scanType = h.ScanType().Name()
+		}
 		headerColumns = append(headerColumns, HeaderColumn{
 			Name:         h.Name(),
-			ScanType:     h.ScanType().Name(),
+			ScanType:     scanType,
 			DatabaseType: h.DatabaseTypeName(),
 		})
 	}
@@ -299,12 +310,12 @@ func (c Config) hex(value interface{}) string {
 	switch c.Server {
 	case "postgres":
 		return `'\x` + hex.EncodeToString([]byte(ret)) + "'::bytea"
-	case "sqlite":
+	case "sqlite", "sqlite3":
 		return "X'" + hex.EncodeToString([]byte(ret)) + "'"
 	case "oracle":
 		// https://www.sqlines.com/oracle/datatypes/raw
 		return c.QuoteString(strings.ToUpper(hex.EncodeToString([]byte(ret))))
-	case "csvq", "clickhouse", "presto":
+	case "csvq", "csv", "clickhouse", "presto":
 		// not support binary data
 	default: // mysql, mariadb, tidb, sql server
 		return "0x" + hex.EncodeToString([]byte(ret))

common/db_test.go

@@ -129,7 +129,9 @@ func TestNewConnection(t *testing.T) {
 		"oracle",
 		"sqlserver",
 		"sqlite",
+		"sqlite3",
 		"csvq",
+		"csv",
 		"clickhouse",
 		"presto",
 	}

common/flag.go

@@ -35,7 +35,7 @@ func ParseFlags() (Config, error) {
 	var c Config
 
 	type option struct {
-		Verbose bool         `short:"v" long:"verbose" required:"false" description:"verbose mode"`
+		Verbose []bool       `short:"v" long:"verbose" required:"false" description:"verbose mode"`
 		Help    func() error `long:"help" required:"false" description:"Show this help message"`
 
 		// database config
@@ -283,7 +283,7 @@ func ParseFlags() (Config, error) {
 		c.Table = strings.Split(filepath.Base(c.File), ".")[0]
 	}
 
-	if c.Server == "sqlite" &&
+	if (c.Server == "sqlite" || c.Server == "sqlite3") &&
 		c.Database == "" && c.DSN == "" {
 		println("sqlite should specified `--database DATA_FILE` arg")
 		os.Exit(1)

detect/detect.go

@@ -241,7 +241,7 @@ func (d *DetectStruct) ShowStatus() error {
 	fmt.Println(string(s))
 
 	// verbose mode
-	if !d.Config.Verbose {
+	if len(d.Config.Verbose) == 0 {
 		return nil
 	}
 

detect/html_test.go

@@ -19,7 +19,7 @@ import (
 	"github.com/LianjiaTech/d18n/common"
 )
 
-func TestEmportHTML(t *testing.T) {
+func TestDetectHTML(t *testing.T) {
 	orgCfg := common.TestConfig
 
 	common.TestConfig.File = common.TestPath + "/test/actor.html"