refact - mysql sql parser

README.md

@@ -17,5 +17,4 @@ Revert replacement:
 ```bash
 go mod edit -dropreplace github.com/debeando/go-common
 go get -u
-go mod tidy
 ```

cast/main.go

@@ -30,6 +30,11 @@ func StringToFloat64(value string) float64 {
 	return i
 }
 
+func StringToDateTime(timestamp string, layout string) time.Time {
+	t, _ := time.Parse(layout, timestamp)
+	return t
+}
+
 func ToDateTime(timestamp string, layout string) string {
 	t, err := time.Parse(layout, timestamp)
 	if err != nil {

mysql/sql/digest.go → mysql/sql/parser/digest/digest.go

@@ -1,24 +1,23 @@
-package sql
+package digest
 
 import (
-	// Uncomment only for debug:
-	// "fmt"
 	"strings"
 	"unicode"
 )
 
 func Digest(s string) string {
+	sql := []rune(strings.Trim(strings.ToLower(s), " "))
 	comment := false
 	endnumber := []rune{' ', ',', '+', '-', '*', '/', '^', '%', '(', ')'}
+	length := len(sql)
 	list := false
-	values := false
 	multiline := false
 	number := false
 	quote := rune(0)
 	result := []rune("")
-	sql := []rune(strings.ToLower(s))
+	values := false
 	whitespace := false
-	length := len(sql)
+	whitespaces := 0
 
 	IsNumber := func(r rune) bool {
 		if unicode.IsNumber(r) || r == '.' {
@@ -38,7 +37,7 @@ func Digest(s string) string {
 
 	for x := 0; x < length; x++ {
 		// Uncomment only for debug:
-		// fmt.Printf("--> %d %s %s\n", x, string(sql[x]), string(result))
+		// fmt.Printf("--> %02d/%d %s %s\n", x, length, string(sql[x]), string(result))
 
 		// Remove comments:
 		if !comment && !multiline && sql[x] == '#' {
@@ -76,6 +75,19 @@ func Digest(s string) string {
 			number = false
 		}
 
+		// Remove whitespaces until semicolon:
+		if sql[x] == ' ' {
+			for y := 1; y < (length - x); y++ {
+				if whitespaces >= 0 && sql[x+y] == ';' {
+					return string(append(result, ';'))
+				} else if sql[x+y] == ' ' {
+					whitespaces++
+				} else {
+					break
+				}
+			}
+		}
+
 		// Remove literals inside of list " IN (":
 		if x >= 1 && sql[x-1] == ' ' && sql[x] == 'i' && x+1 < length && sql[x+1] == 'n' {
 			for y := 2; y < (length - x); y++ {
@@ -175,7 +187,6 @@ func Digest(s string) string {
 			continue
 		}
 
-		// Add character:
 		result = append(result, sql[x])
 	}
 

mysql/sql/digest_test.go → mysql/sql/parser/digest/digest_test.go

@@ -1,9 +1,9 @@
-package sql_test
+package digest_test
 
 import (
 	"testing"
 
-	"github.com/debeando/zenit/common/sql"
+	"github.com/debeando/go-common/mysql/sql/parser/digest"
 )
 
 var queries = []struct{ ID, Input, Expected string }{
@@ -64,6 +64,9 @@ var queries = []struct{ ID, Input, Expected string }{
 	{"string_number_alias_1",
 		`select if(foo = "3", 1, 2) as "test";`,
 		"select if(foo = '?', ?, ?) as '?';"},
+	{"string_number_alias_2",
+		`select if(foo = "3", 1, 2) as "test"  ;`,
+		"select if(foo = '?', ?, ?) as '?';"},
 	{"number_1",
 		`select 1234;`,
 		"select ?;"},
@@ -80,7 +83,7 @@ var queries = []struct{ ID, Input, Expected string }{
 		`select -0.1;`,
 		"select -?;"},
 	{"number_6",
-		`SELECT -.1;`,
+		`SELECT -.1 ;`,
 		"select -?;"},
 	{"number_7",
 		`select - 1;`,
@@ -122,13 +125,19 @@ var queries = []struct{ ID, Input, Expected string }{
 		"SELECT count(*) AS total FROM foo JOIN (SELECT DISTINCT * FROM bar WHERE fk = 1);",
 		"select count(*) as total from foo join (select distinct * from bar where fk = ?);"},
 	{"subquery_case_4",
-		"SELECT * FROM foo INNER JOIN (SELECT * FROM bar WHERE fk = 1);",
+		"SELECT * FROM foo INNER JOIN (SELECT * FROM bar WHERE fk = 1) ;",
 		"select * from foo inner join (select * from bar where fk = ?);"},
+	{"whitespace_case_1",
+		"  SELECT   *    FROM foo   ; ",
+		"select * from foo;"},
+	{"whitespace_case_2",
+		"SELECT   *\n    FROM\n    foo ;",
+		"select * from foo;"},
 }
 
 func TestDigest(t *testing.T) {
 	for _, test := range queries {
-		actual := sql.Digest(test.Input)
+		actual := digest.Digest(test.Input)
 
 		if test.Expected != actual {
 			t.Errorf("Test %s - Expected: '%s', got: '%s'.", test.ID, test.Expected, actual)

mysql/sql/parser/slow/logs.go

@@ -0,0 +1,45 @@
+package slow
+
+import (
+	"strings"
+)
+
+func LogsParser(in <-chan string, out chan<- string) {
+	var buffer string
+	var isHeader bool
+	var isQuery bool
+
+	for line := range in {
+		e := ""
+		l := len(line)
+
+		if isQuery == false && strings.HasPrefix(line, "# ") {
+			isHeader = true
+		}
+
+		if isHeader == true && l >= 6 {
+			buffer += line + "\n"
+
+			s := string(line[0:6])
+			s = strings.ToUpper(s)
+
+			if s == "SELECT" || s == "INSERT" || s == "UPDATE" || s == "DELETE" {
+				isQuery = true
+			}
+		}
+
+		if l > 1 {
+			e = string(line[l-1:])
+		} else {
+			e = string(line)
+		}
+
+		if isQuery == true && e == ";" {
+			out <- strings.TrimRight(buffer, "\n")
+
+			buffer = ""
+			isHeader = false
+			isQuery = false
+		}
+	}
+}

mysql/sql/parser/slow/logs_test.go

@@ -0,0 +1,59 @@
+package slow_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/debeando/go-common/mysql/sql/parser/slow"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var logs = []string{
+	"# Time: 2023-10-27T08:51:32.366670Z",
+	"# User@Host: test[test] @  [127.0.0.1]  Id:   725",
+	"# Query_time: 0.000135  Lock_time: 0.000002 Rows_sent: 11  Rows_examined: 11",
+	"SET timestamp=1698396692;",
+	"SELECT * FROM foo WHERE deleted_at IS NULL;",
+	"# Time: 2023-10-27T08:51:34.376283Z",
+	"# User@Host: test[test] @ [127.0.0.1]  Id:  3178",
+	"# Query_time: 0.019560  Lock_time: 0.000002 Rows_sent: 1  Rows_examined: 56914",
+	"SET timestamp=1698397201;",
+	"SELECT",
+	" count(*)",
+	"",
+	"FROM foo",
+	";",
+	"# Time: 2023-10-27T08:51:34.376283Z",
+	"# User@Host: test[test] @  [127.0.0.1]  Id:  1303",
+	"# Query_time: 0.000126  Lock_time: 0.000002 Rows_sent: 18  Rows_examined: 3583",
+	"SET timestamp=1698396694;",
+	"SELECT *",
+	"      FROM foo;",
+}
+
+func TestLogsParser(t *testing.T) {
+	logsCount := 0
+	channelIn := make(chan string)
+	channelOut := make(chan string)
+
+	defer close(channelIn)
+
+	go slow.LogsParser(channelIn, channelOut)
+
+	go func() {
+		defer close(channelOut)
+		for query := range channelOut {
+			t.Log("\n", query)
+			logsCount++
+		}
+	}()
+
+	for _, line := range logs {
+		channelIn <- line
+	}
+
+	time.Sleep(1 * time.Second)
+
+	assert.Equal(t, 3, logsCount)
+}

mysql/sql/parser/slow/slow.go → mysql/sql/parser/slow/query.go

@@ -2,55 +2,33 @@ package slow
 
 import (
 	"strings"
-)
-
-func Event(in <-chan string, out chan<- string) {
-	var buffer string
-	var isHeader bool
-	var isQuery bool
-
-	for line := range in {
-		e := ""
-		l := len(line)
-
-		if isQuery == false && strings.HasPrefix(line, "# ") {
-			isHeader = true
-		}
-
-		if isHeader == true && l >= 6 {
-			buffer += line + "\n"
-
-			s := string(line[0:6])
-			s = strings.ToUpper(s)
+	"time"
 
-			if s == "SELECT" || s == "INSERT" || s == "UPDATE" || s == "DELETE" {
-				isQuery = true
-			}
-		}
-
-		if l > 1 {
-			e = string(line[l-1:])
-		} else {
-			e = string(line)
-		}
-
-		if isQuery == true && e == ";" {
-			out <- strings.TrimRight(buffer, "\n")
+	"github.com/debeando/go-common/cast"
+	"github.com/debeando/go-common/mysql/sql/parser/digest"
+)
 
-			buffer = line + "\n"
-			isHeader = false
-			isQuery = false
-		}
-	}
+type Query struct {
+	Time         time.Time
+	Timestamp    int64
+	User         string
+	ID           int64
+	QueryTime    float64
+	LockTime     float64
+	RowsSent     int64
+	RowsExamined int64
+	Raw          string
+	Digest       string
+	DigestID     string
 }
 
-func Properties(event string) map[string]string {
+func QueryParser(query string) Query {
 	property := map[string]string{}
 	whiteSpaceStart := 0
 	whiteSpaceEnd := 0
 	startQuery := 0
 
-	p := []rune(event)
+	p := []rune(query)
 	l := len(p)
 
 	for x := 0; x < l; x++ {
@@ -73,11 +51,6 @@ func Properties(event string) map[string]string {
 					p[y] = ' '
 				}
 
-				// Replace unnecessary symbols:
-				if p[y] == '@' {
-					p[y] = '_'
-				}
-
 				// Register last White Space:
 				if p[y] == ' ' {
 					whiteSpaceEnd = y
@@ -93,6 +66,7 @@ func Properties(event string) map[string]string {
 			key := string(p[whiteSpaceStart:x])
 			key = strings.TrimSpace(key)
 			key = strings.ToLower(key)
+
 			value := strings.TrimSpace(string(p[x+1 : whiteSpaceEnd]))
 
 			property[key] = value
@@ -104,9 +78,33 @@ func Properties(event string) map[string]string {
 			startQuery = x + 25
 		}
 	}
-	// Find query:
+
 	property["query"] = string(p[startQuery:l])
 	property["query"] = strings.Trim(property["query"], "\n")
+	property["user"] = UserParser(property["user@host"])
+
+	return Query{
+		Time:         cast.StringToDateTime(property["time"], "2006-01-02T15:04:05.000000Z"),
+		ID:           cast.StringToInt64(property["id"]),
+		LockTime:     cast.StringToFloat64(property["lock_time"]),
+		QueryTime:    cast.StringToFloat64(property["query_time"]),
+		RowsExamined: cast.StringToInt64(property["rows_examined"]),
+		RowsSent:     cast.StringToInt64(property["rows_sent"]),
+		Timestamp:    cast.StringToInt64(property["timestamp"]),
+		Raw:          property["query"],
+		User:         property["user"],
+		Digest:       digest.Digest(property["query"]),
+	}
+}
+
+func UserParser(u string) string {
+	p := []rune(u)
+
+	for x := 0; x < len(p); x++ {
+		if p[x] == '[' {
+			return string(p[0:x])
+		}
+	}
 
-	return property
+	return ""
 }