Feature: support shadow type match regex (#365) (#405)

* Feature: support shadow type match regex #365:

1. create shadow tables in init.sql & sharding.sql
2. hide shadow tables for "show tables" command
3. impl shadow match hint rule (only select)
4. support shadow type match regex (only select)

* Feature: support shadow type match regex #365:

add integration test for shadow tables

conf/config.yaml

@@ -87,15 +87,6 @@ data:
               password: "123456"
               database: employees_0003
               weight: r10w10
-        - name: employees_shadow
-          nodes:
-            - name: node_shadow
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_shadow
-              weight: r10w10
   sharding_rule:
     tables:
       - name: employees.student
@@ -121,8 +112,7 @@ data:
   shadow_rule:
     tables:
       - name: employees.student
-        enable: false
-        group_node: employees_shadow
+        enable: true
         match_rules:
           - operation: [insert,update]
             match_type: value
@@ -133,7 +123,7 @@ data:
             match_type: regex
             attributes:
               - column: name
-                regex: "^hanmeimei$"
+                value: "^hanmeimei$"
           - operation: [select]
             match_type: hint
             attributes:

integration_test/config/shadow/config.yaml

@@ -87,15 +87,6 @@ data:
               password: "123456"
               database: employees_0003
               weight: r10w10
-        - name: employees_shadow
-          nodes:
-            - name: node_shadow
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_shadow
-              weight: r10w10
   sharding_rule:
     tables:
       - name: employees.student
@@ -121,7 +112,6 @@ data:
     tables:
       - name: employees.student
         enable: true
-        group_node: employees_shadow
         match_rules:
           - operation: [select]
             match_type: hint

integration_test/scene/shadow/integration_test.go

@@ -58,37 +58,18 @@ func (s *IntegrationSuite) TestShadowScene() {
 	assert.NoError(t, err, "should begin a new tx")
 
 	cases := s.TestCases()
-	for _, sqlCase := range cases.ExecCases {
-		for _, sense := range sqlCase.Sense {
-			if strings.Compare(strings.TrimSpace(sense), "shadow") == 1 {
-				params := strings.Split(sqlCase.Parameters, ",")
-				args := make([]interface{}, 0, len(params))
-				for _, param := range params {
-					k, _ := test.GetValueByType(param)
-					args = append(args, k)
-				}
-
-				// Execute sql
-				result, err := tx.Exec(sqlCase.SQL, args...)
-				assert.NoError(t, err, "exec not right")
-				err = sqlCase.ExpectedResult.CompareRow(result)
-				assert.NoError(t, err, err)
-			}
-		}
-	}
-
 	for _, sqlCase := range cases.QueryRowCases {
 		for _, sense := range sqlCase.Sense {
-			if strings.Compare(strings.TrimSpace(sense), "shadow") == 1 {
+			if strings.Compare(strings.TrimSpace(sense), "shadow") == 0 {
 				params := strings.Split(sqlCase.Parameters, ",")
 				args := make([]interface{}, 0, len(params))
 				for _, param := range params {
 					k, _ := test.GetValueByType(param)
 					args = append(args, k)
 				}
 
-				result := tx.QueryRow(sqlCase.SQL, args...)
-				err = sqlCase.ExpectedResult.CompareRow(result)
+				result := tx.QueryRow(sqlCase.SQL)
+				err := sqlCase.ExpectedResult.CompareRow(result)
 				assert.NoError(t, err, err)
 			}
 		}

integration_test/testcase/casetest.yaml

@@ -42,6 +42,35 @@ query_row_cases:
     expected:
       type: "valueInt"
       value: "1"
+  - sql: "SELECT COUNT(1) FROM student WHERE uid=1"
+    parameters: "1:int"
+    sense:
+      - shadow
+    expected:
+      type: "valueInt"
+      value: "1"
+  - sql: "/*A! shadow(shadow) */ SELECT COUNT(1) FROM student WHERE uid=1"
+    parameters: "1:int"
+    sense:
+      - shadow
+    expected:
+      type: "valueInt"
+      value: "0"
+  - sql: "SELECT COUNT(1) FROM student WHERE name='lilei'"
+    parameters: "lilei:string"
+    sense:
+      - shadow
+    expected:
+      type: "valueInt"
+      value: "0"
+  - sql: "SELECT COUNT(1) FROM student WHERE name='hanmeimei'"
+    parameters: "hanmeimei:string"
+    sense:
+      - shadow
+    expected:
+      type: "valueInt"
+      value: "0"
+
 exec_cases:
   - sql: "INSERT INTO sequence(name,value,modified_at) VALUES(?,?,NOW())"
     parameters: "1:string, 2:string"
@@ -60,9 +89,9 @@ exec_cases:
       value: 1
   - sql: "/*A! shadow(shadow) */ update student set score=100.0 where uid = ?"
     parameters: "2:int"
-      sense:
+    sense:
         - shadow
-      expected:
+    expected:
         type: "rowAffect"
         value: 1
   - sql: "/*A! shadow(shadow) */ delete from student"

pkg/constants/const.go

@@ -28,6 +28,12 @@ const (
 	VariableNameMaxAllowedPacket = "max_allowed_packet"
 )
 
+const (
+	HeaderPrefix           = "Tables_in_"
+	AranaSystemTablePrefix = "__arana_"
+	ShadowTablePrefix      = "__shadow_"
+)
+
 const (
 	ShadowMatchRegex = "regex"
 	ShadowMatchValue = "value"

pkg/proto/rule/shadow.go

@@ -18,6 +18,7 @@
 package rule
 
 import (
+	"regexp"
 	"sync"
 )
 
@@ -30,6 +31,7 @@ type ShadowRuleManager interface {
 	MatchHintBy(action, hint string) bool
 	MatchRegexBy(action, column, value string) bool
 	GetDatabase() string
+	GetTableName() string
 }
 
 // ShadowRule represents the shadow of databases and tables.
@@ -70,11 +72,18 @@ func (s *ShadowRule) MatchRegexBy(tableName, action, column, value string) bool
 	return rule.MatchRegexBy(action, column, value)
 }
 
-func (s *ShadowRule) GetDatabase(tableName string) string {
+func (s *ShadowRule) GetDatabase(database string) string {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
-	return s.rules[tableName].GetDatabase()
+	return database
+}
+
+func (s *ShadowRule) GetTableName(tableName string) string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	return constants.ShadowTablePrefix + tableName
 }
 
 func (s *ShadowRule) SetRuleManager(tableName string, ruleManager ShadowRuleManager) {
@@ -91,15 +100,20 @@ func NewShadowRule() *ShadowRule {
 }
 
 type Operation struct {
-	enable   bool
-	database string
-	actions  map[string][]*Attribute // map[action][]*Attribute, action in (select, update, delete, update)
+	enable    bool
+	database  string
+	tablename string
+	actions   map[string][]*Attribute // map[action][]*Attribute, action in (select, update, delete, update)
 }
 
 func (o *Operation) GetDatabase() string {
 	return o.database
 }
 
+func (o *Operation) GetTableName() string {
+	return o.tablename
+}
+
 func (o *Operation) MatchValueBy(action, column, value string) bool {
 	if !o.enable {
 		return false
@@ -140,11 +154,20 @@ func (o *Operation) MatchRegexBy(action, column, value string) bool {
 	if !o.enable {
 		return false
 	}
-	_, ok := o.actions[action]
+	attrs, ok := o.actions[action]
 	if !ok {
 		return false
 	}
 	// TODO impl regex rule below
+	for _, attr := range attrs {
+		if attr.typ == constants.ShadowMatchRegex {
+			reg, err := regexp.Compile(attr.value)
+			if err != nil {
+				return false
+			}
+			return reg.MatchString(value)
+		}
+	}
 
 	return false
 }

pkg/runtime/ast/expression_atom.go

@@ -132,7 +132,7 @@ func (u *UnaryExpressionAtom) Accept(visitor Visitor) (interface{}, error) {
 }
 
 func (u *UnaryExpressionAtom) IsOperatorNot() bool {
-	switch u.Operator {
+	switch strings.ToUpper(strings.TrimSpace(u.Operator)) {
 	case "!", "NOT":
 		return true
 	}

pkg/runtime/optimize/dml/delete.go

@@ -60,13 +60,15 @@ func optimizeDelete(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 	if shards == nil {
 		transparent := plan.Transparent(stmt, o.Args)
 		if matchShadow {
-			transparent.SetDB(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
+			//TODO: fix it
+			//transparent.SetDB(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
 		}
 		return transparent, nil
 	}
 
 	if matchShadow {
-		shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
+		//TODO: fix it
+		//shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
 	}
 
 	ret := dml.NewSimpleDeletePlan(stmt)

pkg/runtime/optimize/dml/insert.go

@@ -133,7 +133,8 @@ func optimizeInsert(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 
 		for k, v := range shards {
 			if matchShadow {
-				k = o.ShadowRule.GetDatabase(tableName.Suffix())
+				//TODO: fix it
+				//k = o.ShadowRule.GetDatabase(tableName.Suffix())
 			}
 			db = k
 			table = v[0]
@@ -148,7 +149,8 @@ func optimizeInsert(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 
 	for db, slot := range slots {
 		if matchShadow {
-			db = o.ShadowRule.GetDatabase(tableName.Suffix())
+			//TODO: fix it
+			//db = o.ShadowRule.GetDatabase(tableName.Suffix())
 		}
 		for table, indexes := range slot {
 			// clone insert stmt without values

pkg/runtime/optimize/dml/select.go

@@ -93,11 +93,17 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 			return nil, errors.Wrap(err, "calculate hints failed")
 		}
 
-		shards = hintLoader.GetShards()
 		matchShadow = hintLoader.GetMatchBy(tableName.Suffix(), constants.ShadowSelect)
 	}
 
 	if shards == nil {
+		//first shadow_rule, and then sharding_rule
+		if o.ShadowRule != nil && !matchShadow {
+			if matchShadow, err = (*optimize.ShadowSharder)(o.ShadowRule).Shard(tableName, constants.ShadowSelect, stmt.Where, o.Args...); err != nil && fullScan == false {
+				return nil, errors.Wrap(err, "calculate shards failed")
+			}
+		}
+
 		if shards, fullScan, err = (*optimize.Sharder)(o.Rule).Shard(tableName, stmt.Where, o.Args...); err != nil && fullScan == false {
 			return nil, errors.Wrap(err, "calculate shards failed")
 		}
@@ -137,7 +143,7 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 		}
 
 		if matchShadow {
-			db0 = o.ShadowRule.GetDatabase(tableName.Suffix())
+			tbl0 = o.ShadowRule.GetTableName(tbl0)
 		}
 		return toSingle(db0, tbl0)
 	}
@@ -147,10 +153,10 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 		var db, tbl string
 		for k, v := range shards {
 			db = k
+			tbl = v[0]
 			if matchShadow {
-				db = o.ShadowRule.GetDatabase(tableName.Suffix())
+				tbl = o.ShadowRule.GetTableName(v[0])
 			}
-			tbl = v[0]
 		}
 		return toSingle(db, tbl)
 	}
@@ -178,7 +184,9 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 	plans := make([]proto.Plan, 0, len(shards))
 	for k, v := range shards {
 		if matchShadow {
-			k = o.ShadowRule.GetDatabase(tableName.Suffix())
+			for vi, vt := range v {
+				v[vi] = o.ShadowRule.GetTableName(vt)
+			}
 		}
 		next := &dml.SimpleQueryPlan{
 			Database: k,

pkg/runtime/optimize/dml/update.go

@@ -103,7 +103,8 @@ func optimizeUpdate(_ context.Context, o *optimize.Optimizer) (proto.Plan, error
 	}
 
 	if matchShadow {
-		shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
+		//TODO: fix it
+		//shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
 	}
 
 	ret := dml.NewUpdatePlan(stmt)

pkg/runtime/optimize/sharder.go

@@ -46,6 +46,43 @@ func IsErrArgumentOutOfRange(err error) bool {
 
 // Sharder computes the shards from a SQL statement.
 type Sharder rule.Rule
+type ShadowSharder rule.ShadowRule
+
+func (ss *ShadowSharder) rule() *rule.ShadowRule {
+	return (*rule.ShadowRule)(ss)
+}
+
+// Shard returns shards.
+func (ss *ShadowSharder) Shard(tableName ast.TableName, action string, filter ast.ExpressionNode, args ...interface{}) (matchShadow bool, err error) {
+	var (
+		sh Sharder
+		sc shardCtx
+		lo logical.Logical
+		ev rrule.Evaluator
+	)
+
+	// 0. prepare shard context
+	sc.tableName = tableName
+	sc.args = args
+
+	// 1. expression to logical
+	if lo, err = sh.processExpression(&sc, filter); err != nil {
+		err = errors.Wrap(err, "compute shard logical failed")
+		return
+	}
+	// 2. logical to evaluator
+	if ev, err = rrule.EvalShadow(lo, tableName.Suffix(), action, ss.rule()); err != nil {
+		err = errors.Wrap(err, "compute shard evaluator failed")
+		return
+	}
+	// 3. match regex
+	matchShadow = false
+	if ev != nil {
+		matchShadow = true
+	}
+
+	return
+}
 
 // Shard returns shards.
 func (sh *Sharder) Shard(tableName ast.TableName, filter ast.ExpressionNode, args ...interface{}) (shards rule.DatabaseTables, fullScan bool, err error) {