Feature: support shadow type match regex (#365)

conf/config.yaml

@@ -87,15 +87,6 @@ data:
               password: "123456"
               database: employees_0003
               weight: r10w10
-        - name: employees_shadow
-          nodes:
-            - name: node_shadow
-              host: arana-mysql
-              port: 3306
-              username: root
-              password: "123456"
-              database: employees_shadow
-              weight: r10w10
   sharding_rule:
     tables:
       - name: employees.student
@@ -121,8 +112,7 @@ data:
   shadow_rule:
     tables:
       - name: employees.student
-        enable: false
-        group_node: employees_shadow
+        enable: true
         match_rules:
           - operation: [insert,update]
             match_type: value

pkg/constants/const.go

@@ -28,6 +28,12 @@ const (
 	VariableNameMaxAllowedPacket = "max_allowed_packet"
 )
 
+const (
+	HeaderPrefix           = "Tables_in_"
+	AranaSystemTablePrefix = "__arana_"
+	ShadowTablePrefix      = "__shadow_"
+)
+
 const (
 	ShadowMatchRegex = "regex"
 	ShadowMatchValue = "value"

pkg/proto/rule/shadow.go

@@ -18,6 +18,7 @@
 package rule
 
 import (
+	"regexp"
 	"sync"
 )
 
@@ -30,6 +31,7 @@ type ShadowRuleManager interface {
 	MatchHintBy(action, hint string) bool
 	MatchRegexBy(action, column, value string) bool
 	GetDatabase() string
+	GetTableName() string
 }
 
 // ShadowRule represents the shadow of databases and tables.
@@ -70,11 +72,18 @@ func (s *ShadowRule) MatchRegexBy(tableName, action, column, value string) bool
 	return rule.MatchRegexBy(action, column, value)
 }
 
-func (s *ShadowRule) GetDatabase(tableName string) string {
+func (s *ShadowRule) GetDatabase(database string) string {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
-	return s.rules[tableName].GetDatabase()
+	return database
+}
+
+func (s *ShadowRule) GetTableName(tableName string) string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	return constants.ShadowTablePrefix + tableName
 }
 
 func (s *ShadowRule) SetRuleManager(tableName string, ruleManager ShadowRuleManager) {
@@ -91,15 +100,20 @@ func NewShadowRule() *ShadowRule {
 }
 
 type Operation struct {
-	enable   bool
-	database string
-	actions  map[string][]*Attribute // map[action][]*Attribute, action in (select, update, delete, update)
+	enable    bool
+	database  string
+	tablename string
+	actions   map[string][]*Attribute // map[action][]*Attribute, action in (select, update, delete, update)
 }
 
 func (o *Operation) GetDatabase() string {
 	return o.database
 }
 
+func (o *Operation) GetTableName() string {
+	return o.tablename
+}
+
 func (o *Operation) MatchValueBy(action, column, value string) bool {
 	if !o.enable {
 		return false
@@ -140,11 +154,20 @@ func (o *Operation) MatchRegexBy(action, column, value string) bool {
 	if !o.enable {
 		return false
 	}
-	_, ok := o.actions[action]
+	attrs, ok := o.actions[action]
 	if !ok {
 		return false
 	}
 	// TODO impl regex rule below
+	for _, attr := range attrs {
+		if attr.typ == constants.ShadowMatchRegex {
+			reg, err := regexp.Compile(attr.value)
+			if err != nil {
+				return false
+			}
+			return reg.MatchString(value)
+		}
+	}
 
 	return false
 }

pkg/runtime/ast/expression_atom.go

@@ -132,7 +132,7 @@ func (u *UnaryExpressionAtom) Accept(visitor Visitor) (interface{}, error) {
 }
 
 func (u *UnaryExpressionAtom) IsOperatorNot() bool {
-	switch u.Operator {
+	switch strings.ToUpper(strings.Trim(u.Operator, " ")) {
 	case "!", "NOT":
 		return true
 	}

pkg/runtime/optimize/dml/delete.go

@@ -60,13 +60,15 @@ func optimizeDelete(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 	if shards == nil {
 		transparent := plan.Transparent(stmt, o.Args)
 		if matchShadow {
-			transparent.SetDB(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
+			//TODO: fix it
+			//transparent.SetDB(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
 		}
 		return transparent, nil
 	}
 
 	if matchShadow {
-		shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
+		//TODO: fix it
+		//shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
 	}
 
 	ret := dml.NewSimpleDeletePlan(stmt)

pkg/runtime/optimize/dml/insert.go

@@ -133,7 +133,8 @@ func optimizeInsert(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 
 		for k, v := range shards {
 			if matchShadow {
-				k = o.ShadowRule.GetDatabase(tableName.Suffix())
+				//TODO: fix it
+				//k = o.ShadowRule.GetDatabase(tableName.Suffix())
 			}
 			db = k
 			table = v[0]
@@ -148,7 +149,8 @@ func optimizeInsert(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 
 	for db, slot := range slots {
 		if matchShadow {
-			db = o.ShadowRule.GetDatabase(tableName.Suffix())
+			//TODO: fix it
+			//db = o.ShadowRule.GetDatabase(tableName.Suffix())
 		}
 		for table, indexes := range slot {
 			// clone insert stmt without values

pkg/runtime/optimize/dml/select.go

@@ -93,11 +93,17 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 			return nil, errors.Wrap(err, "calculate hints failed")
 		}
 
-		shards = hintLoader.GetShards()
 		matchShadow = hintLoader.GetMatchBy(tableName.Suffix(), constants.ShadowSelect)
 	}
 
 	if shards == nil {
+		//first shadow_rule, and then sharding_rule
+		if o.ShadowRule != nil && !matchShadow {
+			if matchShadow, err = (*optimize.ShadowSharder)(o.ShadowRule).Shard(tableName, constants.ShadowSelect, stmt.Where, o.Args...); err != nil && fullScan == false {
+				return nil, errors.Wrap(err, "calculate shards failed")
+			}
+		}
+
 		if shards, fullScan, err = (*optimize.Sharder)(o.Rule).Shard(tableName, stmt.Where, o.Args...); err != nil && fullScan == false {
 			return nil, errors.Wrap(err, "calculate shards failed")
 		}
@@ -137,7 +143,7 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 		}
 
 		if matchShadow {
-			db0 = o.ShadowRule.GetDatabase(tableName.Suffix())
+			tbl0 = o.ShadowRule.GetTableName(tbl0)
 		}
 		return toSingle(db0, tbl0)
 	}
@@ -147,10 +153,10 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 		var db, tbl string
 		for k, v := range shards {
 			db = k
+			tbl = v[0]
 			if matchShadow {
-				db = o.ShadowRule.GetDatabase(tableName.Suffix())
+				tbl = o.ShadowRule.GetTableName(v[0])
 			}
-			tbl = v[0]
 		}
 		return toSingle(db, tbl)
 	}
@@ -178,7 +184,9 @@ func optimizeSelect(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err
 	plans := make([]proto.Plan, 0, len(shards))
 	for k, v := range shards {
 		if matchShadow {
-			k = o.ShadowRule.GetDatabase(tableName.Suffix())
+			for vi, vt := range v {
+				v[vi] = o.ShadowRule.GetTableName(vt)
+			}
 		}
 		next := &dml.SimpleQueryPlan{
 			Database: k,

pkg/runtime/optimize/dml/update.go

@@ -103,7 +103,8 @@ func optimizeUpdate(_ context.Context, o *optimize.Optimizer) (proto.Plan, error
 	}
 
 	if matchShadow {
-		shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
+		//TODO: fix it
+		//shards.ReplaceDb(o.ShadowRule.GetDatabase(stmt.Table.Suffix()))
 	}
 
 	ret := dml.NewUpdatePlan(stmt)

pkg/runtime/optimize/sharder.go

@@ -46,6 +46,43 @@ func IsErrArgumentOutOfRange(err error) bool {
 
 // Sharder computes the shards from a SQL statement.
 type Sharder rule.Rule
+type ShadowSharder rule.ShadowRule
+
+func (ss *ShadowSharder) rule() *rule.ShadowRule {
+	return (*rule.ShadowRule)(ss)
+}
+
+// Shard returns shards.
+func (ss *ShadowSharder) Shard(tableName ast.TableName, action string, filter ast.ExpressionNode, args ...interface{}) (matchShadow bool, err error) {
+	var (
+		sh Sharder
+		sc shardCtx
+		lo logical.Logical
+		ev rrule.Evaluator
+	)
+
+	// 0. prepare shard context
+	sc.tableName = tableName
+	sc.args = args
+
+	// 1. expression to logical
+	if lo, err = sh.processExpression(&sc, filter); err != nil {
+		err = errors.Wrap(err, "compute shard logical failed")
+		return
+	}
+	// 2. logical to evaluator
+	if ev, err = rrule.EvalShadow(lo, tableName.Suffix(), action, ss.rule()); err != nil {
+		err = errors.Wrap(err, "compute shard evaluator failed")
+		return
+	}
+	// 3. match regex
+	matchShadow = false
+	if ev != nil {
+		matchShadow = true
+	}
+
+	return
+}
 
 // Shard returns shards.
 func (sh *Sharder) Shard(tableName ast.TableName, filter ast.ExpressionNode, args ...interface{}) (shards rule.DatabaseTables, fullScan bool, err error) {

pkg/runtime/plan/dal/show_tables.go

@@ -28,6 +28,7 @@ import (
 )
 
 import (
+	constdb "github.com/arana-db/arana/pkg/constants"
 	constant "github.com/arana-db/arana/pkg/constants/mysql"
 	"github.com/arana-db/arana/pkg/dataset"
 	"github.com/arana-db/arana/pkg/mysql"
@@ -41,11 +42,6 @@ import (
 
 var _ proto.Plan = (*ShowTablesPlan)(nil)
 
-const (
-	headerPrefix           = "Tables_in_"
-	aranaSystemTablePrefix = "__arana_"
-)
-
 type ShowTablesPlan struct {
 	plan.BasePlan
 	Database       string
@@ -94,7 +90,7 @@ func (st *ShowTablesPlan) ExecIn(ctx context.Context, conn proto.VConn) (proto.R
 
 	fields, _ := ds.Fields()
 
-	fields[0] = mysql.NewField(headerPrefix+rcontext.Schema(ctx), constant.FieldTypeVarString)
+	fields[0] = mysql.NewField(constdb.HeaderPrefix+rcontext.Schema(ctx), constant.FieldTypeVarString)
 
 	// filter duplicates
 	duplicates := make(map[string]struct{})
@@ -132,7 +128,10 @@ func (st *ShowTablesPlan) ExecIn(ctx context.Context, conn proto.VConn) (proto.R
 			}
 
 			tableName := vr.Values()[0].(string)
-			if strings.HasPrefix(tableName, aranaSystemTablePrefix) {
+			if strings.HasPrefix(tableName, constdb.AranaSystemTablePrefix) {
+				return false
+			}
+			if strings.HasPrefix(tableName, constdb.ShadowTablePrefix) {
 				return false
 			}
 			if _, ok := duplicates[tableName]; ok {

pkg/runtime/rule/evaluator.go

@@ -317,6 +317,54 @@ func (t *KeyedEvaluator) Not() Evaluator {
 	return ret
 }
 
+func EvalShadow(l logical.Logical, tableName, action string, rule *rule.ShadowRule) (Evaluator, error) {
+	ret, err := logical.Eval(l, func(a, b interface{}) (interface{}, error) {
+		x := a.(Evaluator)
+		y := b.(Evaluator)
+		return mergeShadow(tableName, action, rule, x, y)
+	}, func(a, b interface{}) (interface{}, error) {
+		x := a.(Evaluator)
+		y := b.(Evaluator)
+		return mergeShadow(tableName, action, rule, x, y)
+	}, func(i interface{}) interface{} {
+		x := i.(Evaluator)
+		return x.Not()
+	})
+	if err != nil {
+		return nil, err
+	}
+	//check atom logical, again
+	ret, err = mergeShadow(tableName, action, rule, ret.(Evaluator), _noopEvaluator)
+	if err != nil {
+		return nil, err
+	}
+	if ret == _emptyEvaluator || ret == _noopEvaluator {
+		return nil, nil
+	}
+
+	return ret.(Evaluator), nil
+}
+
+func mergeShadow(tableName, action string, rule *rule.ShadowRule, first, second Evaluator) (Evaluator, error) {
+	k1, ok1 := first.(*KeyedEvaluator)
+	k2, ok2 := second.(*KeyedEvaluator)
+
+	if ok1 {
+		s1, ok := k1.v.(string)
+		if ok && k1.op == cmp.Ceq && rule.MatchRegexBy(tableName, action, k1.k, s1) {
+			return k1, nil
+		}
+	}
+	if ok2 {
+		s2, ok := k2.v.(string)
+		if ok && k2.op == cmp.Ceq && rule.MatchRegexBy(tableName, action, k2.k, s2) {
+			return k2, nil
+		}
+	}
+
+	return _noopEvaluator, nil
+}
+
 func Eval(l logical.Logical, tableName string, rule *rule.Rule) (Evaluator, error) {
 	ret, err := logical.Eval(l, func(a, b interface{}) (interface{}, error) {
 		x := a.(Evaluator)

scripts/init.sql

@@ -40,7 +40,6 @@
 --
 
 CREATE DATABASE IF NOT EXISTS employees_0000 CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-CREATE DATABASE IF NOT EXISTS employees_shadow CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 USE employees_0000;