From 639a260ee4aa93dab64ef88c1944a1ca57c39bc2 Mon Sep 17 00:00:00 2001 From: Jeffsky Date: Sat, 1 Jul 2023 17:21:39 +0800 Subject: [PATCH] feat: multiple sharding keys implementation (#681) * composite_sharding (#665) * feat: support multiple shard keys --------- Co-authored-by: binbin.zhang --- .github/workflows/reviewdog.yml | 2 +- .golangci.yml | 2 +- .pre-commit-config.yaml | 2 +- conf/config.yaml | 33 +- example/local_server/main.go | 2 +- go.mod | 11 +- go.sum | 26 +- integration_test/config/db/config.yaml | 10 +- integration_test/config/db_tbl/config.yaml | 8 +- integration_test/config/db_tbl_rw/config.yaml | 8 +- integration_test/config/tbl/config.yaml | 8 +- pkg/admin/admin.go | 6 +- pkg/admin/router/tables.go | 4 +- pkg/boot/boot.go | 1 + pkg/boot/discovery.go | 115 -- pkg/boot/discovery_test.go | 1 + pkg/boot/misc.go | 260 +++-- pkg/config/equals.go | 21 +- pkg/config/model.go | 15 +- pkg/config/model_test.go | 5 +- pkg/config/tenant.go | 2 +- pkg/proto/rule/rule.go | 293 +++--- pkg/proto/rule/rule_test.go | 77 +- pkg/proto/rule/shards.go | 9 + pkg/proto/rule/topology.go | 14 + pkg/proto/value.go | 8 + pkg/registry/nacos/client.go | 2 +- pkg/runtime/ast/ast.go | 4 +- pkg/runtime/ast/create_table.go | 10 +- pkg/runtime/ast/expression.go | 15 +- pkg/runtime/ast/insert.go | 22 +- pkg/runtime/ast/select_element.go | 1 + pkg/runtime/ast/show.go | 1 + .../builtin/shard_computer_javascript.go | 210 ++++ .../shard_computer_javascript_test.go} | 63 +- pkg/runtime/calc/calculus.go | 460 ++++++++ pkg/runtime/calc/calculus_test.go | 284 +++++ pkg/runtime/calc/logic/bool.go | 75 ++ pkg/runtime/calc/logic/logic.go | 981 ++++++++++++++++++ pkg/runtime/calc/logic/logic_test.go | 245 +++++ pkg/runtime/calc/logic/string.go | 33 + pkg/runtime/calc/misc.go | 248 +++++ .../{rule/route_test.go => calc/misc_test.go} | 72 +- pkg/runtime/cmp/cmp.go | 4 + pkg/runtime/logical/logical.go | 533 ---------- pkg/runtime/logical/logical_test.go | 59 -- pkg/runtime/misc/extvalue/div_test.go | 1 - pkg/runtime/misc/like_test.go | 3 +- pkg/runtime/optimize/dml/insert.go | 87 +- pkg/runtime/optimize/dml/update.go | 10 +- pkg/runtime/optimize/shard_visitor.go | 190 +++- pkg/runtime/optimize/shard_visitor_test.go | 41 +- pkg/runtime/plan/dal/show_database_rules.go | 21 +- .../plan/dal/show_database_rules_test.go | 3 +- pkg/runtime/plan/dal/show_sharding_table.go | 6 +- pkg/runtime/rule/evaluator.go | 775 -------------- pkg/runtime/rule/evaluator_test.go | 138 --- pkg/runtime/rule/iterator.go | 103 -- pkg/runtime/rule/iterator_test.go | 268 ----- pkg/runtime/rule/route.go | 130 --- pkg/runtime/rule/shard.go | 140 --- pkg/runtime/rule/shard_script.go | 134 --- pkg/runtime/rule/shard_test.go | 146 --- pkg/runtime/runtime.go | 5 +- pkg/runtime/transaction/fault_decision.go | 4 - pkg/runtime/transaction/xa.go | 4 +- pkg/security/tenant.go | 2 +- pkg/sequence/sequence.go | 2 +- pkg/util/config/config.go | 1 - pkg/util/math/compare.go | 40 + pkg/util/misc/misc.go | 36 + scripts/sharding.sql | 692 ++---------- test/integration_test.go | 10 +- testdata/fake_config.yaml | 10 +- testdata/mock_rule.go | 84 +- 75 files changed, 3588 insertions(+), 3763 deletions(-) create mode 100644 pkg/runtime/builtin/shard_computer_javascript.go rename pkg/runtime/{rule/shard_script_test.go => builtin/shard_computer_javascript_test.go} (60%) create mode 100644 pkg/runtime/calc/calculus.go create mode 100644 pkg/runtime/calc/calculus_test.go create mode 100644 pkg/runtime/calc/logic/bool.go create mode 100644 pkg/runtime/calc/logic/logic.go create mode 100644 pkg/runtime/calc/logic/logic_test.go create mode 100644 pkg/runtime/calc/logic/string.go create mode 100644 pkg/runtime/calc/misc.go rename pkg/runtime/{rule/route_test.go => calc/misc_test.go} (53%) delete mode 100644 pkg/runtime/logical/logical.go delete mode 100644 pkg/runtime/logical/logical_test.go delete mode 100644 pkg/runtime/rule/evaluator.go delete mode 100644 pkg/runtime/rule/evaluator_test.go delete mode 100644 pkg/runtime/rule/iterator.go delete mode 100644 pkg/runtime/rule/iterator_test.go delete mode 100644 pkg/runtime/rule/route.go delete mode 100644 pkg/runtime/rule/shard.go delete mode 100644 pkg/runtime/rule/shard_script.go delete mode 100644 pkg/runtime/rule/shard_test.go create mode 100644 pkg/util/math/compare.go diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index d91d1a71..3cc2e4f7 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -36,7 +36,7 @@ jobs: - uses: reviewdog/action-golangci-lint@v2 with: go_version: "1.18" - golangci_lint_version: "v1.46.2" # use latest version by default + golangci_lint_version: "v1.47.3" # use latest version by default golangci_lint_flags: "-v --timeout 10m" fail_on_error: true diff --git a/.golangci.yml b/.golangci.yml index 6f6c461c..37d14bb5 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -74,6 +74,6 @@ issues: # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration service: - golangci-lint-version: 1.16.x # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.47.3 # use the fixed version to not introduce new linters unexpectedly prepare: - echo "here I can run custom commands, but no preparation needed for this repo" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51cf416d..459aef81 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,6 +39,6 @@ repos: - id: go-mod-tidy - id: go-staticcheck-mod - repo: https://github.com/golangci/golangci-lint - rev: v1.46.2 + rev: v1.47.3 hooks: - id: golangci-lint diff --git a/conf/config.yaml b/conf/config.yaml index c211664b..9fc76f72 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -65,20 +65,39 @@ data: type: snowflake option: db_rules: - - column: uid - type: scriptExpr - expr: parseInt($value % 32 / 8) + - columns: + - name: uid + expr: parseInt($0 % 32 / 8) tbl_rules: - - column: uid - type: scriptExpr - expr: $value % 32 - step: 32 + - columns: + - name: uid + expr: $0 % 32 topology: db_pattern: employees_${0000..0003} tbl_pattern: student_${0000..0031} attributes: allow_full_scan: true sqlMaxLimit: -1 + - name: employees.friendship + sequence: + type: snowflake + option: + db_rules: + - columns: + - name: uid + - name: friend_id + expr: parseInt(($0*31+$1) % 32 / 8) + tbl_rules: + - columns: + - name: uid + - name: friend_id + expr: ($0*31+$1) % 32 + topology: + db_pattern: employees_${0000..0003} + tbl_pattern: friendship_${0000..0031} + attributes: + allow_full_scan: true + sqlMaxLimit: -1 nodes: node0: name: node0 diff --git a/example/local_server/main.go b/example/local_server/main.go index 34d7ae5b..1ebbd2c8 100644 --- a/example/local_server/main.go +++ b/example/local_server/main.go @@ -23,6 +23,6 @@ import ( ) func main() { - bootstrap := testdata.Path("../conf/bootstrap.local-etcd.yaml") + bootstrap := testdata.Path("../conf/bootstrap.yaml") start.Run(bootstrap) } diff --git a/go.mod b/go.mod index fbfe4460..e7551590 100644 --- a/go.mod +++ b/go.mod @@ -10,13 +10,13 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 github.com/creasty/defaults v1.6.0 github.com/docker/go-units v0.4.0 - github.com/dop251/goja v0.0.0-20220422102209-3faab1d8f20e + github.com/dop251/goja v0.0.0-20230427124612-428fc442ff5f github.com/dubbogo/gost v1.12.3 github.com/gin-gonic/gin v1.8.1 github.com/go-playground/validator/v10 v10.11.1 github.com/go-sql-driver/mysql v1.6.0 github.com/golang/mock v1.5.0 - github.com/google/btree v1.0.0 + github.com/google/btree v1.1.2 github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru v0.5.4 github.com/nacos-group/nacos-sdk-go/v2 v2.0.1 @@ -42,7 +42,7 @@ require ( golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d golang.org/x/net v0.4.0 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/text v0.5.0 + golang.org/x/text v0.9.0 google.golang.org/grpc v1.42.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -60,7 +60,7 @@ require ( github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect + github.com/dlclark/regexp2 v1.9.0 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.11+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect @@ -78,6 +78,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/pprof v0.0.0-20230429030804-905365eefe3e // indirect github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 // indirect @@ -123,7 +124,7 @@ require ( go.etcd.io/etcd/raft/v3 v3.5.0-alpha.0 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/crypto v0.4.0 // indirect - golang.org/x/sys v0.3.0 // indirect + golang.org/x/sys v0.5.0 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect google.golang.org/genproto v0.0.0-20211104193956-4c6863e31247 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 6757a687..88770909 100644 --- a/go.sum +++ b/go.sum @@ -148,8 +148,11 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= @@ -278,8 +281,10 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8l github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.9.0 h1:pTK/l/3qYIKaRXuHnEnIf7Y5NxfRPfpb7dis6/gdlVI= +github.com/dlclark/regexp2 v1.9.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -299,8 +304,8 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNE github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20220422102209-3faab1d8f20e h1:e9YSzijUx20vAPPXStf1gDpx85H1kocNqrWt61JIP9k= -github.com/dop251/goja v0.0.0-20220422102209-3faab1d8f20e/go.mod h1:TQJQ+ZNyFVvUtUEtCZxBhfWiH7RJqR3EivNmvD6Waik= +github.com/dop251/goja v0.0.0-20230427124612-428fc442ff5f h1:3Z9NjtffvA8Qoh8xjgUpPmyKawJw/mDRcJlR9oPCvqI= +github.com/dop251/goja v0.0.0-20230427124612-428fc442ff5f/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dubbogo/go-zookeeper v1.0.4-0.20211212162352-f9d2183d89d5/go.mod h1:fn6n2CAEer3novYgk9ULLwAjuV8/g4DdC2ENwRb6E+c= @@ -450,8 +455,9 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -482,6 +488,9 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= +github.com/google/pprof v0.0.0-20230429030804-905365eefe3e h1:yuPVjc55Y343adYwQLA29DR8KrA0yuLJLlN5LxqlsgQ= +github.com/google/pprof v0.0.0-20230429030804-905365eefe3e/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -546,6 +555,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -1255,11 +1265,13 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= @@ -1272,8 +1284,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/integration_test/config/db/config.yaml b/integration_test/config/db/config.yaml index 3a3be2ee..4feaa6c5 100644 --- a/integration_test/config/db/config.yaml +++ b/integration_test/config/db/config.yaml @@ -61,13 +61,13 @@ data: type: snowflake option: db_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid + step: 32 expr: parseInt($value % 32 / 8) - step: 32 tbl_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid expr: parseInt(0) topology: db_pattern: employees_${0000...0003} diff --git a/integration_test/config/db_tbl/config.yaml b/integration_test/config/db_tbl/config.yaml index 456f95d7..1250f3db 100644 --- a/integration_test/config/db_tbl/config.yaml +++ b/integration_test/config/db_tbl/config.yaml @@ -62,12 +62,12 @@ data: type: snowflake option: db_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid expr: parseInt($value % 32 / 8) tbl_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid expr: $value % 32 topology: db_pattern: employees_${0000..0003} diff --git a/integration_test/config/db_tbl_rw/config.yaml b/integration_test/config/db_tbl_rw/config.yaml index ab43afee..a5129bb3 100644 --- a/integration_test/config/db_tbl_rw/config.yaml +++ b/integration_test/config/db_tbl_rw/config.yaml @@ -101,12 +101,12 @@ data: type: snowflake option: db_rules: - - column: uid - type: scriptExpr + - column: + - name: uid expr: parseInt($value % 32 / 8) tbl_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid expr: $value % 32 topology: db_pattern: employees_${0000..0003} diff --git a/integration_test/config/tbl/config.yaml b/integration_test/config/tbl/config.yaml index 0f09f1fc..ec579283 100644 --- a/integration_test/config/tbl/config.yaml +++ b/integration_test/config/tbl/config.yaml @@ -52,12 +52,12 @@ data: type: snowflake option: db_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid expr: parseInt(0) tbl_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid expr: $value % 32 topology: db_pattern: employees_0000 diff --git a/pkg/admin/admin.go b/pkg/admin/admin.go index 07818042..4585252e 100644 --- a/pkg/admin/admin.go +++ b/pkg/admin/admin.go @@ -47,8 +47,10 @@ const ( _defaultUIPath = "/var/www/arana" ) -const _ADMIN_SERVICE = "ARANA_ADMIN_SERVICE" -const _SERVICE_DISCOVERY = "ARANA_SERVICE_DISCOVERY" +const ( + _ADMIN_SERVICE = "ARANA_ADMIN_SERVICE" + _SERVICE_DISCOVERY = "ARANA_SERVICE_DISCOVERY" +) var _hooks []Hook diff --git a/pkg/admin/router/tables.go b/pkg/admin/router/tables.go index df8ebca2..d1dba58b 100644 --- a/pkg/admin/router/tables.go +++ b/pkg/admin/router/tables.go @@ -42,9 +42,7 @@ func init() { } func ListAllTables(c *gin.Context) error { - var ( - result = make([]*admin.TableDTO, 0) - ) + result := make([]*admin.TableDTO, 0) service := admin.GetService(c) tenants, err := service.ListTenants(c) diff --git a/pkg/boot/boot.go b/pkg/boot/boot.go index da66f23f..2782e8e5 100644 --- a/pkg/boot/boot.go +++ b/pkg/boot/boot.go @@ -257,6 +257,7 @@ func buildNamespace(ctx context.Context, tenant string, provider Discovery, clus continue } ru.SetVTable(table, vt) + log.Infof("add logical table '%s' success", table) } initCmds = append(initCmds, namespace.UpdateRule(&ru)) diff --git a/pkg/boot/discovery.go b/pkg/boot/discovery.go index 50083356..871ca64f 100644 --- a/pkg/boot/discovery.go +++ b/pkg/boot/discovery.go @@ -19,11 +19,6 @@ package boot import ( "context" - "fmt" - "regexp" - "strconv" - "strings" - "sync" ) import ( @@ -37,7 +32,6 @@ import ( import ( "github.com/arana-db/arana/pkg/config" "github.com/arana-db/arana/pkg/proto/rule" - rrule "github.com/arana-db/arana/pkg/runtime/rule" "github.com/arana-db/arana/pkg/security" "github.com/arana-db/arana/pkg/trace" uconfig "github.com/arana-db/arana/pkg/util/config" @@ -47,24 +41,12 @@ import ( var _ Discovery = (*discovery)(nil) -var ( - _regexpRuleExpr *regexp.Regexp - _regexpRuleExprSync sync.Once -) - var ( ErrorNoTenant = errors.New("no tenant") ErrorNoDataSourceCluster = errors.New("no datasourceCluster") ErrorNoGroup = errors.New("no group") ) -func getRuleExprRegexp() *regexp.Regexp { - _regexpRuleExprSync.Do(func() { - _regexpRuleExpr = regexp.MustCompile(`([a-zA-Z0-9_]+)\(\s*([0-9]|[1-9][0-9]+)?\s*\)`) - }) - return _regexpRuleExpr -} - type discovery struct { inited uatomic.Bool path string @@ -435,103 +417,6 @@ func (fp *discovery) GetOptions() *config.BootOptions { return fp.options } -var ( - _regexpTopology *regexp.Regexp - _regexpTopologyOnce sync.Once -) - -func getTopologyRegexp() *regexp.Regexp { - _regexpTopologyOnce.Do(func() { - _regexpTopology = regexp.MustCompile(`\${(?P\d+)\.{2,}(?P\d+)}`) - }) - return _regexpTopology -} - -func parseTopology(input string) (format string, begin, end int, err error) { - mats := getTopologyRegexp().FindAllStringSubmatch(input, -1) - - if len(mats) < 1 { - format = input - begin = -1 - end = -1 - return - } - - if len(mats) > 1 { - err = errors.Errorf("invalid topology expression: %s", input) - return - } - - var beginStr, endStr string - for i := 1; i < len(mats[0]); i++ { - switch getTopologyRegexp().SubexpNames()[i] { - case "begin": - beginStr = mats[0][i] - case "end": - endStr = mats[0][i] - } - } - - if len(beginStr) != len(endStr) { - err = errors.Errorf("invalid topology expression: %s", input) - return - } - - format = getTopologyRegexp().ReplaceAllString(input, fmt.Sprintf(`%%0%dd`, len(beginStr))) - begin, _ = strconv.Atoi(strings.TrimLeft(beginStr, "0")) - end, _ = strconv.Atoi(strings.TrimLeft(endStr, "0")) - return -} - -func toSharder(input *config.Rule) (rule.ShardComputer, error) { - var ( - computer rule.ShardComputer - mod int - err error - ) - - if mat := getRuleExprRegexp().FindStringSubmatch(input.Expr); len(mat) == 3 { - mod, _ = strconv.Atoi(mat[2]) - } - - switch rrule.ShardType(input.Type) { - case rrule.ModShard: - computer = rrule.NewModShard(mod) - case rrule.HashMd5Shard: - computer = rrule.NewHashMd5Shard(mod) - case rrule.HashBKDRShard: - computer = rrule.NewHashBKDRShard(mod) - case rrule.HashCrc32Shard: - computer = rrule.NewHashCrc32Shard(mod) - case rrule.ScriptExpr: - computer, err = rrule.NewJavascriptShardComputer(input.Expr) - default: - panic(fmt.Errorf("error config, unsupport shard type: %s", input.Type)) - } - return computer, err -} - -func toRawShard(input *config.Rule) *rule.RawShardRule { - var res rule.RawShardRule - res.Expr = input.Expr - res.Type = input.Type - res.Column = input.Column - if input.Step > 0 { - res.Step = input.Step - } - return &res -} -func getRender(format string) func(int) string { - if strings.ContainsRune(format, '%') { - return func(i int) string { - return fmt.Sprintf(format, i) - } - } - return func(i int) string { - return format - } -} - func NewDiscovery(path string) Discovery { return &discovery{ path: path, diff --git a/pkg/boot/discovery_test.go b/pkg/boot/discovery_test.go index fda85294..b2e26b17 100644 --- a/pkg/boot/discovery_test.go +++ b/pkg/boot/discovery_test.go @@ -29,6 +29,7 @@ import ( import ( "github.com/arana-db/arana/pkg/constants" + _ "github.com/arana-db/arana/pkg/runtime/builtin" "github.com/arana-db/arana/testdata" ) diff --git a/pkg/boot/misc.go b/pkg/boot/misc.go index ebc7ef0c..0886b73b 100644 --- a/pkg/boot/misc.go +++ b/pkg/boot/misc.go @@ -18,8 +18,10 @@ package boot import ( + "fmt" "regexp" "strconv" + "strings" "sync" ) @@ -30,8 +32,57 @@ import ( import ( "github.com/arana-db/arana/pkg/config" "github.com/arana-db/arana/pkg/proto/rule" + "github.com/arana-db/arana/pkg/util/math" ) +var ( + _regexpTopology *regexp.Regexp + _regexpTopologyOnce sync.Once +) + +func getTopologyRegexp() *regexp.Regexp { + _regexpTopologyOnce.Do(func() { + _regexpTopology = regexp.MustCompile(`\${(?P\d+)\.{2,}(?P\d+)}`) + }) + return _regexpTopology +} + +func parseTopology(input string) (format string, begin, end int, err error) { + mats := getTopologyRegexp().FindAllStringSubmatch(input, -1) + + if len(mats) < 1 { + format = input + begin = -1 + end = -1 + return + } + + if len(mats) > 1 { + err = errors.Errorf("invalid topology expression: %s", input) + return + } + + var beginStr, endStr string + for i := 1; i < len(mats[0]); i++ { + switch getTopologyRegexp().SubexpNames()[i] { + case "begin": + beginStr = mats[0][i] + case "end": + endStr = mats[0][i] + } + } + + if len(beginStr) != len(endStr) { + err = errors.Errorf("invalid topology expression: %s", input) + return + } + + format = getTopologyRegexp().ReplaceAllString(input, fmt.Sprintf(`%%0%dd`, len(beginStr))) + begin, _ = strconv.Atoi(strings.TrimLeft(beginStr, "0")) + end, _ = strconv.Atoi(strings.TrimLeft(endStr, "0")) + return +} + func makeVTable(tableName string, table *config.Table) (*rule.VTable, error) { var ( vt rule.VTable @@ -40,7 +91,6 @@ func makeVTable(tableName string, table *config.Table) (*rule.VTable, error) { dbBegin, tbBegin int dbEnd, tbEnd int err error - ok bool ) if table.Topology != nil { @@ -57,122 +107,42 @@ func makeVTable(tableName string, table *config.Table) (*rule.VTable, error) { } topology.SetRender(getRender(dbFormat), getRender(tbFormat)) - var ( - keys map[string]struct{} - dbSharder, tbSharder map[string]rule.ShardComputer - dbSteps, tbSteps map[string]int - dbRules []*rule.RawShardRule - tblRules []*rule.RawShardRule - ) - for _, it := range table.DbRules { - var shd rule.ShardComputer - if shd, err = toSharder(it); err != nil { - return nil, err - } - if dbSharder == nil { - dbSharder = make(map[string]rule.ShardComputer) - } - if keys == nil { - keys = make(map[string]struct{}) - } - if dbSteps == nil { - dbSteps = make(map[string]int) - } - dbSharder[it.Column] = shd - keys[it.Column] = struct{}{} - dbSteps[it.Column] = it.Step - dbRules = append(dbRules, toRawShard(it)) - } + dbAmount := dbEnd - dbBegin + 1 + tbAmount := tbEnd - tbBegin + 1 + tblsPerDB := tbAmount / dbAmount - for _, it := range table.TblRules { - var shd rule.ShardComputer - if shd, err = toSharder(it); err != nil { - return nil, err - } - if tbSharder == nil { - tbSharder = make(map[string]rule.ShardComputer) - } - if keys == nil { - keys = make(map[string]struct{}) - } - if tbSteps == nil { - tbSteps = make(map[string]int) - } - tbSharder[it.Column] = shd - keys[it.Column] = struct{}{} - tbSteps[it.Column] = it.Step - tblRules = append(tblRules, toRawShard(it)) - } - - for k := range keys { + for i := 0; i < dbAmount; i++ { var ( - shd rule.ShardComputer - dbMetadata, tbMetadata *rule.ShardMetadata + x = dbBegin + i + y []int ) - if shd, ok = dbSharder[k]; ok { - dbMetadata = &rule.ShardMetadata{ - Computer: shd, - Stepper: rule.DefaultNumberStepper, - } - if s, ok := dbSteps[k]; ok && s > 0 { - dbMetadata.Steps = s - } else if dbBegin >= 0 && dbEnd >= 0 { - dbMetadata.Steps = 1 + dbEnd - dbBegin - } - } - if shd, ok = tbSharder[k]; ok { - tbMetadata = &rule.ShardMetadata{ - Computer: shd, - Stepper: rule.DefaultNumberStepper, - } - if s, ok := tbSteps[k]; ok && s > 0 { - tbMetadata.Steps = s - } else if tbBegin >= 0 && tbEnd >= 0 { - tbMetadata.Steps = 1 + tbEnd - tbBegin - } + for j := 0; j < tblsPerDB; j++ { + y = append(y, tbBegin+tblsPerDB*i+j) } - vt.SetShardMetadata(k, dbMetadata, tbMetadata) - rawShardMetadata := rule.RawShardMetadata{ - Name: table.Name, - SequenceType: table.Sequence.Type, - DbRules: dbRules, - TblRules: tblRules, - } - rawShardMetadata.Attributes = make(map[string]interface{}) - for k := range table.Attributes { - rawShardMetadata.Attributes[k] = table.Attributes[k] - } - vt.SetRawShardMetaData(&rawShardMetadata) + topology.SetTopology(x, y...) + } - tpRes := make(map[int][]int) - step := tbMetadata.Steps - if dbMetadata.Steps > step { - step = dbMetadata.Steps - } - rng, _ := tbMetadata.Stepper.Ascend(0, step) - for rng.HasNext() { - var ( - seed = rng.Next() - dbIdx = -1 - tbIdx = -1 - ) - if dbMetadata != nil { - if dbIdx, err = dbMetadata.Computer.Compute(seed); err != nil { - return nil, errors.WithStack(err) - } - } - if tbMetadata != nil { - if tbIdx, err = tbMetadata.Computer.Compute(seed); err != nil { - return nil, errors.WithStack(err) - } - } - tpRes[dbIdx] = append(tpRes[dbIdx], tbIdx) - } + defaultSteps := math.Max(dbAmount, tbAmount) - for dbIndex, tbIndexes := range tpRes { - topology.SetTopology(dbIndex, tbIndexes...) + dbSm, err := toShardMetadata(table.DbRules, defaultSteps) + if err != nil { + return nil, errors.Wrap(err, "cannot parse db rules") + } + tbSm, err := toShardMetadata(table.TblRules, defaultSteps) + if err != nil { + return nil, errors.Wrap(err, "cannot parse table rules") + } + + for i := 0; i < math.Max(len(dbSm), len(tbSm)); i++ { + var vs rule.VShard + if i < len(dbSm) { + vs.DB = dbSm[i] } + if i < len(tbSm) { + vs.Table = tbSm[i] + } + vt.AddVShards(&vs) } allowFullScan, err := strconv.ParseBool(table.Attributes["allow_full_scan"]) @@ -215,3 +185,75 @@ func parseDatabaseAndTable(name string) (db, tbl string, err error) { } return } + +func toSharder(input *config.Rule) (rule.ShardComputer, error) { + columns := make([]string, 0, len(input.Columns)) + for i := range input.Columns { + columns = append(columns, input.Columns[i].Name) + } + return rule.NewComputer(input.Type, columns, input.Expr) +} + +func getRender(format string) func(int) string { + if strings.ContainsRune(format, '%') { + return func(i int) string { + return fmt.Sprintf(format, i) + } + } + return func(i int) string { + return format + } +} + +func toShardMetadata(rules []*config.Rule, defaultSteps int) ([]*rule.ShardMetadata, error) { + toShardColumn := func(ru *config.ColumnRule, dst *[]*rule.ShardColumn, defaultSteps int) { + unit := rule.Unum + switch strings.ToLower(ru.Type) { + case "string", "str": + unit = rule.Ustr + case "year": + unit = rule.Uyear + case "month": + unit = rule.Umonth + case "week": + unit = rule.Uweek + case "day": + unit = rule.Uday + case "hour": + unit = rule.Uhour + } + c := &rule.ShardColumn{ + Name: ru.Name, + Steps: ru.Step, + Stepper: rule.Stepper{ + N: 1, + U: unit, + }, + } + + if c.Steps == 0 { + c.Steps = defaultSteps + } + + *dst = append(*dst, c) + } + + var ret []*rule.ShardMetadata + for i := range rules { + ru := rules[i] + c, err := toSharder(ru) + if err != nil { + return nil, errors.WithStack(err) + } + + var shardColumns []*rule.ShardColumn + for _, next := range ru.Columns { + toShardColumn(next, &shardColumns, defaultSteps) + } + ret = append(ret, &rule.ShardMetadata{ + ShardColumns: shardColumns, + Computer: c, + }) + } + return ret, nil +} diff --git a/pkg/config/equals.go b/pkg/config/equals.go index 30432357..d8fc4ce3 100644 --- a/pkg/config/equals.go +++ b/pkg/config/equals.go @@ -20,6 +20,7 @@ package config import ( "reflect" + "strings" ) func (u *User) Equals(o *User) bool { @@ -74,21 +75,33 @@ func (r Rules) Equals(o Rules) bool { newTmp := map[string]*Rule{} oldTmp := map[string]*Rule{} + toKey := func(r *Rule) string { + var sb strings.Builder + if len(r.Columns) > 0 { + sb.WriteString(r.Columns[0].Name) + for i := 1; i < len(r.Columns); i++ { + sb.WriteByte('+') + sb.WriteString(r.Columns[i].Name) + } + } + return sb.String() + } + for i := range r { - newTmp[r[i].Column] = r[i] + newTmp[toKey(r[i])] = r[i] } for i := range o { - oldTmp[o[i].Column] = o[i] + oldTmp[toKey(o[i])] = o[i] } for i := range r { - if _, ok := oldTmp[o[i].Column]; !ok { + if _, ok := oldTmp[toKey(o[i])]; !ok { newT = append(newT, o[i]) } } for i := range o { - val, ok := newTmp[o[i].Column] + val, ok := newTmp[toKey(o[i])] if !ok { deleteT = append(deleteT, o[i]) continue diff --git a/pkg/config/model.go b/pkg/config/model.go index 6c9084ec..4922d1ab 100644 --- a/pkg/config/model.go +++ b/pkg/config/model.go @@ -184,7 +184,7 @@ type ( Sequence *Sequence `yaml:"sequence" json:"sequence"` DbRules []*Rule `yaml:"db_rules" json:"db_rules"` TblRules []*Rule `yaml:"tbl_rules" json:"tbl_rules"` - Topology *Topology `yaml:"topology" json:"topology"` + Topology *Topology `validate:"required" yaml:"topology" json:"topology"` ShadowTopology *Topology `yaml:"shadow_topology" json:"shadow_topology"` Attributes map[string]string `yaml:"attributes" json:"attributes"` } @@ -195,10 +195,15 @@ type ( } Rule struct { - Column string `validate:"required" yaml:"column" json:"column"` - Type string `validate:"required" yaml:"type" json:"type"` - Expr string `validate:"required" yaml:"expr" json:"expr"` - Step int `yaml:"step" json:"step"` + Columns []*ColumnRule `validate:"required" yaml:"columns" json:"columns"` + Type string `validate:"required" yaml:"type" json:"type"` + Expr string `validate:"required" yaml:"expr" json:"expr"` + } + + ColumnRule struct { + Name string `validate:"required" yaml:"name" json:"name"` + Type string `yaml:"type" json:"type"` + Step int `yaml:"step" json:"step"` } Topology struct { diff --git a/pkg/config/model_test.go b/pkg/config/model_test.go index 1bf507e8..ac74f891 100644 --- a/pkg/config/model_test.go +++ b/pkg/config/model_test.go @@ -92,11 +92,11 @@ func TestShardingRuleConf(t *testing.T) { assert.Equal(t, "employees.student", table.Name) assert.Len(t, table.DbRules, 1) - assert.Equal(t, "uid", table.DbRules[0].Column) + assert.Equal(t, "uid", table.DbRules[0].Columns[0].Name) assert.Equal(t, "parseInt($value % 32 / 8)", table.DbRules[0].Expr) assert.Len(t, table.TblRules, 1) - assert.Equal(t, "uid", table.TblRules[0].Column) + assert.Equal(t, "uid", table.TblRules[0].Columns[0].Name) assert.Equal(t, "$value % 32", table.TblRules[0].Expr) assert.Equal(t, "employees_${0000..0003}", table.Topology.DbPattern) @@ -105,7 +105,6 @@ func TestShardingRuleConf(t *testing.T) { // assert.Equal(t, "__test_student_${0000...0007}", table.ShadowTopology.TblPattern) assert.Len(t, table.Attributes, 2) assert.Equal(t, "true", table.Attributes["allow_full_scan"]) - } func TestUnmarshalTextForProtocolTypeNil(t *testing.T) { diff --git a/pkg/config/tenant.go b/pkg/config/tenant.go index bd85fe90..70834570 100644 --- a/pkg/config/tenant.go +++ b/pkg/config/tenant.go @@ -310,7 +310,7 @@ func (tp *tenantOperator) UpdateTenantUser(tenant, username, password, oldUserna if err := yaml.Unmarshal(prev, &users); err != nil { return errors.WithStack(err) } - var found = false + found := false for i := 0; i < len(users); i++ { if users[i].Username == oldUsername { users[i].Password = password diff --git a/pkg/proto/rule/rule.go b/pkg/proto/rule/rule.go index 9865a57f..2f212759 100644 --- a/pkg/proto/rule/rule.go +++ b/pkg/proto/rule/rule.go @@ -28,55 +28,104 @@ import ( "github.com/pkg/errors" ) +import ( + "github.com/arana-db/arana/pkg/proto" +) + +const ( + attrAllowFullScan byte = 0x01 +) + type ( + // ShardColumn represents the shard column. + ShardColumn struct { + Name string + Steps int + Stepper Stepper + } + // ShardMetadata represents the metadata of shards. ShardMetadata struct { - Steps int // steps - Stepper Stepper // stepper - Computer ShardComputer // compute shards + ShardColumns []*ShardColumn + Computer ShardComputer // compute shards } // ShardComputer computes the shard index from an input value. ShardComputer interface { + // Variables returns the variable names. + Variables() []string // Compute computes the shard index. - Compute(value interface{}) (int, error) + Compute(values ...proto.Value) (int, error) + } + + VShard struct { + sync.Once + DB, Table *ShardMetadata + variables []string } - DirectShardComputer func(interface{}) (int, error) + ShardComputerFactory interface { + Apply(columns []string, expr string) (ShardComputer, error) + } ) -func (d DirectShardComputer) Compute(value interface{}) (int, error) { - return d(value) -} +var _shardComputers map[string]ShardComputerFactory -const ( - attrAllowFullScan byte = 0x01 -) +type FuncShardComputerFactory func([]string, string) (ShardComputer, error) -type ( - // RawShardRule represents the raw database_rule and table_rule of shards. - RawShardRule struct { - Column string `json:"column"` - Type string `json:"type"` - Expr string `json:"expr"` - Step int `json:"step,omitempty"` +func (f FuncShardComputerFactory) Apply(columns []string, expr string) (ShardComputer, error) { + return f(columns, expr) +} + +func RegisterShardComputer(typ string, factory ShardComputerFactory) { + if _shardComputers == nil { + _shardComputers = make(map[string]ShardComputerFactory) } - // RawShardMetadata represents the raw metadata of shards. - RawShardMetadata struct { - Name string `json:"name,omitempty"` - SequenceType string `json:"sequence_type,omitempty"` - DbRules []*RawShardRule `json:"db_rules"` - TblRules []*RawShardRule `json:"tbl_rules"` - Attributes map[string]interface{} `json:"attributes,omitempty"` + _shardComputers[typ] = factory +} + +func NewComputer(typ string, columns []string, expr string) (ShardComputer, error) { + f, ok := _shardComputers[typ] + if !ok { + return nil, errors.Errorf("no such shard computer type '%s'", typ) } -) + return f.Apply(columns, expr) +} -func (r RawShardMetadata) JSONMarshal() (string, error) { - jsons, errs := json.Marshal(r) - if errs != nil { - return "", errors.Errorf("cannot marshal the shard metadata to json") +func (sm *ShardMetadata) GetShardColumn(name string) *ShardColumn { + for i := range sm.ShardColumns { + if sm.ShardColumns[i].Name == name { + return sm.ShardColumns[i] + } } - return string(jsons), nil + return nil +} + +func (vs *VShard) Len() int { + return len(vs.Variables()) +} + +func (vs *VShard) Variables() []string { + vs.Do(func() { + visits := make(map[string]struct{}) + + handle := func(vars []string) { + for i := range vars { + if _, ok := visits[vars[i]]; ok { + continue + } + visits[vars[i]] = struct{}{} + vs.variables = append(vs.variables, vars[i]) + } + } + if vs.DB != nil { + handle(vs.DB.Computer.Variables()) + } + if vs.Table != nil { + handle(vs.Table.Computer.Variables()) + } + }) + return vs.variables } // VTable represents a virtual/logical table. @@ -85,13 +134,8 @@ type VTable struct { name string // TODO: set name autoIncrement *AutoIncrement topology *Topology - shards map[string][2]*ShardMetadata // column -> [db shard metadata,table shard metadata] - rawShards *RawShardMetadata -} - -func (vt *VTable) HasColumn(column string) bool { - _, ok := vt.shards[column] - return ok + shards []*VShard + ext map[string]interface{} } func (vt *VTable) Name() string { @@ -115,44 +159,71 @@ func (vt *VTable) AllowFullScan() bool { return ret } -func (vt *VTable) GetShardKeys() []string { - keys := make([]string, 0, len(vt.shards)) - for k := range vt.shards { - keys = append(keys, k) +func (vt *VTable) HasVShard(keys ...string) bool { + _, ok := vt.SearchVShard(keys...) + return ok +} + +func (vt *VTable) SearchVShard(keys ...string) (*VShard, bool) { + vShards := vt.GetVShards() +L: + for i := range vShards { + variables := vShards[i].Variables() + if len(variables) != len(keys) { + continue + } + for j := range keys { + if variables[j] != keys[j] { + continue L + } + } + return vShards[i], true } - return keys + return nil, false } -func (vt *VTable) SetRawShardMetaData(rawData *RawShardMetadata) { - vt.rawShards = rawData + +func (vt *VTable) GetVShards() []*VShard { + return vt.shards } func (vt *VTable) GetShardMetaDataJSON() (map[string]string, error) { res := make(map[string]string) + + res["name"] = vt.Name() + res["sequence_type"] = vt.GetAutoIncrement().Type + var ( - val []byte - err error + dbRules []interface{} + tblRules []interface{} ) + for _, vs := range vt.GetVShards() { + dbShardColumns := make([]string, 0, len(vs.DB.ShardColumns)) + for i := range vs.DB.ShardColumns { + dbShardColumns = append(dbShardColumns, vs.DB.ShardColumns[i].Name) + } + tblShardColumns := make([]string, 0, len(vs.Table.ShardColumns)) + for i := range vs.Table.ShardColumns { + tblShardColumns = append(tblShardColumns, vs.Table.ShardColumns[i].Name) + } - res["name"] = vt.rawShards.Name - res["sequence_type"] = vt.rawShards.SequenceType - - val, err = json.Marshal(vt.rawShards.DbRules) - if err != nil { - return res, err + dbRules = append(dbRules, map[string]interface{}{ + "columns": dbShardColumns, + "expression": fmt.Sprintf("%s", vs.DB.Computer), + }) + tblRules = append(tblRules, map[string]interface{}{ + "columns": tblShardColumns, + "expression": fmt.Sprintf("%s", vs.Table.Computer), + }) } - res["db_rules"] = string(val) - val, err = json.Marshal(vt.rawShards.TblRules) - if err != nil { - return res, err - } - res["tbl_rules"] = string(val) + dbRulesJson, _ := json.Marshal(dbRules) + tblRulesJson, _ := json.Marshal(tblRules) - val, err = json.Marshal(vt.rawShards.Attributes) - if err != nil { - return res, err - } - res["attributes"] = string(val) + res["db_rules"] = string(dbRulesJson) + res["tbl_rules"] = string(tblRulesJson) + + b, _ := json.Marshal(vt.ext) + res["attributes"] = string(b) return res, nil } @@ -163,56 +234,55 @@ func (vt *VTable) Topology() *Topology { } // Shard returns the shard result. -func (vt *VTable) Shard(column string, value interface{}) (uint32 /* db */, uint32 /* table */, error) { - var ( - db, table int - err error - ) - sm, ok := vt.shards[column] - if !ok { - return 0, 0, errors.Errorf("no shard metadata for column %s", column) +func (vt *VTable) Shard(inputs map[string]proto.Value) (uint32 /* db */, uint32 /* table */, error) { + var bingo *VShard +L: + for i := range vt.shards { + for _, key := range vt.shards[i].Variables() { + if _, ok := inputs[key]; !ok { + continue L + } + } + if bingo != nil { + return 0, 0, errors.New("conflict vshards") + } + bingo = vt.shards[i] } - if sm[0] != nil { // compute the index of db - if db, err = sm[0].Computer.Compute(value); err != nil { - return 0, 0, errors.WithStack(err) - } + if bingo == nil { + return 0, 0, errors.Errorf("no available vshards") } - if sm[1] != nil { // compute the index of table - if table, err = sm[1].Computer.Compute(value); err != nil { - return 0, 0, errors.WithStack(err) + + compute := func(c ShardComputer) (int, error) { + args := make([]proto.Value, 0, len(c.Variables())) + for _, variable := range c.Variables() { + args = append(args, inputs[variable]) } + return c.Compute(args...) } - return uint32(db), uint32(table), nil -} + var ( + db, table int + err error + ) -// GetShardMetadata returns the shard metadata with given column. -func (vt *VTable) GetShardMetadata(column string) (db *ShardMetadata, tbl *ShardMetadata, ok bool) { - var exist [2]*ShardMetadata - if exist, ok = vt.shards[column]; !ok { - return + if bingo.DB != nil { + if db, err = compute(bingo.DB.Computer); err != nil { + return 0, 0, errors.Wrap(err, "cannot compute db shard") + } } - db, tbl = exist[0], exist[1] - x, y := vt.Topology().Len() - // fix steps - if db != nil && db.Steps == 0 { - db.Steps = x - } - if tbl != nil && tbl.Steps == 0 { - tbl.Steps = y + if bingo.Table != nil { + if table, err = compute(bingo.Table.Computer); err != nil { + return 0, 0, errors.Wrap(err, "cannot compute table shard") + } } - return + return uint32(db), uint32(table), nil } -// SetShardMetadata sets the shard metadata. -func (vt *VTable) SetShardMetadata(column string, dbShardMetadata, tblShardMetadata *ShardMetadata) { - if vt.shards == nil { - vt.shards = make(map[string][2]*ShardMetadata) - } - vt.shards[column] = [2]*ShardMetadata{dbShardMetadata, tblShardMetadata} +func (vt *VTable) AddVShards(shard *VShard) { + vt.shards = append(vt.shards, shard) } // SetTopology sets the topology. @@ -230,16 +300,6 @@ type Rule struct { vtabs map[string]*VTable // table name -> *VTable } -// HasColumn returns true if the table and columns exists. -func (ru *Rule) HasColumn(table, column string) bool { - vt, ok := ru.VTable(table) - if !ok { - return false - } - _, _, ok = vt.GetShardMetadata(column) - return ok -} - // Has return true if the table exists. func (ru *Rule) Has(table string) bool { if ru == nil { @@ -287,17 +347,6 @@ func (ru *Rule) VTables() map[string]*VTable { return ru.vtabs } -// DBRule returns all the database rule -func (ru *Rule) DBRule() map[string][]*RawShardRule { - ru.mu.RLock() - defer ru.mu.RUnlock() - allDBRule := make(map[string][]*RawShardRule, 10) - for _, tb := range ru.vtabs { - allDBRule[tb.name] = tb.rawShards.DbRules - } - return allDBRule -} - // MustVTable returns the VTable with given table name, panic if not exist. func (ru *Rule) MustVTable(name string) *VTable { v, ok := ru.VTable(name) diff --git a/pkg/proto/rule/rule_test.go b/pkg/proto/rule/rule_test.go index fac83639..e41bc621 100644 --- a/pkg/proto/rule/rule_test.go +++ b/pkg/proto/rule/rule_test.go @@ -19,7 +19,6 @@ package rule import ( "fmt" - "strconv" "testing" ) @@ -30,6 +29,7 @@ import ( ) import ( + "github.com/arana-db/arana/pkg/proto" "github.com/arana-db/arana/testdata" ) @@ -52,12 +52,25 @@ func TestRule(t *testing.T) { } ) - vtab.SetShardMetadata("uid", &ShardMetadata{ - Stepper: stepper, - Computer: c1, - }, &ShardMetadata{ - Stepper: stepper, - Computer: c2, + vtab.AddVShards(&VShard{ + DB: &ShardMetadata{ + ShardColumns: []*ShardColumn{ + { + Name: "uid", + Stepper: stepper, + }, + }, + Computer: c1, + }, + Table: &ShardMetadata{ + ShardColumns: []*ShardColumn{ + { + Name: "uid", + Stepper: stepper, + }, + }, + Computer: c2, + }, }) // table topology: 4 databases, 16 tables @@ -84,65 +97,55 @@ func TestRule(t *testing.T) { c1 := testdata.NewMockShardComputer(ctrl) c1.EXPECT(). Compute(gomock.Any()). - DoAndReturn(func(value interface{}) (int, error) { - return value.(int) % 4, nil + DoAndReturn(func(value proto.Value) (int, error) { + x, _ := value.Int64() + return int(x) % 16 / 4, nil }). MinTimes(1) + c1.EXPECT().Variables().Return([]string{"uid"}).MinTimes(1) c2 := testdata.NewMockShardComputer(ctrl) c2.EXPECT(). Compute(gomock.Any()). - DoAndReturn(func(value interface{}) (int, error) { - return value.(int) % 16, nil + DoAndReturn(func(value proto.Value) (int, error) { + x, _ := value.Int64() + return int(x % 16), nil }). MinTimes(1) + c2.EXPECT().Variables().Return([]string{"uid"}).MinTimes(1) ru := buildRule(c1, c2) assert.True(t, ru.Has("student")) assert.False(t, ru.Has("fake_table")) - assert.True(t, ru.HasColumn("student", "uid")) - assert.False(t, ru.HasColumn("student", "fake_field")) - assert.False(t, ru.HasColumn("fake_table", "uid")) - assert.False(t, ru.HasColumn("fake_table", "fake_field")) + assert.True(t, ru.MustVTable("student").HasVShard("uid")) + assert.False(t, ru.MustVTable("student").HasVShard("fake_field")) vtab := ru.MustVTable("student") - var ok bool - _, _, ok = vtab.GetShardMetadata("name") - assert.False(t, ok) - _, _, ok = vtab.GetShardMetadata("fake_field") - assert.False(t, ok) - _, _, ok = vtab.GetShardMetadata("uid") - assert.True(t, ok) - - assert.Len(t, vtab.GetShardKeys(), 1, "length shard keys should be 1") + assert.Equal(t, 1, vtab.GetVShards()[0].Len(), "length shard keys should be 1") - dbIdx, tblIdx, err := vtab.Shard("uid", 42) + dbIdx, tblIdx, err := vtab.Shard(map[string]proto.Value{ + "uid": proto.NewValueInt64(42), + }) assert.NoError(t, err) assert.Equal(t, uint32(2), dbIdx) assert.Equal(t, uint32(10), tblIdx) db, tbl, ok := vtab.Topology().Render(int(dbIdx), int(tblIdx)) assert.True(t, ok) + assert.Equal(t, "school_0002", db) + assert.Equal(t, "student_0010", tbl) t.Logf("shard result: %s.%s\n", db, tbl) - _, _, noDataErr := vtab.Shard("name", 42) - assert.Error(t, noDataErr) + _, _, err = vtab.Shard(map[string]proto.Value{ + "name": proto.NewValueInt64(42), + }) + assert.Error(t, err) ru.RemoveVTable("student") assert.False(t, ru.Has("student")) assert.False(t, (*Rule)(nil).Has("student")) } - -func TestDirectShardComputer_Compute(t *testing.T) { - dc := DirectShardComputer(func(i interface{}) (int, error) { - n, _ := strconv.Atoi(fmt.Sprintf("%v", i)) - return n % 32, nil - }) - res, err := dc.Compute(33) - assert.NoError(t, err) - assert.Equal(t, 1, res, "should compute correctly") -} diff --git a/pkg/proto/rule/shards.go b/pkg/proto/rule/shards.go index db0d9fdd..d4b171b5 100644 --- a/pkg/proto/rule/shards.go +++ b/pkg/proto/rule/shards.go @@ -91,6 +91,9 @@ func (sd *Shards) Add(db, table uint32, otherTables ...uint32) { } func (sd *Shards) Each(h func(db, tb uint32) bool) { + if sd == nil { + return + } (*btree.BTree)(sd).Ascend(func(i btree.Item) bool { s := i.(shard) return h(s.getDB(), s.getTable()) @@ -122,7 +125,12 @@ func (sd *Shards) Len() int { } func (sd *Shards) String() string { + if sd == nil { + return "*" + } + var sb strings.Builder + sb.WriteByte('[') prev := int64(-1) sd.Each(func(db, tb uint32) bool { if prev != int64(db) { @@ -138,6 +146,7 @@ func (sd *Shards) String() string { _, _ = fmt.Fprint(&sb, tb) return true }) + sb.WriteByte(']') return sb.String() } diff --git a/pkg/proto/rule/topology.go b/pkg/proto/rule/topology.go index e3f77f43..31e22ef9 100644 --- a/pkg/proto/rule/topology.go +++ b/pkg/proto/rule/topology.go @@ -23,6 +23,10 @@ import ( "sync" ) +import ( + "golang.org/x/exp/slices" +) + // Topology represents the topology of databases and tables. type Topology struct { mu sync.RWMutex @@ -104,6 +108,16 @@ func (to *Topology) Enumerate() DatabaseTables { return dt } +func (to *Topology) Exists(dbIdx, tbIdx int) bool { + exist, ok := to.idx.Load(dbIdx) + if !ok { + return false + } + + _, ok = slices.BinarySearch(exist.([]int), tbIdx) + return ok +} + // Each enumerates items in current Topology. func (to *Topology) Each(onEach func(dbIdx, tbIdx int) (ok bool)) bool { done := true diff --git a/pkg/proto/value.go b/pkg/proto/value.go index ea1d205a..ffea8d33 100644 --- a/pkg/proto/value.go +++ b/pkg/proto/value.go @@ -98,6 +98,14 @@ var ( _ Value = (*timeValue)(nil) ) +func MustNewValue(input interface{}) Value { + v, err := NewValue(input) + if err != nil { + panic(err.Error()) + } + return v +} + func NewValue(input interface{}) (Value, error) { if input == nil { return nil, nil diff --git a/pkg/registry/nacos/client.go b/pkg/registry/nacos/client.go index 1de65d15..7afe1b37 100644 --- a/pkg/registry/nacos/client.go +++ b/pkg/registry/nacos/client.go @@ -45,7 +45,7 @@ type NacosServiceClient struct { } func NewNacosServiceClient(options map[string]interface{}) (*NacosServiceClient, error) { - var nsc = &NacosServiceClient{} + nsc := &NacosServiceClient{} if val, ok := options[u_conf.NamespaceIdKey]; ok { nsc.NamespaceId = val.(string) diff --git a/pkg/runtime/ast/ast.go b/pkg/runtime/ast/ast.go index 00828728..443581ac 100644 --- a/pkg/runtime/ast/ast.go +++ b/pkg/runtime/ast/ast.go @@ -39,7 +39,6 @@ import ( "github.com/arana-db/arana/pkg/proto" "github.com/arana-db/arana/pkg/proto/hint" "github.com/arana-db/arana/pkg/runtime/cmp" - "github.com/arana-db/arana/pkg/runtime/logical" ) var _opcode2comparison = map[opcode.Op]cmp.Comparison{ @@ -1594,13 +1593,12 @@ func (cc *convCtx) convBinaryOperationExpr(expr *ast.BinaryOperationExpr) interf } case opcode.LogicAnd: return &LogicalExpressionNode{ - Op: logical.Land, Left: toExpressionNode(left), Right: toExpressionNode(right), } case opcode.LogicOr: return &LogicalExpressionNode{ - Op: logical.Lor, + Or: true, Left: toExpressionNode(left), Right: toExpressionNode(right), } diff --git a/pkg/runtime/ast/create_table.go b/pkg/runtime/ast/create_table.go index 12a39dbe..c8315624 100644 --- a/pkg/runtime/ast/create_table.go +++ b/pkg/runtime/ast/create_table.go @@ -32,19 +32,19 @@ var _ Statement = (*CreateTableStmt)(nil) type CreateTableStmt struct { IfNotExists bool - //TemporaryKeyword + // TemporaryKeyword // Meanless when TemporaryKeyword is not TemporaryGlobal. // ON COMMIT DELETE ROWS => true // ON COMMIT PRESERVE ROW => false - //OnCommitDelete bool + // OnCommitDelete bool Table *TableName ReferTable *TableName Cols []*ast.ColumnDef Constraints []*ast.Constraint Options []*ast.TableOption - //Partition *PartitionOptions - //OnDuplicate OnDuplicateKeyHandlingType - //Select ResultSetNode + // Partition *PartitionOptions + // OnDuplicate OnDuplicateKeyHandlingType + // Select ResultSetNode } func NewCreateTableStmt() *CreateTableStmt { diff --git a/pkg/runtime/ast/expression.go b/pkg/runtime/ast/expression.go index f2c245cf..9d9a85ce 100644 --- a/pkg/runtime/ast/expression.go +++ b/pkg/runtime/ast/expression.go @@ -25,10 +25,6 @@ import ( "github.com/pkg/errors" ) -import ( - "github.com/arana-db/arana/pkg/runtime/logical" -) - const ( _ ExpressionMode = iota EmLogical @@ -51,7 +47,7 @@ type ExpressionNode interface { } type LogicalExpressionNode struct { - Op logical.Op + Or bool Left ExpressionNode Right ExpressionNode } @@ -65,13 +61,10 @@ func (l *LogicalExpressionNode) Restore(flag RestoreFlag, sb *strings.Builder, a return errors.WithStack(err) } - switch l.Op { - case logical.Land: - sb.WriteString(" AND ") - case logical.Lor: + if l.Or { sb.WriteString(" OR ") - default: - panic("unreachable") + } else { + sb.WriteString(" AND ") } if err := l.Right.Restore(flag, sb, args); err != nil { diff --git a/pkg/runtime/ast/insert.go b/pkg/runtime/ast/insert.go index fccf9692..fbde92f8 100644 --- a/pkg/runtime/ast/insert.go +++ b/pkg/runtime/ast/insert.go @@ -47,6 +47,14 @@ type baseInsertStatement struct { Hint *HintNode } +func (b *baseInsertStatement) restoreHint(sb *strings.Builder) { + if b.Hint == nil { + return + } + _ = b.Hint.Restore(0, sb, nil) + sb.WriteString(" ") +} + func (b *baseInsertStatement) IsSetSyntax() bool { return b.flag&_flagInsertSetSyntax != 0 } @@ -142,12 +150,7 @@ func NewInsertStatement(table TableName, columns []string) *InsertStatement { func (is *InsertStatement) Restore(flag RestoreFlag, sb *strings.Builder, args *[]int) error { sb.WriteString("INSERT ") - if is.Hint != nil { - if err := is.Hint.Restore(flag, sb, args); err != nil { - return errors.WithStack(err) - } - sb.WriteString(" ") - } + is.restoreHint(sb) // write priority if is.IsLowPriority() { @@ -284,12 +287,7 @@ type InsertSelectStatement struct { func (is *InsertSelectStatement) Restore(flag RestoreFlag, sb *strings.Builder, args *[]int) error { sb.WriteString("INSERT ") - if is.Hint != nil { - if err := is.Hint.Restore(flag, sb, args); err != nil { - return errors.WithStack(err) - } - sb.WriteString(" ") - } + is.restoreHint(sb) // write priority if is.IsLowPriority() { diff --git a/pkg/runtime/ast/select_element.go b/pkg/runtime/ast/select_element.go index 97ba0814..53f7cbe8 100644 --- a/pkg/runtime/ast/select_element.go +++ b/pkg/runtime/ast/select_element.go @@ -304,6 +304,7 @@ func NewSelectElementExpr(expr ExpressionNode, alias string) *SelectElementExpr alias: alias, } } + func NewSelectElementExprFull(expr ExpressionNode, alias string, originalText string) *SelectElementExpr { return &SelectElementExpr{ inner: expr, diff --git a/pkg/runtime/ast/show.go b/pkg/runtime/ast/show.go index 82a3799c..4b2c7997 100644 --- a/pkg/runtime/ast/show.go +++ b/pkg/runtime/ast/show.go @@ -637,6 +637,7 @@ type ShowShardingTable struct { func (s *ShowShardingTable) Mode() SQLType { return SQLTypeShowShardingTable } + func (s *ShowShardingTable) Restore(flag RestoreFlag, sb *strings.Builder, args *[]int) error { val, ok := s.BaseShow.filter.(string) if !ok { diff --git a/pkg/runtime/builtin/shard_computer_javascript.go b/pkg/runtime/builtin/shard_computer_javascript.go new file mode 100644 index 00000000..c5efd1ef --- /dev/null +++ b/pkg/runtime/builtin/shard_computer_javascript.go @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package builtin + +import ( + "fmt" + "log" + "runtime" + "strings" +) + +import ( + "github.com/dop251/goja" + + "github.com/pkg/errors" +) + +import ( + "github.com/arana-db/arana/pkg/proto" + "github.com/arana-db/arana/pkg/proto/rule" +) + +var _ rule.ShardComputer = (*jsShardComputer)(nil) + +func init() { + f := rule.FuncShardComputerFactory(func(columns []string, script string) (rule.ShardComputer, error) { + return NewJavascriptShardComputer(script, columns[0], columns[1:]...) + }) + rule.RegisterShardComputer("", f) + rule.RegisterShardComputer("javascript", f) + rule.RegisterShardComputer("js", f) +} + +const _jsEntrypoint = "__compute__" // shard method name + +type jsShardComputer struct { + // runtime is not thread-safe, wrap as leaky buffer. + // please see https://go.dev/doc/effective_go#leaky_buffer + freelist chan *goja.Runtime + script string + variables []string +} + +func (j *jsShardComputer) String() string { + return j.script +} + +// MustNewJavascriptShardComputer returns a shard computer which is based on Javascript, panic if failed. +func MustNewJavascriptShardComputer(script string, column string, otherColumns ...string) rule.ShardComputer { + c, err := NewJavascriptShardComputer(script, column, otherColumns...) + if err != nil { + panic(err.Error()) + } + return c +} + +// NewJavascriptShardComputer returns a shard computer which is based on Javascript. +func NewJavascriptShardComputer(script string, column string, otherColumns ...string) (rule.ShardComputer, error) { + ret := &jsShardComputer{ + freelist: make(chan *goja.Runtime, runtime.NumCPU()*2), + script: generateJavaScript(script, len(otherColumns)+1), + } + ret.variables = make([]string, 0, len(otherColumns)+1) + ret.variables = append(ret.variables, column) + ret.variables = append(ret.variables, otherColumns...) + + vm, err := createVM(ret.script) + if err != nil { + return nil, errors.Wrapf(err, "failed to create javascript shard computer") + } + + ret.freelist <- vm + + return ret, nil +} + +func (j *jsShardComputer) Variables() []string { + return j.variables +} + +func (j *jsShardComputer) Compute(values ...proto.Value) (int, error) { + if len(j.variables) != len(values) { + return 0, errors.Errorf("the length of params doesn't match: expect=%d, actual=%d", len(j.variables), len(values)) + } + + vm, err := j.getVM() + if err != nil { + return 0, err + } + + defer func() { + j.putVM(vm) + }() + + inputs := make([]goja.Value, 0, len(values)) + for i := range values { + var next interface{} + switch values[i].Family() { + case proto.ValueFamilySign: + next, _ = values[i].Int64() + case proto.ValueFamilyUnsigned: + next, _ = values[i].Uint64() + case proto.ValueFamilyFloat: + next, _ = values[i].Float64() + case proto.ValueFamilyDecimal: + next, _ = values[i].Float64() + case proto.ValueFamilyBool: + next, _ = values[i].Bool() + default: + next = values[i].String() + } + + inputs = append(inputs, vm.ToValue(next)) + } + + fn, _ := goja.AssertFunction(vm.Get(_jsEntrypoint)) + res, err := fn(goja.Undefined(), inputs...) + if err != nil { + return 0, errors.WithStack(err) + } + + return int(res.ToInteger()), nil +} + +func (j *jsShardComputer) getVM() (*goja.Runtime, error) { + select { + case next := <-j.freelist: + return next, nil + default: + return createVM(j.script) + } +} + +func (j *jsShardComputer) putVM(vm *goja.Runtime) { + select { + case j.freelist <- vm: + default: + } +} + +func generateJavaScript(script string, columnLen int) string { + // will generate normalized javascript: + // + // INPUT: + // columns: + // - school_id + // - uid + // script: ($0*31+$1) % 32 + // + // OUTPUT: + // function($0,$1) { // $0=school_id, $1=uid + // return ($0*31+$1) % 32; + // } + var sb strings.Builder + sb.WriteString("function ") + sb.WriteString(_jsEntrypoint) + sb.WriteString("(") + + if columnLen > 0 { + sb.WriteString("$0") + for i := 1; i < columnLen; i++ { + _, _ = fmt.Fprintf(&sb, ",$%d", i) + } + } + + sb.WriteString(") {\n") + + // NOTICE: compatibility with prev version expression + script = strings.NewReplacer("$value", "$0").Replace(script) + + if strings.Contains(script, "return ") { + sb.WriteString(script) + } else { + sb.WriteString("return ") + sb.WriteString(script) + } + + sb.WriteString("\n}") + + return sb.String() +} + +func createVM(script string) (*goja.Runtime, error) { + vm := goja.New() + if _, err := vm.RunString(script); err != nil { + return nil, errors.WithStack(err) + } + + // TODO: add prelude functions, includes some hash/utils + _ = vm.Set("log", func(format string, args ...interface{}) { + log.Printf(format+"\n", args...) + }) + + return vm, nil +} diff --git a/pkg/runtime/rule/shard_script_test.go b/pkg/runtime/builtin/shard_computer_javascript_test.go similarity index 60% rename from pkg/runtime/rule/shard_script_test.go rename to pkg/runtime/builtin/shard_computer_javascript_test.go index d4c771fb..bd75968c 100644 --- a/pkg/runtime/rule/shard_script_test.go +++ b/pkg/runtime/builtin/shard_computer_javascript_test.go @@ -15,10 +15,9 @@ * limitations under the License. */ -package rule +package builtin import ( - "fmt" "strconv" "testing" ) @@ -27,8 +26,12 @@ import ( "github.com/stretchr/testify/assert" ) +import ( + "github.com/arana-db/arana/pkg/proto" +) + func TestBadScript(t *testing.T) { - _, err := NewJavascriptShardComputer(")))BAD(((") + _, err := NewJavascriptShardComputer(")))BAD(((", "fake") assert.Error(t, err) } @@ -38,24 +41,25 @@ if ($value < 0) { throw new Error('oops'); } return $value; -`) +`, "fake") assert.NoError(t, err) - _, err = c.Compute(0) + assert.Equal(t, []string{"fake"}, c.Variables()) + _, err = c.Compute(proto.NewValueInt64(0)) assert.NoError(t, err) - _, err = c.Compute(-1) + _, err = c.Compute(proto.NewValueInt64(-1)) assert.Error(t, err) } func TestSimpleScript(t *testing.T) { // tables: 4*32 var ( - db, _ = NewJavascriptShardComputer("($value % 128) / 32") - tb, _ = NewJavascriptShardComputer("$value % 128") + db, _ = NewJavascriptShardComputer("($value % 128) / 32", "fake") + tb, _ = NewJavascriptShardComputer("$value % 128", "fake") ) type tt struct { - input int + input int64 db, tbl int } @@ -67,17 +71,28 @@ func TestSimpleScript(t *testing.T) { {128, 0, 0}, // DB_0000.TBL_0001 {129, 0, 1}, // DB_0000.TBL_0001 } { - t.Run(strconv.Itoa(it.input), func(t *testing.T) { - v, err := db.Compute(it.input) + t.Run(strconv.FormatInt(it.input, 10), func(t *testing.T) { + v, err := db.Compute(proto.NewValueInt64(it.input)) assert.NoError(t, err) assert.Equal(t, it.db, v) - v, err = tb.Compute(it.input) + v, err = tb.Compute(proto.NewValueInt64(it.input)) assert.NoError(t, err) assert.Equal(t, it.tbl, v) }) } } +func TestMultipleArguments(t *testing.T) { + script := `$1*31+$0` + + c, err := NewJavascriptShardComputer(script, "foo", "bar") + assert.NoError(t, err) + + val, err := c.Compute(proto.NewValueInt64(100), proto.NewValueInt64(7)) + assert.NoError(t, err) + assert.Equal(t, 7*31+100, val) +} + func TestComplexScript(t *testing.T) { script := ` // throw error if value is not string: @@ -92,28 +107,30 @@ if ($value.length < 8) { let n = parseInt($value.substring(2, 8)); if (isNaN(n)) { - return 0; + return 0; } return n%32; ` - c, err := NewJavascriptShardComputer(script) + c, err := NewJavascriptShardComputer(script, "fake") assert.NoError(t, err) + assert.Equal(t, []string{"fake"}, c.Variables()) type tt struct { + scene string input interface{} output int } for _, it := range []tt{ - {1234, 0}, // not string -> 0 - {"SN7777", 0}, // length<8 -> 0 - {"SN000042CN", 10}, // 42%32 -> 10 - {"SNxxxxxxJP", 0}, // NaN -> 0 + {"1234", 1234, 0}, // not string -> 0 + {"SN7777", "SN7777", 0}, // length<8 -> 0 + {"SN000042CN", "SN000042CN", 10}, // 42%32 -> 10 + {"SNxxxxxxJP", "SNxxxxxxJP", 0}, // NaN -> 0 } { - t.Run(fmt.Sprint(it.input), func(t *testing.T) { - actual, err := c.Compute(it.input) + t.Run(it.scene, func(t *testing.T) { + actual, err := c.Compute(proto.MustNewValue(it.input)) assert.NoError(t, err) assert.Equal(t, it.output, actual) }) @@ -121,14 +138,14 @@ return n%32; } func BenchmarkJavascriptShardComputer(b *testing.B) { - computer, _ := NewJavascriptShardComputer("$value % 32") - _, _ = computer.Compute(42) + computer, _ := NewJavascriptShardComputer("$value % 32", "fake") + _, _ = computer.Compute(proto.NewValueInt64(42)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, _ = computer.Compute(42) + _, _ = computer.Compute(proto.MustNewValue(42)) } }) } diff --git a/pkg/runtime/calc/calculus.go b/pkg/runtime/calc/calculus.go new file mode 100644 index 00000000..c00bdc5c --- /dev/null +++ b/pkg/runtime/calc/calculus.go @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package calc + +import ( + stdErrors "errors" + "strings" +) + +import ( + "github.com/pkg/errors" + + "golang.org/x/sync/errgroup" +) + +import ( + "github.com/arana-db/arana/pkg/proto" + "github.com/arana-db/arana/pkg/proto/rule" + "github.com/arana-db/arana/pkg/runtime/calc/logic" + "github.com/arana-db/arana/pkg/runtime/cmp" + "github.com/arana-db/arana/pkg/util/misc" +) + +var ErrNoShardMatched = stdErrors.New("no virtual shard matched") + +var Zero = &Calculus{ + s: rule.NewShards(), +} + +func Wrap(c *cmp.Comparative) logic.Logic[*Calculus] { + return logic.Wrap[*Calculus](&Calculus{ + c: c, + }) +} + +type Calculus struct { + c *cmp.Comparative + s *rule.Shards +} + +func (ca *Calculus) Compare(item logic.Item) int { + return compareComparative(ca.c, item.(*Calculus).c) +} + +func (ca *Calculus) String() string { + var sb strings.Builder + sb.WriteString("Calculus{") + if ca.c != nil { + sb.WriteString("c=") + sb.WriteString(ca.c.String()) + } + if ca.s != nil { + if ca.c != nil { + sb.WriteByte(',') + } + sb.WriteString("s=") + sb.WriteString(ca.s.String()) + } + sb.WriteByte('}') + return sb.String() +} + +type calculusOperator rule.VTable + +func (co *calculusOperator) AND(first *Calculus, others ...*Calculus) (*Calculus, error) { + groups := make(map[string]map[cmp.Comparison][]*Calculus) + add := func(c *Calculus) { + if c == nil { + return + } + + key := c.c.Key() + comparison := c.c.Comparison() + if _, ok := groups[key]; !ok { + groups[key] = map[cmp.Comparison][]*Calculus{ + comparison: {c}, + } + return + } + + if _, ok := groups[key][comparison]; !ok { + groups[key][comparison] = append(groups[key][comparison], c) + return + } + + switch comparison { + case cmp.Ceq: + case cmp.Cne: + groups[key][comparison] = append(groups[key][comparison], c) + case cmp.Cgt, cmp.Cgte: // minimum is the first value. + case cmp.Clt, cmp.Clte: // maximum is the last value. + groups[key][comparison][0] = c + } + } + add(first) + for i := range others { + add(others[i]) + } + + vShard := searchVShard((*rule.VTable)(co), groups) + + if vShard == nil { + return nil, ErrNoShardMatched + } + + type valuePair struct { + db, tbl []interface{} + } + + values := make(map[string]valuePair) + for i := range vShard.Variables() { + name := vShard.Variables()[i] + cm := calculusMap(groups[name]) + begin, end := cm.getRange() + switch { + case begin != nil && end != nil: + var vp valuePair + if vShard.DB != nil { + vp.db = computeRange(vShard.DB, begin.c, end.c) + } + if vShard.Table != nil { + vp.tbl = computeRange(vShard.Table, begin.c, end.c) + } + values[name] = vp + case begin != nil && end == nil: + var vp valuePair + if vShard.DB != nil { + vp.db = computeLRange(vShard.DB, begin.c) + } + if vShard.Table != nil { + vp.tbl = computeLRange(vShard.Table, begin.c) + } + values[name] = vp + case begin == nil && end != nil: + var vp valuePair + if vShard.DB != nil { + vp.db = computeRRange(vShard.DB, end.c) + } + if vShard.Table != nil { + vp.tbl = computeRRange(vShard.Table, end.c) + } + values[name] = vp + case begin == nil && end == nil: + if cm.has(cmp.Ceq) { + return Zero, nil + } + return nil, nil + } + } + + compute := func(computer rule.ShardComputer, dst *[]int, vals [][]interface{}) error { + var args []proto.Value + for _, next := range misc.CartesianProduct(vals) { + for i := range next { + arg, err := proto.NewValue(next[i]) + if err != nil { + return err + } + args = append(args, arg) + } + idx, err := computer.Compute(args...) + if err != nil { + return err + } + *dst = append(*dst, idx) + args = args[:0] + } + return nil + } + + computeDB := func(computer rule.ShardComputer, dst *[]int) error { + var vals [][]interface{} + for _, name := range computer.Variables() { + vals = append(vals, values[name].db) + } + return compute(computer, dst, vals) + } + + computeTable := func(computer rule.ShardComputer, dst *[]int) error { + var vals [][]interface{} + for _, name := range computer.Variables() { + vals = append(vals, values[name].tbl) + } + return compute(computer, dst, vals) + } + + var ( + g errgroup.Group + dbIndexes, tblIndexes []int + ) + + g.Go(func() error { + if vShard.DB == nil { + dbIndexes = append(dbIndexes, 0) + return nil + } + return computeDB(vShard.DB.Computer, &dbIndexes) + }) + g.Go(func() error { + if vShard.Table == nil { + tblIndexes = append(tblIndexes, 0) + return nil + } + return computeTable(vShard.Table.Computer, &tblIndexes) + }) + + if err := g.Wait(); err != nil { + return nil, err + } + + shards := rule.NewShards() + cp := misc.CartesianProduct[int]([][]int{dbIndexes, tblIndexes}) + topology := (*rule.VTable)(co).Topology() + for i := range cp { + x := cp[i][0] + y := cp[i][1] + if topology.Exists(x, y) { + shards.Add(uint32(x), uint32(y)) + } + } + + if shards.Len() < 1 { + return Zero, nil + } + + return &Calculus{ + s: shards, + }, nil +} + +func (co calculusOperator) OR(first *Calculus, others ...*Calculus) (*Calculus, error) { + if first == nil { + return nil, nil + } + + shards := rule.NewShards() + + if first.c != nil { + var err error + if first, err = co.AND(first); err != nil { + return nil, err + } + } + + if first.s != nil { + first.s.Each(func(db, tb uint32) bool { + shards.Add(db, tb) + return true + }) + } + + for _, next := range others { + if next == nil { + return nil, nil + } + if next.c != nil { + var err error + if next, err = co.AND(next); err != nil { + return nil, err + } + } + if next.s != nil { + next.s.Each(func(db, tb uint32) bool { + shards.Add(db, tb) + return true + }) + } + } + + return &Calculus{ + s: shards, + }, nil +} + +func (co calculusOperator) NOT(input *Calculus) (*Calculus, error) { + if input.c != nil { + var nextCmp cmp.Comparison + switch input.c.Comparison() { + case cmp.Ceq: + nextCmp = cmp.Cne + case cmp.Cne: + nextCmp = cmp.Ceq + case cmp.Cgt: + nextCmp = cmp.Clte + case cmp.Cgte: + nextCmp = cmp.Clt + case cmp.Clt: + nextCmp = cmp.Cgte + case cmp.Clte: + nextCmp = cmp.Cgt + default: + panic("unreachable") + } + + c := new(cmp.Comparative) + *c = *input.c + c.SetComparison(nextCmp) + + return &Calculus{ + c: c, + }, nil + } + _ = input + panic("implement me") +} + +type calculusMap map[cmp.Comparison][]*Calculus + +func (cm calculusMap) has(c cmp.Comparison) bool { + _, ok := cm[c] + return ok +} + +func (cm calculusMap) getRange() (begin, end *Calculus) { + if eq, hasEq := cm[cmp.Ceq]; hasEq { + if neList, hasNe := cm[cmp.Cne]; hasNe { + for _, ne := range neList { + if ne.c == nil { + continue + } + // a == 1 && a <> 1 + if compareComparativeValue(ne.c, eq[0].c) == 0 { + return + } + } + } + + if gte, hasGte := cm[cmp.Cgte]; hasGte { + switch compareComparativeValue(eq[0].c, gte[0].c) { + // a==1 && a>=2 ---> NaN + case -1: + return + } + } + if gt, hasGt := cm[cmp.Cgt]; hasGt { + switch compareComparativeValue(eq[0].c, gt[0].c) { + // a==2 && a>2 ---> NaN + // a==1 && a>2 ---> NaN + case 0, -1: + return + } + } + + if lte, hasLte := cm[cmp.Clte]; hasLte { + switch compareComparativeValue(eq[0].c, lte[0].c) { + // a==2 && a<1 ---> NaN + case 1: + return + } + } + + if lt, hasLt := cm[cmp.Clt]; hasLt { + switch compareComparativeValue(eq[0].c, lt[0].c) { + // a==2 && a<2 ---> NaN + // a==2 && a<1 ---> NaN + case 0, 1: + return + } + } + + begin = eq[0] + + return + } + + gte, hasGte := cm[cmp.Cgte] + gt, hasGt := cm[cmp.Cgt] + switch { + case hasGte && hasGt: + switch compareComparativeValue(gte[0].c, gt[0].c) { + // a >= 1 && a > 1 ---> a > 1 + // a >= 1 && a > 2 ---> a > 2 + case 0, -1: + begin = gt[0] + // a >= 2 && a > 1 ---> a >= 2 + case 1: + begin = gte[0] + } + case hasGte: + begin = gte[0] + case hasGt: + begin = gt[0] + } + + lte, hasLte := cm[cmp.Clte] + lt, hasLt := cm[cmp.Clt] + + switch { + case hasLte && hasLt: + switch compareComparativeValue(lte[0].c, lt[0].c) { + // a<=1 && a<1 ---> a < 1 + // a<=2 && a<1 ---> a < 1 + case 0, 1: + end = lt[0] + // a<=1 && a<2 ---> a <= 1 + case -1: + end = lte[0] + } + case hasLte: + end = lte[0] + case hasLt: + end = lt[0] + } + + return +} + +func Eval(vtab *rule.VTable, l logic.Logic[*Calculus]) (*rule.Shards, error) { + sh, err := innerEval(vtab, l) + if err == nil { + return sh, nil + } + if errors.Is(err, logic.ErrEmptyIntersection) { + return rule.NewShards(), nil + } + if errors.Is(err, logic.ErrEmptyUnion) { + return nil, nil + } + return nil, err +} + +func innerEval(vtab *rule.VTable, l logic.Logic[*Calculus]) (*rule.Shards, error) { + c, err := l.Eval((*calculusOperator)(vtab)) + if err != nil { + return nil, err + } + + if c == nil { + return nil, nil + } + + if c.s != nil { + return c.s, nil + } + + if c, err = (*calculusOperator)(vtab).AND(c); err != nil { + return nil, err + } + + if c == nil { + return nil, nil + } + + return c.s, nil +} diff --git a/pkg/runtime/calc/calculus_test.go b/pkg/runtime/calc/calculus_test.go new file mode 100644 index 00000000..08362ae1 --- /dev/null +++ b/pkg/runtime/calc/calculus_test.go @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package calc + +import ( + "testing" +) + +import ( + "github.com/pkg/errors" + + "github.com/stretchr/testify/assert" +) + +import ( + "github.com/arana-db/arana/pkg/proto/rule" + rrule "github.com/arana-db/arana/pkg/runtime/builtin" + "github.com/arana-db/arana/pkg/runtime/calc/logic" + "github.com/arana-db/arana/pkg/runtime/cmp" +) + +func getMultipleKeysVTab() *rule.VTable { + var vtab rule.VTable + var topology rule.Topology + topology.SetTopology(0, 0, 1, 2, 3) + topology.SetTopology(1, 4, 5, 6, 7) + topology.SetTopology(2, 8, 9, 10, 11) + topology.SetTopology(3, 12, 13, 14, 15) + vtab.SetTopology(&topology) + + var vs rule.VShard + vs.DB = &rule.ShardMetadata{ + ShardColumns: []*rule.ShardColumn{ + { + Name: "uid", + Steps: 16, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + { + Name: "sid", + Steps: 16, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + }, + + Computer: rrule.MustNewJavascriptShardComputer("parseInt((($0*31+$1) % 16)/4)", "uid", "sid"), + } + vs.Table = &rule.ShardMetadata{ + ShardColumns: []*rule.ShardColumn{ + { + Name: "uid", + Steps: 16, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + { + Name: "sid", + Steps: 16, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + }, + Computer: rrule.MustNewJavascriptShardComputer("parseInt(($0*31+$1) % 16)", "uid", "sid"), + } + vtab.AddVShards(&vs) + + return &vtab +} + +func getSingleKeyVTab() *rule.VTable { + var vtab rule.VTable + var topology rule.Topology + topology.SetTopology(0, 0, 1, 2, 3) + topology.SetTopology(1, 4, 5, 6, 7) + topology.SetTopology(2, 8, 9, 10, 11) + topology.SetTopology(3, 12, 13, 14, 15) + vtab.SetTopology(&topology) + + var vs rule.VShard + vs.DB = &rule.ShardMetadata{ + ShardColumns: []*rule.ShardColumn{ + { + Name: "uid", + Steps: 16, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + }, + + Computer: rrule.MustNewJavascriptShardComputer("parseInt(($0 % 16)/4)", "uid"), + } + vs.Table = &rule.ShardMetadata{ + ShardColumns: []*rule.ShardColumn{ + { + Name: "uid", + Steps: 16, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + }, + Computer: rrule.MustNewJavascriptShardComputer("parseInt($0 % 16)", "uid"), + } + vtab.AddVShards(&vs) + + return &vtab +} + +func TestMultipleCalculus(t *testing.T) { + type tt struct { + scene string + input logic.Logic[*Calculus] + want string + } + + for _, next := range []tt{ + { + "uid = 8 and sid = 1", + logic.AND( + Wrap(cmp.NewInt64("uid", cmp.Ceq, 8)), + Wrap(cmp.NewInt64("sid", cmp.Ceq, 1)), + ), + + func() string { + const ( + uid uint32 = 8 + sid uint32 = 1 + ) + sh := rule.NewShards() + h := uid*31 + sid + sh.Add((h%16)/4, h%16) + return sh.String() + }(), + }, + { + "uid >= 8 and uid <= 10 and sid >= 1 and sid <= 3", + logic.AND( + logic.AND( + Wrap(cmp.NewInt64("uid", cmp.Cgte, 8)), + Wrap(cmp.NewInt64("uid", cmp.Clte, 10)), + ), + logic.AND( + Wrap(cmp.NewInt64("sid", cmp.Cgte, 1)), + Wrap(cmp.NewInt64("sid", cmp.Clte, 3)), + ), + ), + func() string { + shards := rule.NewShards() + for _, next := range [][2]int{ + {8, 1}, + {8, 2}, + {8, 3}, + {9, 1}, + {9, 2}, + {9, 3}, + {10, 1}, + {10, 2}, + {10, 3}, + } { + h := uint32(next[0]*31 + next[1]) + shards.Add((h%16)/4, h%16) + } + return shards.String() + }(), + }, + } { + t.Run(next.scene, func(t *testing.T) { + shards, err := Eval(getMultipleKeysVTab(), next.input) + assert.NoError(t, err) + t.Logf("scene=%s, logic=%s, shards=%s\n", next.scene, next.input, shards.String()) + assert.Equal(t, next.want, shards.String()) + }) + } +} + +func TestSingleCalculus(t *testing.T) { + type tt struct { + scene string + input logic.Logic[*Calculus] + want string + } + + for _, next := range []tt{ + { + "uid = 8 and dummy = 1", + logic.AND( + Wrap(cmp.NewInt64("uid", cmp.Ceq, 8)), + Wrap(cmp.NewInt64("dummy", cmp.Ceq, 1)), + ), + "[2:8]", + }, + { + "uid > 8 and uid <= 10", + logic.AND( + Wrap(cmp.NewInt64("uid", cmp.Cgt, 8)), + Wrap(cmp.NewInt64("uid", cmp.Clte, 12)), + ), + "[2:9,10,11;3:12]", + }, + { + "uid = 1 or uid = 2 or uid = 3", + logic.OR( + logic.OR( + Wrap(cmp.NewInt64("uid", cmp.Ceq, 1)), + Wrap(cmp.NewInt64("uid", cmp.Ceq, 2)), + ), + Wrap(cmp.NewInt64("uid", cmp.Ceq, 3)), + ), + "[0:1,2,3]", + }, + { + "a = 1", + Wrap(cmp.NewInt64("a", cmp.Ceq, 1)), + "*", + }, + { + "not (uid = 1 or uid = 2)", + logic.NOT(logic.OR( + Wrap(cmp.NewInt64("uid", cmp.Ceq, 1)), + Wrap(cmp.NewInt64("uid", cmp.Ceq, 2)), + )), + "*", + }, + { + "not uid = 1", + logic.NOT(Wrap(cmp.NewInt64("uid", cmp.Ceq, 1))), + "*", + }, + { + "uid == 1 and uid <> 1", + logic.AND( + Wrap(cmp.NewInt64("uid", cmp.Ceq, 1)), + Wrap(cmp.NewInt64("uid", cmp.Cne, 1)), + ), + "[]", + }, + { + "uid == 1 and not (uid == 1)", + logic.AND( + Wrap(cmp.NewInt64("uid", cmp.Ceq, 1)), + logic.NOT(Wrap(cmp.NewInt64("uid", cmp.Ceq, 1))), + ), + "[]", + }, + } { + t.Run(next.scene, func(t *testing.T) { + shards, err := Eval(getSingleKeyVTab(), next.input) + if errors.Is(err, ErrNoShardMatched) { + err = nil + } + assert.NoError(t, err) + t.Logf("scene=%s, logic=%s, shards=%s\n", next.scene, next.input, shards.String()) + assert.Equal(t, next.want, shards.String()) + }) + } +} diff --git a/pkg/runtime/calc/logic/bool.go b/pkg/runtime/calc/logic/bool.go new file mode 100644 index 00000000..ad50ce00 --- /dev/null +++ b/pkg/runtime/calc/logic/bool.go @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logic + +var _ Item = (*Bool)(nil) + +type Bool bool + +func (b Bool) Compare(c Item) int { + switch that := c.(type) { + case Bool: + var x, y int + if b { + x = 1 + } + if that { + y = 1 + } + switch { + case x < y: + return -1 + case x > y: + return 1 + default: + return 0 + } + default: + return -1 + } +} + +type BoolOperator struct{} + +func (b BoolOperator) AND(first Bool, others ...Bool) (Bool, error) { + if !first { + return false, nil + } + for i := range others { + if !others[i] { + return false, nil + } + } + return true, nil +} + +func (b BoolOperator) OR(first Bool, others ...Bool) (Bool, error) { + if first { + return true, nil + } + for i := range others { + if others[i] { + return true, nil + } + } + return false, nil +} + +func (b BoolOperator) NOT(input Bool) (Bool, error) { + return !input, nil +} diff --git a/pkg/runtime/calc/logic/logic.go b/pkg/runtime/calc/logic/logic.go new file mode 100644 index 00000000..b1548132 --- /dev/null +++ b/pkg/runtime/calc/logic/logic.go @@ -0,0 +1,981 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logic + +import ( + stdErrors "errors" + "fmt" + "strings" +) + +import ( + "github.com/google/btree" +) + +const ( + NONE = "" + ANY = "" +) + +var ( + SymbolAND = "&&" + SymbolOR = "||" + SymbolNOT = "!" +) + +var ( + ErrEmptyUnion = stdErrors.New("no union items found") + ErrEmptyIntersection = stdErrors.New("no intersection items found") +) + +type Item interface { + Compare(Item) int +} + +// Logic represents a logical target which compile to (.. ∩ .. ∩ ..) ∪ (.. ∩ .. ∩ ..) ∪ (.. ∩ .. ∩ ..) ... +// It follows the rules below: +// 1. A ∩ !A <=> 0 +// 2. A ∩ B <=> B ∩ A +// 3. A ∪ B <=> B ∪ A +// 4. (A ∩ B) ∩ C <=> A ∩ (B ∩ C) +// 5. (A ∪ B) ∪ C <=> A ∪ (B ∪ C) +// 7. A ∪ (B ∩ C) <=> (A ∪ B) ∩ (A ∪ C) +// 8. A ∩ (B ∪ C) <=> (A ∩ B) ∪ (A ∩ C) +// 9. A ∪ (A ∩ B) <=> A +// 10. A ∩ (A ∪ B) <=> A +// 11. !(A ∩ B) <=> !A ∪ !B +// 12. !(A ∪ B) <=> !A ∩ !B +// 13. A ∪ !A <=> 1 +type Logic[T Item] interface { + fmt.Stringer + Eval(o Operator[T]) (T, error) + mustNotImplement() +} + +type Operator[T Item] interface { + AND(first T, others ...T) (T, error) + OR(first T, others ...T) (T, error) + NOT(input T) (T, error) +} + +type notLogic[T Item] struct { + origin atomLogic[T] +} + +func (n notLogic[T]) String() string { + var sb strings.Builder + sb.WriteString(SymbolNOT) + sb.WriteString(n.origin.String()) + + return sb.String() +} + +func (n notLogic[T]) Eval(o Operator[T]) (T, error) { + v, err := n.origin.Eval(o) + if err != nil { + return v, err + } + return o.NOT(v) +} + +func (n notLogic[T]) Value() T { + return n.origin.Value() +} + +func (n notLogic[T]) mustNotImplement() { +} + +type atomLogic[T Item] struct { + t T +} + +func (al atomLogic[T]) Eval(o Operator[T]) (T, error) { + return al.t, nil +} + +func (al atomLogic[T]) String() string { + return fmt.Sprint(al.Value()) +} + +func (al atomLogic[T]) Value() T { + return al.t +} + +func (al atomLogic[T]) mustNotImplement() { +} + +type intersectionLogic[T Item] []Logic[T] + +func (in intersectionLogic[T]) Eval(o Operator[T]) (T, error) { + if len(in) < 1 { + var t T + return t, ErrEmptyIntersection + } + var values []T + for i := range in { + next, err := in[i].Eval(o) + if err != nil { + return next, err + } + values = append(values, next) + } + return o.AND(values[0], values[1:]...) +} + +func (in intersectionLogic[T]) String() string { + if len(in) < 1 { + return NONE + } + var sb strings.Builder + + writeNext := func(l Logic[T]) { + switch v := l.(type) { + case atomLogic[T], notLogic[T], intersectionLogic[T]: + sb.WriteString(v.String()) + case unionLogic[T]: + sb.WriteByte('(') + sb.WriteString(v.String()) + sb.WriteByte(')') + } + } + + writeNext(in[0]) + + for i := 1; i < len(in); i++ { + sb.WriteByte(' ') + sb.WriteString(SymbolAND) + sb.WriteByte(' ') + writeNext(in[i]) + } + + return sb.String() +} + +func (in intersectionLogic[T]) mustNotImplement() { +} + +type unionLogic[T Item] []Logic[T] + +func (un unionLogic[T]) String() string { + if len(un) < 1 { + return "" + } + + var sb strings.Builder + + writeNext := func(l Logic[T]) { + switch v := l.(type) { + case atomLogic[T], notLogic[T], unionLogic[T]: + sb.WriteString(v.String()) + case intersectionLogic[T]: + sb.WriteByte('(') + sb.WriteString(v.String()) + sb.WriteByte(')') + } + } + + writeNext(un[0]) + for i := 1; i < len(un); i++ { + sb.WriteByte(' ') + sb.WriteString(SymbolOR) + sb.WriteByte(' ') + writeNext(un[i]) + } + return sb.String() +} + +func (un unionLogic[T]) Eval(o Operator[T]) (T, error) { + if len(un) < 1 { + var t T + return t, ErrEmptyUnion + } + var values []T + for i := range un { + next, err := un[i].Eval(o) + if err != nil { + return next, err + } + values = append(values, next) + } + return o.OR(values[0], values[1:]...) +} + +func (un unionLogic[T]) mustNotImplement() { +} + +func AND[T Item](first, second Logic[T]) Logic[T] { + switch a := first.(type) { + case atomLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return atomAndAtom[T](a, b) + case notLogic[T]: + return atomAndNot[T](a, b) + case intersectionLogic[T]: + return atomAndIntersection[T](a, b) + case unionLogic[T]: + return atomAndUnion[T](a, b) + } + case notLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return atomAndNot[T](b, a) + case notLogic[T]: + return notAndNot[T](a, b) + case intersectionLogic[T]: + return notAndIntersection[T](a, b) + case unionLogic[T]: + return notAndUnion[T](a, b) + } + case intersectionLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return atomAndIntersection[T](b, a) + case notLogic[T]: + return notAndIntersection[T](b, a) + case intersectionLogic[T]: + return intersectionAndIntersection[T](a, b) + case unionLogic[T]: + return intersectionAndUnion[T](a, b) + } + case unionLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return AND[T](b, a) + case notLogic[T]: + return notAndUnion[T](b, a) + case intersectionLogic[T]: + return intersectionAndUnion[T](b, a) + case unionLogic[T]: + return unionAndUnion[T](a, b) + } + } + panic("unreachable") +} + +func Wrap[T Item](v T) Logic[T] { + return atomLogic[T]{t: v} +} + +func NOT[T Item](input Logic[T]) Logic[T] { + switch it := input.(type) { + case atomLogic[T]: + return notLogic[T]{origin: it} + case notLogic[T]: + return it.origin + case intersectionLogic[T]: + ret := make([]Logic[T], 0, len(it)) + for i := range it { + ret = append(ret, NOT(it[i])) + } + return unionLogic[T](ret) + case unionLogic[T]: + ret := make([]Logic[T], 0, len(it)) + for i := range it { + ret = append(ret, NOT(it[i])) + } + return intersectionLogic[T](ret) + } + panic("unreachable") +} + +func True[T Item]() Logic[T] { + return (unionLogic[T])(nil) +} + +func False[T Item]() Logic[T] { + return (intersectionLogic[T])(nil) +} + +func OR[T Item](first, second Logic[T]) Logic[T] { + switch a := first.(type) { + case atomLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return atomOrAtom[T](a, b) + case notLogic[T]: + return atomOrNot[T](a, b) + case intersectionLogic[T]: + return atomOrIntersection[T](a, b) + case unionLogic[T]: + return atomOrUnion[T](a, b) + } + case notLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return atomOrNot[T](b, a) + case notLogic[T]: + return notOrNot[T](a, b) + case intersectionLogic[T]: + return notOrIntersection[T](a, b) + case unionLogic[T]: + return notOrUnion[T](a, b) + } + case intersectionLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return atomOrIntersection[T](b, a) + case notLogic[T]: + return notOrIntersection[T](b, a) + case intersectionLogic[T]: + return intersectionOrIntersection[T](a, b) + case unionLogic[T]: + return intersectionOrUnion(a, b) + } + case unionLogic[T]: + switch b := second.(type) { + case atomLogic[T]: + return atomOrUnion[T](b, a) + case notLogic[T]: + return notOrUnion[T](b, a) + case intersectionLogic[T]: + return intersectionOrUnion(b, a) + case unionLogic[T]: + return unionOrUnion[T](a, b) + } + } + + panic("unreachable") +} + +func atomAndNot[T Item](a atomLogic[T], b notLogic[T]) Logic[T] { + switch a.Value().Compare(b.Value()) { + case 0: + // A ∩ !A => NaN + return (intersectionLogic[T])(nil) + case -1: + return intersectionLogic[T]{a, b} + case 1: + return intersectionLogic[T]{b, a} + } + panic("unreachable") +} + +func notAndNot[T Item](a, b notLogic[T]) Logic[T] { + switch a.Value().Compare(b.Value()) { + case 1: + // !B ∩ !A => !A ∩ !B + return intersectionLogic[T]{b, a} + case -1: + // !A ∩ !B => !A ∩ !B + return intersectionLogic[T]{a, b} + case 0: + // !A ∩ !A => !A + return a + } + panic("unreachable") +} + +func atomAndAtom[T Item](a, b atomLogic[T]) Logic[T] { + switch a.Value().Compare(b.Value()) { + case 1: + // B ∩ A => A ∩ B + return intersectionLogic[T]{b, a} + case -1: + // A ∩ B => A ∩ B + return intersectionLogic[T]{a, b} + case 0: + // A ∩ A => A + return a + } + panic("unreachable") +} + +func notOrNot[T Item](a, b notLogic[T]) Logic[T] { + switch a.Value().Compare(b.Value()) { + case 0: + return a + case -1: + return unionLogic[T]{a, b} + case 1: + return unionLogic[T]{b, a} + } + panic("unreachable") +} + +func atomOrNot[T Item](a atomLogic[T], b notLogic[T]) Logic[T] { + switch a.Value().Compare(b.Value()) { + case 0: + return unionLogic[T](nil) + case -1: + return unionLogic[T]{a, b} + case 1: + return unionLogic[T]{b, a} + } + panic("unreachable") +} + +func atomOrAtom[T Item](a, b atomLogic[T]) Logic[T] { + switch a.Value().Compare(b.Value()) { + case 0: + return a + case -1: + return unionLogic[T]{a, b} + case 1: + return unionLogic[T]{b, a} + } + panic("unreachable") +} + +func atomAndIntersection[T Item](a atomLogic[T], b intersectionLogic[T]) Logic[T] { + if len(b) < 1 { + return (intersectionLogic[T])(nil) + } + // A ∩ ( A ∩ B ∩ C ) => A ∩ B ∩ C + ret := make([]Logic[T], 0, len(b)+1) + var added bool + for i := range b { + if added { + ret = append(ret, b[i]) + continue + } + + switch it := b[i].(type) { + case atomLogic[T]: + switch a.Value().Compare(it.Value()) { + case 0: + ret = append(ret, b[i]) + added = true + continue + case -1: + ret = append(ret, a, b[i]) + added = true + continue + } + case notLogic[T]: + switch a.Value().Compare(it.Value()) { + case 0: + return (intersectionLogic[T])(nil) + case -1: + ret = append(ret, a, b[i]) + added = true + continue + } + } + + ret = append(ret, b[i]) + } + if !added { + ret = append(ret, a) + } + return intersectionLogic[T](ret) +} + +func atomAndUnion[T Item](a atomLogic[T], b unionLogic[T]) Logic[T] { + // A ∩ (B ∪ C) => (A ∩ B) ∪ (A ∩ C) + // A ∩ (A ∪ B) => A + for i := range b { + switch next := b[i].(type) { + case atomLogic[T]: + if next.Value().Compare(a.Value()) == 0 { + return a + } + } + } + var ret []Logic[T] + for i := range b { + ret = append(ret, AND[T](a, b[i])) + } + return unionLogic[T](ret) +} + +func notAndUnion[T Item](a notLogic[T], b unionLogic[T]) Logic[T] { + if len(b) < 1 { + return a + } + + // !A ∩ (A ∪ B) => (!A ∩ A) ∪ (!A ∩ B) => !A ∩ B + var ret []Logic[T] + for i := range b { + next := AND[T](a, b[i]) + if next == nil { + continue + } + switch v := next.(type) { + case unionLogic[T]: + if len(v) < 1 { + return (unionLogic[T])(nil) + } + case intersectionLogic[T]: + if len(v) < 1 { + continue + } + } + ret = append(ret, next) + } + + if len(ret) < 1 { + return (intersectionLogic[T])(nil) + } + + return unionLogic[T](ret) +} + +func notAndIntersection[T Item](a notLogic[T], b intersectionLogic[T]) Logic[T] { + if len(b) < 1 { + return nil + } + + // !B ∩ (A ∩ C) => A ∩ !B ∩ C + // !B ∩ (B ∩ C) => NaN + ret := make([]Logic[T], 0, len(b)+1) + + for i := range b { + next := AND[T](a, b[i]) + + switch v := next.(type) { + case intersectionLogic[T]: + if len(v) < 1 { + return (intersectionLogic[T])(nil) + } + case unionLogic[T]: + if len(v) < 1 { + continue + } + } + + ret = append(ret, next) + } + + return (intersectionLogic[T])(ret) +} + +func atomOrIntersection[T Item](a atomLogic[T], b intersectionLogic[T]) Logic[T] { + if len(b) < 1 { + return a + } + // A ∪ (B ∩ C) + // A ∪ (A ∩ B) => A + for i := range b { + switch it := b[i].(type) { + case atomLogic[T]: + if a.Value().Compare(it.Value()) == 0 { + return a + } + } + } + return unionLogic[T]{a, b} +} + +func notOrIntersection[T Item](a notLogic[T], b intersectionLogic[T]) Logic[T] { + if len(b) < 1 { + return a + } + // A ∪ (B ∩ C) + // A ∪ (A ∩ B) => A + for i := range b { + switch it := b[i].(type) { + case notLogic[T]: + if a.Value().Compare(it.Value()) == 0 { + return a + } + } + } + return unionLogic[T]{a, b} +} + +func atomOrUnion[T Item](a atomLogic[T], b unionLogic[T]) Logic[T] { + if len(b) < 1 { + return b + } + + ret := make([]Logic[T], 0, len(b)+1) + var added bool + for i := range b { + if added { + ret = append(ret, b[i]) + continue + } + switch next := b[i].(type) { + case atomLogic[T]: + switch a.Value().Compare(next.Value()) { + case -1: + ret = append(ret, a, next) + added = true + case 0: // A ∪ (A ∪ B) => A ∪ B + added = true + ret = append(ret, a) + case 1: + ret = append(ret, next) + } + continue + case notLogic[T]: + switch a.Value().Compare(next.Value()) { + case -1: + ret = append(ret, a, next) + added = true + case 0: // A ∪ (!A ∪ B) => B + added = true + case 1: + ret = append(ret, next) + } + continue + default: + ret = append(ret, next) + } + } + + if !added { + ret = append(ret, a) + } + + if len(ret) < 1 { + return unionLogic[T](nil) + } + + return unionLogic[T](ret) +} + +func notOrUnion[T Item](a notLogic[T], b unionLogic[T]) Logic[T] { + if len(b) < 1 { + return b + } + + // !A ∪ (!A ∪ B) => !A ∪ B + ret := make([]Logic[T], 0, len(b)+1) + var added bool + + for i := range b { + if added { + ret = append(ret, b[i]) + continue + } + switch next := b[i].(type) { + case notLogic[T]: + switch a.Value().Compare(next.Value()) { + case -1: + added = true + ret = append(ret, a, next) + case 1: + ret = append(ret, next) + case 0: + added = true + ret = append(ret, a) + } + continue + case atomLogic[T]: + switch a.Value().Compare(next.Value()) { + case -1: + added = true + ret = append(ret, a, next) + case 1: + ret = append(ret, next) + case 0: + added = true + } + continue + default: + ret = append(ret, next) + } + } + + if len(ret) < 1 { + return unionLogic[T](nil) + } + + return unionLogic[T](ret) +} + +func intersectionOrUnion[T Item](a intersectionLogic[T], b unionLogic[T]) Logic[T] { + if len(b) < 1 { + return unionLogic[T](nil) + } + if len(a) < 1 { + return b + } + + // (C ∩ D) ∪ (A ∪ B) => A ∪ B ∪ (C ∩ D) + // (A ∩ D) ∪ (A ∪ B) => A ∪ B + + for i := range a { + for j := range b { + switch x := a[i].(type) { + case atomLogic[T]: + switch y := b[j].(type) { + case atomLogic[T]: + if x.Value().Compare(y.Value()) == 0 { + return b + } + } + case notLogic[T]: + switch y := b[j].(type) { + case notLogic[T]: + if x.Value().Compare(y.Value()) == 0 { + return b + } + } + } + } + } + + bt := btree.NewG[Logic[T]](2, Less[T]) + + bt.ReplaceOrInsert(a) + for i := range b { + bt.ReplaceOrInsert(b[i]) + } + + ret := make([]Logic[T], 0, bt.Len()) + bt.Ascend(func(item Logic[T]) bool { + ret = append(ret, item) + return true + }) + + return (unionLogic[T])(ret) +} + +func intersectionOrIntersection[T Item](a, b intersectionLogic[T]) Logic[T] { + if len(a) < 1 { + return b + } + if len(b) < 1 { + return a + } + + // (A ∩ B) ∪ (C ∩ D) + return unionLogic[T]{a, b} +} + +func unionAndUnion[T Item](a, b unionLogic[T]) Logic[T] { + if len(a) < 1 { + return b + } + if len(b) < 1 { + return a + } + + // (A ∪ B) ∩ (C ∪ D) => (A ∩ C) ∪ (A ∩ D) ∪ (B ∩ C) ∪ (B ∩ D) + + var result Logic[T] + for i := range a { + for j := range b { + next := AND(a[i], b[j]) + if result == nil { + result = next + } else { + result = OR(result, next) + } + } + } + + return result +} + +func unionOrUnion[T Item](a, b unionLogic[T]) Logic[T] { + if len(a) < 1 || len(b) < 1 { + return (unionLogic[T])(nil) + } + + // (A ∪ B) ∪ (C ∪ D) => A ∪ B ∪ C ∪ D + + bt := btree.NewG[Logic[T]](2, Less[T]) + for i := range b { + switch next := OR[T](a, b[i]).(type) { + case atomLogic[T], notLogic[T], intersectionLogic[T]: + bt.ReplaceOrInsert(next) + case unionLogic[T]: + if len(next) < 1 { + return unionLogic[T](nil) + } + for j := range next { + bt.ReplaceOrInsert(next[j]) + } + } + } + + var ret unionLogic[T] + bt.Ascend(func(item Logic[T]) bool { + ret = append(ret, item) + return true + }) + return ret +} + +func intersectionAndIntersection[T Item](a, b intersectionLogic[T]) Logic[T] { + if len(a) < 1 || len(b) < 1 { + return intersectionLogic[T](nil) + } + + bt := btree.NewG[Logic[T]](2, Less[T]) + + for i := range a { + next := AND[T](a[i], b) + switch val := next.(type) { + case atomLogic[T], notLogic[T]: + bt.ReplaceOrInsert(next) + case intersectionLogic[T]: + if len(val) < 1 { + return intersectionLogic[T](nil) + } + for j := range val { + bt.ReplaceOrInsert(val[j]) + } + case unionLogic[T]: + panic("unreachable") + } + } + + var ret intersectionLogic[T] + bt.Ascend(func(item Logic[T]) bool { + ret = append(ret, item) + return true + }) + + return ret +} + +func intersectionAndUnion[T Item](a intersectionLogic[T], b unionLogic[T]) Logic[T] { + if len(a) < 1 || len(b) < 1 { + return a + } + // (A ∩ B) ∩ (C ∪ A) => A ∩ B + for i := range a { + for j := range b { + switch x := a[i].(type) { + case atomLogic[T]: + switch y := b[j].(type) { + case atomLogic[T]: + if x.Value().Compare(y.Value()) == 0 { + return a + } + } + case notLogic[T]: + switch y := b[j].(type) { + case notLogic[T]: + if x.Value().Compare(y.Value()) == 0 { + return a + } + } + } + } + } + + // (A ∩ B) ∩ (C ∪ D) => (A ∩ B ∩ C) ∪ (A ∩ B ∩ D) + bt := btree.NewG[Logic[T]](2, Less[T]) + for i := range b { + next := AND[T](a, b[i]) + bt.ReplaceOrInsert(next) + } + + var ret unionLogic[T] + bt.Ascend(func(item Logic[T]) bool { + ret = append(ret, item) + return true + }) + + return ret +} + +func Compare[T Item](prev, next Logic[T]) int { + switch a := prev.(type) { + case atomLogic[T]: + switch b := next.(type) { + case atomLogic[T]: + return a.Value().Compare(b.Value()) + case notLogic[T]: + return a.Value().Compare(b.Value()) + default: + return -1 + } + case notLogic[T]: + switch b := next.(type) { + case atomLogic[T]: + switch a.Value().Compare(b.Value()) { + case -1, 0: + return -1 + default: + return 1 + } + case notLogic[T]: + return a.Value().Compare(b.Value()) + default: + return -1 + } + case intersectionLogic[T]: + switch b := next.(type) { + case intersectionLogic[T]: + if len(a) < len(b) { + return -1 + } + if len(a) > len(b) { + return 1 + } + for i := 0; i < len(a); i++ { + c := Compare(a[i], b[i]) + switch c { + case 1, -1: + return c + } + } + return 0 + case unionLogic[T]: + if len(a) < len(b) { + return -1 + } + if len(a) > len(b) { + return 1 + } + for i := 0; i < len(a); i++ { + c := Compare(a[i], b[i]) + switch c { + case 1, -1: + return c + } + } + return 1 + } + case unionLogic[T]: + switch b := next.(type) { + case intersectionLogic[T]: + if len(a) < len(b) { + return -1 + } + if len(a) > len(b) { + return 1 + } + for i := 0; i < len(a); i++ { + c := Compare(a[i], b[i]) + switch c { + case 1, -1: + return c + } + } + return -1 + case unionLogic[T]: + if len(a) < len(b) { + return -1 + } + if len(a) > len(b) { + return 1 + } + for i := 0; i < len(a); i++ { + c := Compare(a[i], b[i]) + switch c { + case 1, -1: + return c + } + } + return 0 + } + } + panic("unreachable") +} + +func Less[T Item](prev, next Logic[T]) bool { + return Compare(prev, next) == -1 +} diff --git a/pkg/runtime/calc/logic/logic_test.go b/pkg/runtime/calc/logic/logic_test.go new file mode 100644 index 00000000..32c55132 --- /dev/null +++ b/pkg/runtime/calc/logic/logic_test.go @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logic + +import ( + "testing" +) + +import ( + "github.com/stretchr/testify/assert" +) + +func TestLogic(t *testing.T) { + a := Wrap[String]("a") + b := Wrap[String]("b") + c := Wrap[String]("c") + d := Wrap[String]("d") + + type tt[T Item] struct { + scene string + input func() Logic[T] + expect string + } + + for _, next := range []tt[String]{ + // --- NOT --- + { + "!!a", + func() Logic[String] { + return NOT(NOT(a)) + }, + "a", + }, + { + "!a", + func() Logic[String] { + return NOT(a) + }, + "!a", + }, + { + "!(a && b)", + func() Logic[String] { + return NOT(AND(a, b)) + }, + "!a || !b", + }, + { + "!(b || a)", + func() Logic[String] { + return NOT(OR(b, a)) + }, + "!a && !b", + }, + { + "!(b && !a)", + func() Logic[String] { + return NOT(AND(b, NOT(a))) + }, + "a || !b", + }, + // --- AND --- + { + "a && b", + func() Logic[String] { + return AND(a, b) + }, + "a && b", + }, + // --- AND --- + { + "a && (a || c)", func() Logic[String] { + return AND(a, OR(a, c)) + }, + "a", + }, + { + "(a || c) && a", func() Logic[String] { + return AND(OR(a, c), a) + }, + "a", + }, + { + "a && !a", func() Logic[String] { + return AND(a, NOT(a)) + }, + NONE, + }, + { + "!a && a", + func() Logic[String] { + return AND(NOT(a), a) + }, + NONE, + }, + { + "a && (b && !a)", + func() Logic[String] { + return AND(a, AND(b, NOT(a))) + }, + NONE, + }, + { + "b && (a && c)", + func() Logic[String] { + return AND(b, AND(a, c)) + }, + "a && b && c", + }, + { + "(a && c) && b", + func() Logic[String] { + return AND(AND(a, c), b) + }, + "a && b && c", + }, + { + "!a && (a && b)", + func() Logic[String] { + return AND(NOT(a), AND(a, b)) + }, + NONE, + }, + { + "(a && b) && (c && !b)", + func() Logic[String] { + return AND(AND(a, b), AND(c, NOT(b))) + }, + NONE, + }, + { + "(d && c) && (b && a)", + func() Logic[String] { + return AND(AND(d, c), AND(b, a)) + }, + "a && b && c && d", + }, + + // --- OR --- + { + "a || b", + func() Logic[String] { + return OR(a, b) + }, + "a || b", + }, + { + "b || a", + func() Logic[String] { + return OR(b, a) + }, + "a || b", + }, + { + "a || (b || c)", + func() Logic[String] { + return OR(a, OR(b, c)) + }, + "a || b || c", + }, + { + "(a || c) || b", + func() Logic[String] { + return OR(OR(a, c), b) + }, + "a || b || c", + }, + { + "(a || c) || !a", + func() Logic[String] { + return OR(OR(a, c), NOT(a)) + }, + "c", + }, + { + "(d || c) || (b || a)", + func() Logic[String] { + return OR(OR(d, c), OR(b, a)) + }, + "a || b || c || d", + }, + { + "(a || b) && (c || a)", + func() Logic[String] { + return AND(OR(a, b), OR(c, a)) + }, + "a || (b && c)", + }, + { + "(a || b) && (a || b)", + func() Logic[String] { + return AND(OR(a, b), OR(a, b)) + }, + "a || b", + }, + { + "(a && b) && (c || a)", + func() Logic[String] { + return AND(AND(a, b), OR(c, a)) + }, + "a && b", + }, + { + "(a && c) || (a || d)", + func() Logic[String] { + return OR(AND(a, c), OR(a, d)) + }, + "a || d", + }, + { + "(a && b) || (c || d)", + func() Logic[String] { + return OR(AND(a, b), OR(c, d)) + }, + "c || d || (a && b)", + }, + { + "(a || b) && (c || d)", + func() Logic[String] { + return AND(OR(a, b), OR(c, d)) + }, + "(a && c) || (a && d) || (b && c) || (b && d)", + }, + } { + t.Run(next.scene, func(t *testing.T) { + v := next.input() + assert.Equal(t, next.expect, v.String()) + }) + } +} diff --git a/pkg/runtime/calc/logic/string.go b/pkg/runtime/calc/logic/string.go new file mode 100644 index 00000000..ee7832a5 --- /dev/null +++ b/pkg/runtime/calc/logic/string.go @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logic + +import ( + "strings" +) + +type String string + +func (s String) Compare(item Item) int { + switch that := item.(type) { + case String: + return strings.Compare(string(s), string(that)) + default: + return -1 + } +} diff --git a/pkg/runtime/calc/misc.go b/pkg/runtime/calc/misc.go new file mode 100644 index 00000000..f175824b --- /dev/null +++ b/pkg/runtime/calc/misc.go @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package calc + +import ( + "fmt" + "strings" + "time" +) + +import ( + "github.com/arana-db/arana/pkg/proto/rule" + "github.com/arana-db/arana/pkg/runtime/cmp" + "github.com/arana-db/arana/pkg/util/misc" +) + +func compareValue(prev, next interface{}) int { + if prev == nil && next == nil { + return 0 + } + + if prev == nil && next != nil { + return -1 + } + if prev != nil && next == nil { + return 1 + } + + switch x := prev.(type) { + case int64: + switch y := next.(type) { + case int64: + switch { + case x < y: + return -1 + case x > y: + return 1 + default: + return 0 + } + } + case time.Time: + switch y := next.(type) { + case time.Time: + switch { + case x.Before(y): + return -1 + case x.After(y): + return 1 + default: + return 0 + } + } + } + + return strings.Compare(fmt.Sprint(prev), fmt.Sprint(next)) +} + +func compareComparativeValue(prev, next *cmp.Comparative) int { + return innerCompareComparative(prev, next, true) +} + +func compareComparative(prev, next *cmp.Comparative) int { + return innerCompareComparative(prev, next, false) +} + +func innerCompareComparative(prev, next *cmp.Comparative, valueOnly bool) int { + if !valueOnly { + c := strings.Compare(prev.Key(), next.Key()) + if c != 0 { + return c + } + switch { + case prev.Comparison() < next.Comparison(): + return -1 + case prev.Comparison() > next.Comparison(): + return 1 + } + } + + if prev.RawValue() == next.RawValue() { + return 0 + } + + x, _ := prev.Value() + y, _ := next.Value() + + if x == nil && y == nil { + return 0 + } + + if x == nil && y != nil { + return -1 + } + + if x != nil && y == nil { + return 1 + } + + switch a := x.(type) { + case time.Time: + switch b := y.(type) { + case time.Time: + switch { + case a.Before(b): + return -1 + case a.After(b): + return 1 + default: + return 0 + } + } + case int64: + switch b := y.(type) { + case int64: + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } + } + } + + return strings.Compare(prev.RawValue(), next.RawValue()) +} + +func computeLRange(m *rule.ShardMetadata, begin *cmp.Comparative) (ret []interface{}) { + if begin.Comparison() == cmp.Ceq { + ret = []interface{}{begin.MustValue()} + return + } + + column := m.GetShardColumn(begin.Key()) + + nextInclude := begin.Comparison() == cmp.Cgte + iter, _ := column.Stepper.Ascend(begin.MustValue(), column.Steps) + for iter.HasNext() { + next := iter.Next() + if nextInclude { + ret = append(ret, next) + } else { + nextInclude = true + } + } + return +} + +func computeRRange(m *rule.ShardMetadata, end *cmp.Comparative) (ret []interface{}) { + if end.Comparison() == cmp.Ceq { + ret = []interface{}{ + end.MustValue(), + } + return + } + + column := m.GetShardColumn(end.Key()) + + nextInclude := end.Comparison() == cmp.Clte + iter, _ := column.Stepper.Descend(end.MustValue(), column.Steps) + for iter.HasNext() { + next := iter.Next() + if nextInclude { + ret = append(ret, next) + } else { + nextInclude = true + } + } + misc.ReverseSlice(ret) + return +} + +func computeRange(m *rule.ShardMetadata, begin, end *cmp.Comparative) (ret []interface{}) { + var ( + max interface{} + beginInclude = begin.Comparison() == cmp.Cgte + endInclude bool + column = m.GetShardColumn(begin.Key()) + ) + + switch end.Comparison() { + case cmp.Clte: + max = end.MustValue() + endInclude = true + case cmp.Clt: + max = end.MustValue() + } + + iter, _ := column.Stepper.Ascend(begin.MustValue(), column.Steps) +L: + for iter.HasNext() { + next := iter.Next() + switch compareValue(next, max) { + case -1: + if beginInclude { + ret = append(ret, next) + } else if len(ret) == 0 { + beginInclude = true + } + case 0: + if !endInclude { + break L + } + if beginInclude { + ret = append(ret, next) + } else if len(ret) == 0 { + beginInclude = true + } + case 1: + break L + } + } + return +} + +func searchVShard[T any](table *rule.VTable, groups map[string]T) *rule.VShard { + vShards := table.GetVShards() + +L: + for i := range vShards { + // check whether each shard keys contains in groups + for _, it := range vShards[i].Variables() { + if _, ok := groups[it]; !ok { + continue L + } + } + return vShards[i] + } + + return nil +} diff --git a/pkg/runtime/rule/route_test.go b/pkg/runtime/calc/misc_test.go similarity index 53% rename from pkg/runtime/rule/route_test.go rename to pkg/runtime/calc/misc_test.go index 8e16a62e..e7817e98 100644 --- a/pkg/runtime/rule/route_test.go +++ b/pkg/runtime/calc/misc_test.go @@ -15,10 +15,9 @@ * limitations under the License. */ -package rule +package calc import ( - "sort" "testing" ) @@ -27,44 +26,47 @@ import ( ) import ( + "github.com/arana-db/arana/pkg/proto/rule" "github.com/arana-db/arana/pkg/runtime/cmp" ) -func TestRule_Route(t *testing.T) { - ru := makeTestRule(4) - - const key = "uid" - - vtab, _ := ru.VTable(fakeTable) - - tb := []struct { - C *cmp.Comparative - E []int - }{ - // simple - {cmp.NewInt64(key, cmp.Ceq, 1), []int{1}}, - {cmp.NewInt64(key, cmp.Cgt, 1), nil}, - {cmp.NewInt64(key, cmp.Cgte, 1), nil}, - {cmp.NewInt64(key, cmp.Clt, 3), nil}, - {cmp.NewInt64(key, cmp.Clte, 3), nil}, +func TestComputeRange(t *testing.T) { + m := &rule.ShardMetadata{ + ShardColumns: []*rule.ShardColumn{ + { + Name: "x", + Steps: 8, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + }, } - for _, it := range tb { - mat, err := Route(vtab, it.C) - assert.NoError(t, err) - ret, err := mat.Eval() - assert.NoError(t, err, "eval failed") - t.Logf("+++++++++ %s ++++++++\n", it.C) - actual := make([]int, 0) - for ret.HasNext() { - actual = append(actual, int(ret.Next().(int64))) - } + type tt struct { + scene string + begin, end *cmp.Comparative + expect []interface{} + } - t.Log("route result:", actual) - if it.E != nil { - sort.Ints(actual) - sort.Ints(it.E) - assert.EqualValues(t, it.E, actual) - } + for _, next := range []tt{ + { + "x>=1 && x<=4", + cmp.NewInt64("x", cmp.Cgte, 1), + cmp.NewInt64("x", cmp.Clte, 4), + []interface{}{int64(1), int64(2), int64(3), int64(4)}, + }, + { + "x>=1 && x<4", + cmp.NewInt64("x", cmp.Cgte, 1), + cmp.NewInt64("x", cmp.Clt, 4), + []interface{}{int64(1), int64(2), int64(3)}, + }, + } { + t.Run(next.scene, func(t *testing.T) { + actual := computeRange(m, next.begin, next.end) + assert.Equal(t, next.expect, actual) + }) } } diff --git a/pkg/runtime/cmp/cmp.go b/pkg/runtime/cmp/cmp.go index ebf08338..c37e22db 100644 --- a/pkg/runtime/cmp/cmp.go +++ b/pkg/runtime/cmp/cmp.go @@ -122,6 +122,10 @@ type Comparative struct { v string } +func (c *Comparative) SetComparison(comparison Comparison) { + c.c = comparison +} + // Kind returns the Kind. func (c *Comparative) Kind() Kind { return c.k diff --git a/pkg/runtime/logical/logical.go b/pkg/runtime/logical/logical.go deleted file mode 100644 index f7c565b1..00000000 --- a/pkg/runtime/logical/logical.go +++ /dev/null @@ -1,533 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package logical - -import ( - "fmt" - "sort" - "strings" -) - -import ( - "github.com/arana-db/arana/pkg/runtime/misc" -) - -const ( - _and = "&&" - _or = "||" - _not = "!" -) - -const ( - _ Op = iota - Land // AND - Lor // OR -) - -var ( - _ Logical = (*atom)(nil) - _ Logical = (*composite)(nil) -) - -// Op represents the logical operator includes AND, OR. -type Op uint8 - -func (o Op) String() string { - switch o { - case Land: - return "AND" - case Lor: - return "OR" - default: - panic("unreachable") - } -} - -// Option represents the option of Logical. -type Option func(*atom) - -// WithValue attaches a value into Logical. -func WithValue(value interface{}) Option { - return func(a *atom) { - a.value = value - } -} - -// WithSortKey sets the sortable key of Logical item. -func WithSortKey(key string) Option { - return func(a *atom) { - a.sk = key - } -} - -// New creates Logical. -func New(key string, options ...Option) Logical { - ret := new(atom) - for _, it := range options { - it(ret) - } - ret.id = key - return ret -} - -// Logical represents the operation of logical item. -type Logical interface { - fmt.Stringer - // And eval the AND operation. - And(other Logical) Logical - // Not eval the NOT operation. - Not() Logical - // Or eval the OR operation. - Or(other Logical) Logical - // ToString returns a display string. - ToString(and, or string) string - // phantom avoids the implementation out of current package. - phantom() -} - -type atom struct { - not bool - id string - sk string // sort key - value interface{} -} - -func (a *atom) phantom() { - // DO NOTHING -} - -func (a *atom) Not() Logical { - b := new(atom) - *b = *a - b.not = !b.not - return b -} - -func (a *atom) ToString(_, _ string) string { - return a.String() -} - -func (a *atom) equalsWith(other Logical) bool { - switch b := other.(type) { - case *atom: - return a.id == b.id && a.not == b.not - } - return false -} - -func (a *atom) String() string { - if a.not { - return _not + a.id - } - return a.id -} - -func (a *atom) And(other Logical) Logical { - switch b := other.(type) { - case *composite: - switch b.op { - case Land: - // A ∩ ( A ∩ B ∩ C ) => A ∩ B ∩ C - ret := &composite{op: Land} - ret.ch = append(ret.ch, a) - for _, it := range b.ch { - if !a.equalsWith(it) { - ret.ch = append(ret.ch, it) - } - } - return ret - case Lor: - // A ∩ (B ∪ C) => (A ∩ B) ∪ (A ∩ C) - // A ∩ (A ∪ B) => A - for _, it := range b.ch { - if next, ok := it.(*atom); ok && next == a { - return a - } - } - ret := &composite{op: Lor} - for _, it := range b.ch { - ret.ch = append(ret.ch, a.And(it)) - } - return ret - } - case *atom: - if a.equalsWith(b) { - return a - } - reverse := strings.Compare(misc.FirstNonEmptyString(a.sk, a.id), misc.MustFirstNonEmptyString(b.sk, b.id)) == -1 - ret := &composite{op: Land} - if reverse { - ret.ch = []Logical{b, a} - } else { - ret.ch = []Logical{a, b} - } - return ret - } - panic("unreachable") -} - -func (a *atom) Or(other Logical) Logical { - switch b := other.(type) { - case *composite: - switch b.op { - case Land: - // A ∪ (B ∩ C) - // A ∪ (A ∩ B) => A - for _, it := range b.ch { - if a.equalsWith(it) { - return a - } - } - - return &composite{op: Lor, ch: []Logical{a, b}} - case Lor: - // A ∪ (B ∪ C) - if b.contains(a) { - return b - } - - ret := &composite{op: Lor} - ret.ch = append(ret.ch, a) - ret.ch = append(ret.ch, b.ch...) - return ret - } - case *atom: - if a.equalsWith(b) { - return a - } - reverse := strings.Compare(misc.FirstNonEmptyString(a.sk, a.id), misc.MustFirstNonEmptyString(b.sk, b.id)) == -1 - ret := &composite{op: Lor} - if reverse { - ret.ch = []Logical{b, a} - } else { - ret.ch = []Logical{a, b} - } - return ret - } - - panic("unreachable") -} - -type composite struct { - op Op - ch []Logical -} - -func (c *composite) phantom() { -} - -func (c *composite) Not() Logical { - switch c.op { - case Land: - v := c.ch[0].Not() - for i := 1; i < len(c.ch); i++ { - v = v.Or(c.ch[i].Not()) - } - return v - case Lor: - v := c.ch[0].Not() - for i := 1; i < len(c.ch); i++ { - v = v.And(c.ch[i].Not()) - } - return v - default: - panic("unreachable") - } -} - -func (c *composite) ToString(and, or string) string { - if len(c.ch) < 1 { - return "" - } - - var sb strings.Builder - sb.WriteByte('(') - - sb.WriteByte(' ') - sb.WriteString(c.ch[0].ToString(and, or)) - - for i := 1; i < len(c.ch); i++ { - sb.WriteByte(' ') - switch c.op { - case Land: - sb.WriteString(and) - case Lor: - sb.WriteString(or) - default: - panic("unreachable") - } - sb.WriteByte(' ') - sb.WriteString(c.ch[i].ToString(and, or)) - } - - sb.WriteByte(' ') - sb.WriteByte(')') - - return sb.String() -} - -func (c *composite) String() string { - return c.ToString(_and, _or) -} - -func (c *composite) And(other Logical) Logical { - ret := c.and(other) - if it, ok := ret.(*composite); ok { - it.optimize() - } - return ret -} - -func (c *composite) and(other Logical) Logical { - switch d := other.(type) { - case *atom: - return d.And(c) - case *composite: - switch c.op { - case Land: - switch d.op { - case Land: - // (A ∩ B ) ∩ (C ∩ D) - var ret Logical = c - for _, it := range d.ch { - ret = ret.And(it) - } - return ret - case Lor: - // (A ∩ B) ∩ (C ∪ D) => A ∩ B ∩ (C ∪ D) - // (A ∩ B) ∩ (C ∪ A) => A ∩ B - for _, it := range c.ch { - if next, ok := it.(*atom); ok { - if d.contains(next) { - return c - } - } - } - ret := &composite{ - op: Lor, - } - for _, it := range d.ch { - ret.ch = append(ret.ch, c.And(it)) - } - return ret - } - case Lor: - switch d.op { - case Land: - // (A ∪ B) ∩ (C ∩ D) - ret := &composite{op: Lor} - for _, next := range c.ch { - ret.ch = append(ret.ch, next.And(d)) - } - return ret - case Lor: - // (A ∪ B) ∩ (C ∪ D) => (A ∩ C) ∪ (A ∩ D) ∪ (B ∩ C) ∪ (B ∩ D) - var ret Logical - for _, i := range c.ch { - for _, j := range d.ch { - next := i.And(j) - if ret == nil { - ret = next - } else { - ret = ret.Or(next) - } - } - } - return ret - } - } - } - panic("unreachable") -} - -func (c *composite) Or(other Logical) Logical { - ret := c.or(other) - if it, ok := ret.(*composite); ok { - it.optimize() - } - return ret -} - -func (c *composite) or(other Logical) Logical { - switch d := other.(type) { - case *atom: - return d.Or(c) - case *composite: - switch c.op { - case Land: - switch d.op { - case Land: - // (A ∩ B) ∪ (C ∩ D) - return &composite{ - op: Lor, - ch: []Logical{c, d}, - } - case Lor: - // (A ∩ B) ∪ (C ∪ D) - var ret Logical = c - for _, it := range d.ch { - ret = ret.Or(it) - } - return ret - } - case Lor: - switch d.op { - case Land: - // FIXME: more??? - // (A ∪ B) ∪ (C ∩ D) => A ∪ B ∪ (C ∩ D) - // (A ∪ B) ∪ (A ∩ D) => A ∪ B - for _, it := range c.ch { - if next, ok := it.(*atom); ok && d.contains(next) { - return c - } - } - ret := &composite{op: Lor} - ret.ch = append(ret.ch, c.ch...) - ret.ch = append(ret.ch, d) - return ret - case Lor: - // (A ∪ B) ∪ (C ∪ D) - var ret Logical = c - for _, it := range d.ch { - ret = ret.Or(it) - } - return ret - } - } - } - panic("unreachable") -} - -func (c *composite) optimize() int { - sort.Sort(sortLogicals(c.ch)) - removed := make(map[int]struct{}) - switch c.op { - case Lor: - for i := 0; i < len(c.ch); i++ { - next, ok := c.ch[i].(*atom) - if ok { - for j := i; j < len(c.ch); j++ { - if it, ok := c.ch[j].(*composite); ok { - if it.contains(next) { - removed[j] = struct{}{} - } - } - } - } - } - } - - if len(removed) < 1 { - return 0 - } - - var newborn []Logical - for i := 0; i < len(c.ch); i++ { - if _, ok := removed[i]; ok { - continue - } - newborn = append(newborn, c.ch[i]) - } - c.ch = newborn - return len(removed) -} - -func (c *composite) contains(a *atom) bool { - for _, it := range c.ch { - if a.equalsWith(it) { - return true - } - } - return false -} - -type sortLogicals []Logical - -func (s sortLogicals) Len() int { - return len(s) -} - -func (s sortLogicals) Less(i, j int) bool { - switch a := s[i].(type) { - case *atom: - switch b := s[j].(type) { - case *composite: - return true - case *atom: - k1 := misc.FirstNonEmptyString(a.sk, a.id) - k2 := misc.FirstNonEmptyString(b.sk, b.id) - if strings.Compare(k1, k2) == -1 { - return true - } - } - } - return false -} - -func (s sortLogicals) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -func EvalBool(l Logical) (bool, error) { - v, err := Eval(l, func(a, b interface{}) (interface{}, error) { - return a.(bool) && b.(bool), nil - }, func(a, b interface{}) (interface{}, error) { - return a.(bool) || b.(bool), nil - }, func(i interface{}) interface{} { - return !i.(bool) - }) - if err != nil { - return false, err - } - return v.(bool), err -} - -func Eval(l Logical, intersection, union func(a, b interface{}) (interface{}, error), not func(interface{}) interface{}) (ret interface{}, err error) { - switch t := l.(type) { - case *atom: - if t.not { - ret = not(t.value) - } else { - ret = t.value - } - case *composite: - ret, err = Eval(t.ch[0], intersection, union, not) - if err != nil { - return nil, err - } - for i := 1; i < len(t.ch); i++ { - next := t.ch[i] - v2, e1 := Eval(next, intersection, union, not) - if e1 != nil { - return nil, e1 - } - switch t.op { - case Land: - ret, err = intersection(ret, v2) - case Lor: - ret, err = union(ret, v2) - default: - panic("unreachable") - } - - } - default: - panic("unreachable") - } - return -} diff --git a/pkg/runtime/logical/logical_test.go b/pkg/runtime/logical/logical_test.go deleted file mode 100644 index 58830c3e..00000000 --- a/pkg/runtime/logical/logical_test.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package logical - -import ( - "testing" -) - -import ( - "github.com/stretchr/testify/assert" -) - -func TestLogicalAndOr(t *testing.T) { - for _, it := range []Logical{ - New("A").And(New("B")).And(New("C")), - New("A").And(New("B").And(New("C"))), - New("A").And(New("A")), - New("A").Or(New("A")), - New("A").And(New("A").Or(New("B"))), - New("A").And(New("B")).Or(New("A")), - New("A").Or(New("B")).And(New("C").Or(New("D"))), - New("A").And(New("B")).And(New("C").And(New("D"))), - New("A").Or(New("B")).And(New("C").Or(New("A"))), - New("A").Or(New("B")).And(New("A").Or(New("B"))), - New("A").And(New("B")).And(New("C").Or(New("A"))), - } { - t.Log(it) - } -} - -func TestNot(t *testing.T) { - for _, it := range []Logical{ - New("A").And(New("B")).Not(), - } { - t.Log(it) - } -} - -func TestEval(t *testing.T) { - l := New("A", WithValue(true)).And(New("B", WithValue(false))) - result, err := EvalBool(l) - assert.NoError(t, err) - t.Logf("%s = %v\n", l, result) -} diff --git a/pkg/runtime/misc/extvalue/div_test.go b/pkg/runtime/misc/extvalue/div_test.go index 4bc2ceff..5520723d 100644 --- a/pkg/runtime/misc/extvalue/div_test.go +++ b/pkg/runtime/misc/extvalue/div_test.go @@ -52,7 +52,6 @@ func TestDodiv(t *testing.T) { {8, decimal.NewFromFloat(0.001549967), decimal.NewFromFloat(0.002570910), decimal.NewFromFloat(0.60288653)}, } { t.Run(fmt.Sprint(v.want), func(t *testing.T) { - out, err := d.DoDiv( context.WithValue(context.Background(), ctxPrecisionKey, v.p), v.inFirst, diff --git a/pkg/runtime/misc/like_test.go b/pkg/runtime/misc/like_test.go index af587671..b3ca7db3 100644 --- a/pkg/runtime/misc/like_test.go +++ b/pkg/runtime/misc/like_test.go @@ -60,7 +60,8 @@ func TestLiker(t *testing.T) { pattern: "[a-z]at", input: []string{"[a-z]at", "cat", "Cat"}, except: []bool{true, false, false}, - }, { + }, + { pattern: "[!C]at", input: []string{"[!C]at", "Bat", "Cat"}, except: []bool{true, false, false}, diff --git a/pkg/runtime/optimize/dml/insert.go b/pkg/runtime/optimize/dml/insert.go index b1b712eb..a38fcc2f 100644 --- a/pkg/runtime/optimize/dml/insert.go +++ b/pkg/runtime/optimize/dml/insert.go @@ -23,6 +23,8 @@ import ( import ( "github.com/pkg/errors" + + "golang.org/x/exp/slices" ) import ( @@ -57,52 +59,49 @@ func optimizeInsert(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err return ret, nil } - // TODO: handle multiple shard keys. - - bingo := -1 - // check existing shard columns - for i, col := range stmt.Columns { - if _, _, ok = vt.GetShardMetadata(col); ok { - bingo = i - break + vshards := vt.GetVShards() + bingo := slices.IndexFunc(vshards, func(shard *rule.VShard) bool { + keys := shard.Variables() + for _, key := range keys { + if !slices.Contains(stmt.Columns, key) { + return false + } } - } + return true + }) - if bingo < 0 { + if bingo == -1 { return nil, errors.Wrap(optimize.ErrNoShardKeyFound, "failed to insert") } + vShard := vshards[bingo] + keys := vShard.Variables() + // check on duplicated key update for _, upd := range stmt.DuplicatedUpdates { - if upd.Column.Suffix() == stmt.Columns[bingo] { + if slices.Contains(keys, upd.Column.Suffix()) { return nil, errors.New("do not support update sharding key") } } var ( sharder = optimize.NewXSharder(ctx, o.Rule, o.Args) - left = ast.ColumnNameExpressionAtom(make([]string, 1)) - filter = &ast.PredicateExpressionNode{ - P: &ast.BinaryComparisonPredicateNode{ - Left: &ast.AtomPredicateNode{ - A: left, - }, - Op: cmp.Ceq, - }, - } - slots = make(map[string]map[string][]int) // (db,table,valuesIndex) + slots = make(map[string]map[string][]int) // (db,table,valuesIndex) ) - // reset filter - resetFilter := func(column string, value ast.ExpressionNode) { - left[0] = column - filter.P.(*ast.BinaryComparisonPredicateNode).Right = value.(*ast.PredicateExpressionNode).P - } - for i, values := range stmt.Values { - var shards rule.DatabaseTables - value := values[bingo] - resetFilter(stmt.Columns[bingo], value) + var ( + shards rule.DatabaseTables + filter ast.ExpressionNode + ) + + if len(keys) == 1 { + key := keys[0] + idx := slices.Index(stmt.Columns, key) + filter = buildFilter(stmt.Columns[idx], values[idx]) + } else { + filter = buildLogicalFilter(stmt.Columns, values, keys) + } if len(o.Hints) > 0 { if shards, err = optimize.Hints(tableName, o.Hints, o.Rule); err != nil { @@ -288,3 +287,31 @@ func createSequenceIfAbsent(ctx context.Context, vtab *rule.VTable, metadata *pr } return nil } + +func buildFilter(column string, value ast.ExpressionNode) ast.ExpressionNode { + // reset filter + return &ast.PredicateExpressionNode{ + P: &ast.BinaryComparisonPredicateNode{ + Left: &ast.AtomPredicateNode{ + A: ast.ColumnNameExpressionAtom([]string{column}), + }, + Op: cmp.Ceq, + Right: value.(*ast.PredicateExpressionNode).P, + }, + } +} + +func buildLogicalFilter(columns []string, values []ast.ExpressionNode, keys []string) ast.ExpressionNode { + idx0 := slices.Index(columns, keys[0]) + cur := buildFilter(columns[idx0], values[idx0]) + for i := 1; i < len(keys); i++ { + idx := slices.Index(columns, keys[i]) + next := buildFilter(columns[idx], values[idx]) + + cur = &ast.LogicalExpressionNode{ + Left: cur, + Right: next, + } + } + return cur +} diff --git a/pkg/runtime/optimize/dml/update.go b/pkg/runtime/optimize/dml/update.go index 55aaa1c9..2a87c2a0 100644 --- a/pkg/runtime/optimize/dml/update.go +++ b/pkg/runtime/optimize/dml/update.go @@ -23,6 +23,8 @@ import ( import ( "github.com/pkg/errors" + + "golang.org/x/exp/slices" ) import ( @@ -54,9 +56,13 @@ func optimizeUpdate(ctx context.Context, o *optimize.Optimizer) (proto.Plan, err } // check update sharding key + vShards := vt.GetVShards() for _, element := range stmt.Updated { - if _, _, ok := vt.GetShardMetadata(element.Column.Suffix()); ok { - return nil, errors.New("do not support update sharding key") + column := element.Column.Suffix() + for _, vShard := range vShards { + if slices.Index(vShard.Variables(), column) != -1 { + return nil, errors.Errorf("your value to be updated is belong to a sharding key '%s'", column) + } } } diff --git a/pkg/runtime/optimize/shard_visitor.go b/pkg/runtime/optimize/shard_visitor.go index 6173c04b..4dee416b 100644 --- a/pkg/runtime/optimize/shard_visitor.go +++ b/pkg/runtime/optimize/shard_visitor.go @@ -32,15 +32,17 @@ import ( "github.com/arana-db/arana/pkg/proto" "github.com/arana-db/arana/pkg/proto/rule" "github.com/arana-db/arana/pkg/runtime/ast" + "github.com/arana-db/arana/pkg/runtime/calc" + "github.com/arana-db/arana/pkg/runtime/calc/logic" "github.com/arana-db/arana/pkg/runtime/cmp" - "github.com/arana-db/arana/pkg/runtime/logical" "github.com/arana-db/arana/pkg/runtime/misc" "github.com/arana-db/arana/pkg/runtime/misc/extvalue" - rrule "github.com/arana-db/arana/pkg/runtime/rule" ) var _ ast.Visitor = (*ShardVisitor)(nil) +type Calculus = logic.Logic[*calc.Calculus] + type ShardVisitor struct { ctx context.Context ast.BaseVisitor @@ -57,6 +59,10 @@ func NewXSharder(ctx context.Context, ru *rule.Rule, args []proto.Value) *ShardV } } +func (sd *ShardVisitor) Result() []misc.Pair[ast.TableName, *rule.Shards] { + return sd.results +} + func (sd *ShardVisitor) SimpleShard(table ast.TableName, where ast.ExpressionNode) (rule.DatabaseTables, error) { var ( shards rule.DatabaseTables @@ -113,14 +119,13 @@ func (sd *ShardVisitor) ForSingleSelect(table ast.TableName, alias string, where return errors.WithStack(err) } - ev, err := rrule.Eval(l.(logical.Logical), vtab) - // 2. logical to evaluator + // 2. eval shards + shards, err := calc.Eval(vtab, l.(logic.Logic[*calc.Calculus])) if err != nil { return errors.Wrap(err, "compute shard evaluator failed") } // 3. eval - shards, err := ev.Eval(vtab) - if err != nil && !errors.Is(err, rrule.ErrNoRuleMetadata) { + if err != nil && !errors.Is(err, calc.ErrNoShardMatched) { return errors.Wrap(err, "eval shards failed") } @@ -155,15 +160,11 @@ func (sd *ShardVisitor) VisitLogicalExpression(node *ast.LogicalExpressionNode) return nil, errors.WithStack(err) } - ll, lr := left.(logical.Logical), right.(logical.Logical) - switch node.Op { - case logical.Lor: - return ll.Or(lr), nil - case logical.Land: - return ll.And(lr), nil - default: - panic("unreachable") + ll, lr := left.(Calculus), right.(Calculus) + if node.Or { + return logic.OR(ll, lr), nil } + return logic.AND(ll, lr), nil } func (sd *ShardVisitor) VisitNotExpression(node *ast.NotExpressionNode) (interface{}, error) { @@ -171,12 +172,13 @@ func (sd *ShardVisitor) VisitNotExpression(node *ast.NotExpressionNode) (interfa if err != nil { return nil, errors.WithStack(err) } - return ret.(logical.Logical).Not(), nil + return logic.NOT(ret.(Calculus)), nil } func (sd *ShardVisitor) VisitPredicateExpression(node *ast.PredicateExpressionNode) (interface{}, error) { return node.P.Accept(sd) } + func (sd *ShardVisitor) VisitSelectElementExpr(node *ast.SelectElementExpr) (interface{}, error) { switch inner := node.Expression().(type) { case *ast.PredicateExpressionNode: @@ -186,6 +188,7 @@ func (sd *ShardVisitor) VisitSelectElementExpr(node *ast.SelectElementExpr) (int } return node.Expression().Accept(sd) } + func (sd *ShardVisitor) VisitPredicateAtom(node *ast.AtomPredicateNode) (interface{}, error) { return node.A.Accept(sd) } @@ -205,15 +208,65 @@ func (sd *ShardVisitor) VisitPredicateBetween(node *ast.BetweenPredicateNode) (i if node.Not { // convert: f NOT BETWEEN a AND b -> f < a OR f > b - k1 := rrule.NewKeyed(key.Suffix(), cmp.Clt, l) - k2 := rrule.NewKeyed(key.Suffix(), cmp.Cgt, r) - return k1.ToLogical().Or(k2.ToLogical()), nil + k1, err := newCmp(key.Suffix(), cmp.Clt, l) + if err != nil { + return nil, err + } + k2, err := newCmp(key.Suffix(), cmp.Cgt, r) + if err != nil { + return nil, err + } + return logic.OR( + calc.Wrap(k1), + calc.Wrap(k2), + ), nil } // convert: f BETWEEN a AND b -> f >= a AND f <= b - k1 := rrule.NewKeyed(key.Suffix(), cmp.Cgte, l) - k2 := rrule.NewKeyed(key.Suffix(), cmp.Clte, r) - return k1.ToLogical().And(k2.ToLogical()), nil + k1, err := newCmp(key.Suffix(), cmp.Cgte, l) + if err != nil { + return nil, err + } + k2, err := newCmp(key.Suffix(), cmp.Clte, r) + if err != nil { + return nil, err + } + return logic.AND( + calc.Wrap(k1), + calc.Wrap(k2), + ), nil +} + +func newCmp(key string, comparison cmp.Comparison, v proto.Value) (*cmp.Comparative, error) { + switch v.Family() { + case proto.ValueFamilyString: + return cmp.NewString(key, comparison, v.String()), nil + case proto.ValueFamilySign, proto.ValueFamilyUnsigned, proto.ValueFamilyFloat, proto.ValueFamilyDecimal: + i, err := v.Int64() + if err != nil { + return nil, err + } + return cmp.NewInt64(key, comparison, i), nil + + case proto.ValueFamilyBool: + b, err := v.Bool() + if err != nil { + return nil, err + } + if b { + return cmp.NewInt64(key, comparison, 1), nil + } + return cmp.NewInt64(key, comparison, 0), nil + + case proto.ValueFamilyTime: + t, err := v.Time() + if err != nil { + return nil, errors.WithStack(err) + } + return cmp.NewDate(key, comparison, t), nil + default: + return cmp.NewString(key, comparison, v.String()), nil + } } func (sd *ShardVisitor) VisitPredicateBinaryComparison(node *ast.BinaryComparisonPredicateNode) (interface{}, error) { @@ -222,11 +275,15 @@ func (sd *ShardVisitor) VisitPredicateBinaryComparison(node *ast.BinaryCompariso v, err := extvalue.Compute(sd.ctx, node.Right, sd.args...) if err != nil { if extvalue.IsErrNotSupportedValue(err) { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } return nil, errors.WithStack(err) } - return rrule.NewKeyed(k.Suffix(), node.Op, v).ToLogical(), nil + c, err := newCmp(k.Suffix(), node.Op, v) + if err != nil { + return nil, err + } + return calc.Wrap(c), nil } switch k := node.Right.(*ast.AtomPredicateNode).A.(type) { @@ -234,17 +291,21 @@ func (sd *ShardVisitor) VisitPredicateBinaryComparison(node *ast.BinaryCompariso v, err := extvalue.Compute(sd.ctx, node.Left, sd.args...) if err != nil { if extvalue.IsErrNotSupportedValue(err) { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } return nil, errors.WithStack(err) } - return rrule.NewKeyed(k.Suffix(), node.Op, v).ToLogical(), nil + c, err := newCmp(k.Suffix(), node.Op, v) + if err != nil { + return nil, err + } + return calc.Wrap(c), nil } l, _ := extvalue.Compute(sd.ctx, node.Left, sd.args...) r, _ := extvalue.Compute(sd.ctx, node.Right, sd.args...) if l == nil || r == nil { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } var ( @@ -295,38 +356,45 @@ func (sd *ShardVisitor) VisitPredicateBinaryComparison(node *ast.BinaryCompariso } if bingo { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } - return rrule.AlwaysFalseLogical, nil + return alwaysFalse(), nil } func (sd *ShardVisitor) VisitPredicateIn(node *ast.InPredicateNode) (interface{}, error) { key := node.P.(*ast.AtomPredicateNode).A.(ast.ColumnNameExpressionAtom) - var ret logical.Logical + var ret Calculus for i := range node.E { actualValue, err := extvalue.Compute(sd.ctx, node.E[i], sd.args...) if err != nil { if extvalue.IsErrNotSupportedValue(err) { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } return nil, errors.WithStack(err) } + if node.Not { + ke, err := newCmp(key.Suffix(), cmp.Cne, actualValue) + if err != nil { + return nil, err + } // convert: f NOT IN (a,b,c) -> f <> a AND f <> b AND f <> c - ke := rrule.NewKeyed(key.Suffix(), cmp.Cne, actualValue) if ret == nil { - ret = ke.ToLogical() + ret = calc.Wrap(ke) } else { - ret = ret.And(ke.ToLogical()) + ret = logic.AND(ret, calc.Wrap(ke)) } } else { // convert: f IN (a,b,c) -> f = a OR f = b OR f = c - ke := rrule.NewKeyed(key.Suffix(), cmp.Ceq, actualValue) + ke, err := newCmp(key.Suffix(), cmp.Ceq, actualValue) + if err != nil { + return nil, err + } if ret == nil { - ret = ke.ToLogical() + ret = calc.Wrap(ke) } else { - ret = ret.Or(ke.ToLogical()) + ret = logic.OR(ret, calc.Wrap(ke)) } } } @@ -340,28 +408,33 @@ func (sd *ShardVisitor) VisitPredicateLike(node *ast.LikePredicateNode) (interfa like, err := extvalue.Compute(sd.ctx, node.Right, sd.args...) if err != nil { if extvalue.IsErrNotSupportedValue(err) { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } return nil, errors.WithStack(err) } if like == nil { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } if !strings.ContainsAny(like.String(), "%_") { - return rrule.NewKeyed(key.Suffix(), cmp.Ceq, like).ToLogical(), nil + c, err := newCmp(key.Suffix(), cmp.Ceq, like) + if err != nil { + return nil, err + } + + return calc.Wrap(c), nil } - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } -func (sd *ShardVisitor) VisitPredicateRegexp(node *ast.RegexpPredicationNode) (interface{}, error) { - return rrule.AlwaysTrueLogical, nil +func (sd *ShardVisitor) VisitPredicateRegexp(_ *ast.RegexpPredicationNode) (interface{}, error) { + return alwaysTrue(), nil } -func (sd *ShardVisitor) VisitAtomColumn(node ast.ColumnNameExpressionAtom) (interface{}, error) { - return rrule.AlwaysTrueLogical, nil +func (sd *ShardVisitor) VisitAtomColumn(_ ast.ColumnNameExpressionAtom) (interface{}, error) { + return alwaysTrue(), nil } func (sd *ShardVisitor) VisitAtomConstant(node *ast.ConstantExpressionAtom) (interface{}, error) { @@ -380,7 +453,7 @@ func (sd *ShardVisitor) VisitAtomFunction(node *ast.FunctionCallExpressionAtom) val, err := extvalue.Compute(sd.ctx, node, sd.args...) if err != nil { if extvalue.IsErrNotSupportedValue(err) { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } return nil, errors.WithStack(err) } @@ -399,37 +472,44 @@ func (sd *ShardVisitor) VisitAtomMath(node *ast.MathExpressionAtom) (interface{} return sd.fromValueNode(node) } -func (sd *ShardVisitor) VisitAtomSystemVariable(node *ast.SystemVariableExpressionAtom) (interface{}, error) { - return rrule.AlwaysTrueLogical, nil +func (sd *ShardVisitor) VisitAtomSystemVariable(_ *ast.SystemVariableExpressionAtom) (interface{}, error) { + return alwaysTrue(), nil } func (sd *ShardVisitor) VisitAtomVariable(node ast.VariableExpressionAtom) (interface{}, error) { return sd.fromConstant(sd.args[node.N()]) } -func (sd *ShardVisitor) VisitAtomInterval(node *ast.IntervalExpressionAtom) (interface{}, error) { - return rrule.AlwaysTrueLogical, nil +func (sd *ShardVisitor) VisitAtomInterval(_ *ast.IntervalExpressionAtom) (interface{}, error) { + return alwaysTrue(), nil } -func (sd *ShardVisitor) fromConstant(val proto.Value) (logical.Logical, error) { +func (sd *ShardVisitor) fromConstant(val proto.Value) (Calculus, error) { if val == nil { - return rrule.AlwaysFalseLogical, nil + return alwaysFalse(), nil } if b, err := val.Bool(); err == nil && !b { - return rrule.AlwaysFalseLogical, nil + return alwaysFalse(), nil } - - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } func (sd *ShardVisitor) fromValueNode(node ast.Node) (interface{}, error) { val, err := extvalue.Compute(sd.ctx, node, sd.args...) if err != nil { if extvalue.IsErrNotSupportedValue(err) { - return rrule.AlwaysTrueLogical, nil + return alwaysTrue(), nil } return nil, errors.WithStack(err) } return sd.fromConstant(val) } + +func alwaysTrue() Calculus { + return logic.True[*calc.Calculus]() +} + +func alwaysFalse() Calculus { + return logic.False[*calc.Calculus]() +} diff --git a/pkg/runtime/optimize/shard_visitor_test.go b/pkg/runtime/optimize/shard_visitor_test.go index 04bd3d70..145d6b20 100644 --- a/pkg/runtime/optimize/shard_visitor_test.go +++ b/pkg/runtime/optimize/shard_visitor_test.go @@ -20,6 +20,7 @@ package optimize_test import ( "context" "fmt" + "sort" "strconv" "testing" ) @@ -57,6 +58,7 @@ func TestShardNG(t *testing.T) { {"select * from student where uid = PI() div 3", nil, []int{1}}, {"select * from student where uid = PI() div ?", []interface{}{3}, []int{1}}, {"select * from student where 1+2", nil, nil}, + {"select * from student where uid between 1 and 3", nil, []int{1, 2, 3}}, } { t.Run(it.sql, func(t *testing.T) { _, rawStmt := ast.MustParse(it.sql) @@ -71,9 +73,19 @@ func TestShardNG(t *testing.T) { shd := NewXSharder(context.TODO(), fakeRule, args) - shards, err := stmt.Accept(shd) + _, err := stmt.Accept(shd) assert.NoError(t, err) - t.Log("shards:", shards) + + t.Log("shard results:", shd.Result()) + + res := shd.Result()[0] + var actual []int + res.R.Each(func(_, tb uint32) bool { + actual = append(actual, int(tb)) + return true + }) + sort.Ints(actual) + assert.Equal(t, it.expect, actual) }) } } @@ -104,7 +116,7 @@ func makeFakeRule(c *gomock.Controller, mod int) *rule.Rule { computer.EXPECT(). Compute(gomock.Any()). - DoAndReturn(func(value interface{}) (int, error) { + DoAndReturn(func(value proto.Value) (int, error) { n, err := strconv.Atoi(fmt.Sprintf("%v", value)) if err != nil { return 0, err @@ -113,11 +125,26 @@ func makeFakeRule(c *gomock.Controller, mod int) *rule.Rule { }). MinTimes(1) - var sm rule.ShardMetadata - sm.Steps = 8 - sm.Computer = computer + computer.EXPECT().Variables().Return([]string{"uid"}).AnyTimes() + + sm := &rule.ShardMetadata{ + ShardColumns: []*rule.ShardColumn{ + { + Name: "uid", + Steps: 8, + Stepper: rule.Stepper{ + N: 1, + U: rule.Unum, + }, + }, + }, + Computer: computer, + } + + tab.AddVShards(&rule.VShard{ + Table: sm, + }) - tab.SetShardMetadata("uid", nil, &sm) ru.SetVTable("student", &tab) return &ru } diff --git a/pkg/runtime/plan/dal/show_database_rules.go b/pkg/runtime/plan/dal/show_database_rules.go index 67763ac8..9281f20c 100644 --- a/pkg/runtime/plan/dal/show_database_rules.go +++ b/pkg/runtime/plan/dal/show_database_rules.go @@ -19,14 +19,16 @@ package dal import ( "context" - "github.com/arana-db/arana/pkg/mysql/rows" - "github.com/arana-db/arana/pkg/proto/rule" + "fmt" + "strings" ) import ( "github.com/arana-db/arana/pkg/dataset" + "github.com/arana-db/arana/pkg/mysql/rows" "github.com/arana-db/arana/pkg/mysql/thead" "github.com/arana-db/arana/pkg/proto" + "github.com/arana-db/arana/pkg/proto/rule" "github.com/arana-db/arana/pkg/resultx" "github.com/arana-db/arana/pkg/runtime/ast" "github.com/arana-db/arana/pkg/runtime/plan" @@ -53,13 +55,18 @@ func (s *ShowDatabaseRulesPlan) ExecIn(ctx context.Context, _ proto.VConn) (prot Columns: fields, } - dbRules := s.rule.DBRule() - if rules, ok := dbRules[s.Stmt.TableName]; ok { - for _, ruleItem := range rules { + if vt, ok := s.rule.VTable(s.Stmt.TableName); ok { + for _, vs := range vt.GetVShards() { + var columns []string + for i := range vs.DB.ShardColumns { + columns = append(columns, vs.DB.ShardColumns[i].Name) + } ds.Rows = append(ds.Rows, rows.NewTextVirtualRow(fields, []proto.Value{ proto.NewValueString(s.Stmt.TableName), - proto.NewValueString(ruleItem.Column), proto.NewValueString(ruleItem.Type), - proto.NewValueString(ruleItem.Expr), proto.NewValueInt64(int64(ruleItem.Step)), + proto.NewValueString(strings.Join(columns, ",")), + proto.NewValueString(""), + proto.NewValueString(fmt.Sprintf("%s", vs.DB.Computer)), + proto.NewValueInt64(1), })) } } diff --git a/pkg/runtime/plan/dal/show_database_rules_test.go b/pkg/runtime/plan/dal/show_database_rules_test.go index 876f50ab..c6bebf27 100644 --- a/pkg/runtime/plan/dal/show_database_rules_test.go +++ b/pkg/runtime/plan/dal/show_database_rules_test.go @@ -23,11 +23,12 @@ import ( import ( "github.com/arana-db/parser" + "github.com/stretchr/testify/assert" ) func TestShowDatabaseRulesSQL(t *testing.T) { - sql := "SHOW DATABASE RULES FROM employees" + sql := "SHOW DATABASE RULES FROM student" p := parser.New() diff --git a/pkg/runtime/plan/dal/show_sharding_table.go b/pkg/runtime/plan/dal/show_sharding_table.go index c9890bf8..b7bb2b85 100644 --- a/pkg/runtime/plan/dal/show_sharding_table.go +++ b/pkg/runtime/plan/dal/show_sharding_table.go @@ -97,8 +97,10 @@ func (st *ShowShardingTable) ExecIn(ctx context.Context, conn proto.VConn) (prot ds.Rows = append(ds.Rows, rows.NewTextVirtualRow(fields, []proto.Value{ proto.NewValueString(rawMetadata["name"]), - proto.NewValueString(rawMetadata["sequence_type"]), proto.NewValueString(rawMetadata["db_rules"]), - proto.NewValueString(rawMetadata["tbl_rules"]), proto.NewValueString(rawMetadata["attributes"]), + proto.NewValueString(rawMetadata["sequence_type"]), + proto.NewValueString(rawMetadata["db_rules"]), + proto.NewValueString(rawMetadata["tbl_rules"]), + proto.NewValueString(rawMetadata["attributes"]), })) } diff --git a/pkg/runtime/rule/evaluator.go b/pkg/runtime/rule/evaluator.go deleted file mode 100644 index a9536f89..00000000 --- a/pkg/runtime/rule/evaluator.go +++ /dev/null @@ -1,775 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - stdErrors "errors" - "fmt" -) - -import ( - "github.com/pkg/errors" -) - -import ( - "github.com/arana-db/arana/pkg/proto" - "github.com/arana-db/arana/pkg/proto/rule" - "github.com/arana-db/arana/pkg/runtime/cmp" - "github.com/arana-db/arana/pkg/runtime/logical" - "github.com/arana-db/arana/pkg/runtime/misc" -) - -var ( - _emptyEvaluator Evaluator = emptyEvaluator{} - _noopEvaluator Evaluator = noopEvaluator{} - - AlwaysTrueLogical = logical.New("1", logical.WithValue(_noopEvaluator)) - AlwaysFalseLogical = logical.New("0", logical.WithValue(_emptyEvaluator)) -) - -var ( - _ Evaluator = (*KeyedEvaluator)(nil) - _ Evaluator = noopEvaluator{} - _ Evaluator = (*staticEvaluator)(nil) -) - -var ErrNoRuleMetadata = stdErrors.New("no rule metadata found") - -var emptyDatabaseTables rule.DatabaseTables - -func init() { - emptyDatabaseTables = make(map[string][]string) -} - -func toRangeIterator(begin, end rule.Range) rule.Range { - var a []interface{} - - for begin.HasNext() { - a = append(a, begin.Next()) - } - - if len(a) < 1 { - return Multiple() - } - - var max interface{} - if !end.HasNext() { - return Multiple() - } - - max = end.Next() - - var merged []interface{} - - for i := 0; i < len(a); i++ { - if misc.Compare(a[i], max) == 1 { - merged = append(merged, max) - break - } - merged = append(merged, a[i]) - } - - if len(merged) < 1 { - return Multiple() - } - - return Multiple(merged...) -} - -// Evaluator evaluates the sharding result. -type Evaluator interface { - Not() Evaluator - // Eval evaluates the sharding result. - Eval(vtab *rule.VTable) (*rule.Shards, error) -} - -type emptyEvaluator struct{} - -func (e emptyEvaluator) Not() Evaluator { - return _noopEvaluator -} - -func (e emptyEvaluator) String() string { - return "NONE" -} - -func (e emptyEvaluator) Eval(_ *rule.VTable) (*rule.Shards, error) { - return rule.NewShards(), nil -} - -type staticEvaluator rule.Shards - -func (s *staticEvaluator) Not() Evaluator { - return _noopEvaluator -} - -func (s *staticEvaluator) String() string { - return (*rule.Shards)(s).String() -} - -func (s *staticEvaluator) Eval(_ *rule.VTable) (*rule.Shards, error) { - return (*rule.Shards)(s), nil -} - -type noopEvaluator struct{} - -func (n noopEvaluator) Not() Evaluator { - return n -} - -func (n noopEvaluator) String() string { - return "FULL" // Infinity -} - -func (n noopEvaluator) Eval(_ *rule.VTable) (*rule.Shards, error) { - return nil, nil -} - -type KeyedEvaluator struct { - k string - op cmp.Comparison - v proto.Value -} - -func (t *KeyedEvaluator) toComparative(metadata *rule.ShardMetadata) *cmp.Comparative { - var ( - s string - k cmp.Kind - ) - - if t.v == nil { - return nil - } - - switch t.v.Family() { - case proto.ValueFamilyString: - k = cmp.Kstring - s = t.v.String() - case proto.ValueFamilyTime: - k = cmp.Kdate - dt, _ := t.v.Time() - s = dt.Format("2006-01-02 15:04:05") - case proto.ValueFamilyDecimal, proto.ValueFamilySign, proto.ValueFamilyUnsigned, proto.ValueFamilyFloat, proto.ValueFamilyBool: - k = cmp.Kint - s = t.v.String() - } - - if metadata != nil { - switch metadata.Stepper.U { - case rule.Umonth, rule.Uyear, rule.Uweek, rule.Uday, rule.Uhour: - k = cmp.Kdate - case rule.Unum: - k = cmp.Kint - case rule.Ustr: - k = cmp.Kstring - } - } - - return cmp.New(t.k, t.op, s, k) -} - -func (t *KeyedEvaluator) String() string { - return fmt.Sprintf("%s %s %v", t.k, t.op, t.v) -} - -func (t *KeyedEvaluator) ToLogical() logical.Logical { - // NOTICE: sort the logical operations, eg: a > 1 AND a > 2 AND a < 1 AND a < 2 - var suffix string - if t.v != nil { - suffix = t.v.String() - } - return logical.New(t.String(), logical.WithValue(t), logical.WithSortKey(fmt.Sprintf("%s|%d|%s", t.k, t.op, suffix))) -} - -func (t *KeyedEvaluator) Eval(vt *rule.VTable) (*rule.Shards, error) { - var actualMetadata *rule.ShardMetadata - - dbMetadata, tbMetadata, ok := vt.GetShardMetadata(t.k) - if !ok || (dbMetadata == nil && tbMetadata == nil) { - return nil, errors.Wrapf(ErrNoRuleMetadata, "cannot get rule metadata %s.%s", vt.Name(), t.k) - } - - if dbMetadata == tbMetadata || tbMetadata != nil { - actualMetadata = tbMetadata - } else { - actualMetadata = dbMetadata - } - - mat, err := Route(vt, t.toComparative(actualMetadata)) - if err != nil { - return nil, err - } - it, err := mat.Eval() - if err != nil { - return nil, err - } - if it == nil { - return nil, nil - } - return MatchTables(vt, t.k, it) -} - -func (t *KeyedEvaluator) Not() Evaluator { - var op cmp.Comparison - switch t.op { - case cmp.Cgt: - op = cmp.Clte - case cmp.Cgte: - op = cmp.Clt - case cmp.Clt: - op = cmp.Cgte - case cmp.Clte: - op = cmp.Clt - case cmp.Ceq: - op = cmp.Cne - case cmp.Cne: - op = cmp.Ceq - default: - panic("unreachable") - } - - ret := &KeyedEvaluator{} - *ret = *t - ret.op = op - - return ret -} - -func EvalWithVTable(l logical.Logical, vtab *rule.VTable) (Evaluator, error) { - ret, err := logical.Eval(l, func(a, b interface{}) (interface{}, error) { - x := a.(Evaluator) - y := b.(Evaluator) - z, err := and(vtab, x, y) - if err == nil { - return z, nil - } - if errors.Is(err, ErrNoRuleMetadata) { - return _noopEvaluator, nil - } - return nil, errors.WithStack(err) - }, func(a, b interface{}) (interface{}, error) { - x := a.(Evaluator) - y := b.(Evaluator) - z, err := or(vtab, x, y) - - if err == nil { - return z, nil - } - if errors.Is(err, ErrNoRuleMetadata) { - return _noopEvaluator, nil - } - return nil, errors.WithStack(err) - }, func(i interface{}) interface{} { - x := i.(Evaluator) - return x.Not() - }) - if err != nil { - return nil, err - } - return ret.(Evaluator), nil -} - -func Eval(l logical.Logical, vtab *rule.VTable) (Evaluator, error) { - ret, err := logical.Eval(l, func(a, b interface{}) (interface{}, error) { - x := a.(Evaluator) - y := b.(Evaluator) - z, err := and(vtab, x, y) - if err == nil { - return z, nil - } - if errors.Is(err, ErrNoRuleMetadata) { - return _noopEvaluator, nil - } - return nil, errors.WithStack(err) - }, func(a, b interface{}) (interface{}, error) { - x := a.(Evaluator) - y := b.(Evaluator) - z, err := or(vtab, x, y) - - if err == nil { - return z, nil - } - if errors.Is(err, ErrNoRuleMetadata) { - return _noopEvaluator, nil - } - return nil, errors.WithStack(err) - }, func(i interface{}) interface{} { - x := i.(Evaluator) - return x.Not() - }) - if err != nil { - return nil, err - } - return ret.(Evaluator), nil -} - -func or(vt *rule.VTable, first, second Evaluator) (Evaluator, error) { - if first == _noopEvaluator || second == _noopEvaluator { - return _noopEvaluator, nil - } - if first == _emptyEvaluator { - return second, nil - } - if second == _emptyEvaluator { - return first, nil - } - - switch a := first.(type) { - case *KeyedEvaluator: - if !vt.HasColumn(a.k) { - return _noopEvaluator, nil - } - } - - switch b := second.(type) { - case *KeyedEvaluator: - if !vt.HasColumn(b.k) { - return _noopEvaluator, nil - } - } - - v1, err := first.Eval(vt) - if err != nil { - return nil, err - } - v2, err := second.Eval(vt) - if err != nil { - return nil, err - } - - if v1 == nil && v2 == nil { - return _noopEvaluator, nil - } - - union := rule.UnionShards(v1, v2) - if union.Len() < 1 { - return _emptyEvaluator, nil - } - - return (*staticEvaluator)(union), nil -} - -func processRange(vt *rule.VTable, begin, end *KeyedEvaluator) (Evaluator, error) { - dbm1, tbm1, ok := vt.GetShardMetadata(begin.k) - if !ok { - return nil, ErrNoRuleMetadata - } - - dbm2, tbm2, ok := vt.GetShardMetadata(end.k) - if !ok { - return nil, ErrNoRuleMetadata - } - - var m1, m2 *rule.ShardMetadata - - if tbm1 != nil && tbm2 != nil { - m1, m2 = tbm1, tbm2 - } else if dbm1 != nil && dbm2 != nil { - m1, m2 = dbm1, dbm2 - } else { - return nil, errors.Errorf("no available rule metadata found: fields=[%s,%s]", begin.k, end.k) - } - - // string range is not available, use full-scan instead - if m1.Stepper.U == rule.Ustr && m2.Stepper.U == rule.Ustr { - return _noopEvaluator, nil - } - - mat, err := Route(vt, begin.toComparative(m1)) - if err != nil { - return nil, err - } - beginIt, err := mat.Eval() - if err != nil { - return nil, err - } - - mat, err = Route(vt, end.toComparative(m2)) - if err != nil { - return nil, err - } - endIt, err := mat.Eval() - if err != nil { - return nil, err - } - - if beginIt == nil || endIt == nil { - return _noopEvaluator, nil - } - - it := toRangeIterator(beginIt, endIt) - dt, err := MatchTables(vt, begin.k, it) - if err != nil { - return nil, err - } - - if dt == nil { - return _noopEvaluator, nil - } - - if dt.Len() < 1 { - return _emptyEvaluator, nil - } - - return (*staticEvaluator)(dt), nil -} - -func and(vtab *rule.VTable, first, second Evaluator) (Evaluator, error) { - if first == _emptyEvaluator || second == _emptyEvaluator { - return _emptyEvaluator, nil - } - - if first == _noopEvaluator { - return second, nil - } - if second == _noopEvaluator { - return first, nil - } - - k1, ok1 := first.(*KeyedEvaluator) - k2, ok2 := second.(*KeyedEvaluator) - - if ok1 && ok2 { - if k1.k == k2.k { // same key, handle comparison. - var rangeMode int8 // 0: - switch k1.op { - case cmp.Ceq: - switch k2.op { - case cmp.Ceq: - // a = 1 AND a = 1 => a = 1 - if k1.v == k2.v { - return k1, nil - } - // a = 1 AND a = 2 => always false - return _emptyEvaluator, nil - case cmp.Cne: - // a = 1 AND a <> 1 => always false - if k1.v == k2.v { - return _emptyEvaluator, nil - } - // a = 1 AND a <> 2 => a = 1 - return k1, nil - case cmp.Clt: - switch misc.Compare(k2.v, k1.v) { - case -1, 0: - // a = 1 AND a < 1 => always false - return _emptyEvaluator, nil - default: - // a = 1 AND a < 2 => a = 1 - return k1, nil - } - case cmp.Clte: - switch misc.Compare(k2.v, k1.v) { - case -1: - // a = 2 AND a <= 1 => always false - return _emptyEvaluator, nil - default: - // a = 2 AND a <= 2 => a = 2 - return k1, nil - } - case cmp.Cgt: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a = 1 AND a > 1 => always false - return _emptyEvaluator, nil - default: - // a = 1 AND a > 0 => a = 1 - return k1, nil - } - case cmp.Cgte: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a = 1 AND a >= 2 => always false - return _emptyEvaluator, nil - default: - // a = 3 AND a >= 1 => a = 3 - return k1, nil - } - } - case cmp.Cne: - switch k2.op { - case cmp.Ceq: - if k1.v == k2.v { // a <> 1 AND a = 1 -> always false - return _emptyEvaluator, nil - } - // a <> 1 AND a = 2 -> a = 2 - return k2, nil - } - case cmp.Clt: - switch k2.op { - case cmp.Cne: - // FIXME: a < 1 AND a <> 2 - return k1, nil - case cmp.Ceq: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a < 1 AND a = 1 => always false - return _emptyEvaluator, nil - default: - // a < 3 AND a = 1 => a = 1 - return k2, nil - } - case cmp.Cgt: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a < 1 AND a > 1 => always false - return _emptyEvaluator, nil - default: - rangeMode = -1 - } - case cmp.Cgte: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a < 1 AND a >= 2 => always false - // a < 1 AND a >= 1 => always false - return _emptyEvaluator, nil - default: - rangeMode = -1 - } - case cmp.Clt: - switch misc.Compare(k2.v, k1.v) { - case -1: - // a < 2 AND a < 1 => a < 1 - return k2, nil - default: - // a < 1 AND a < 2 => a < 1 - // a < 1 AND a < 1 => a < 1 - return k1, nil - } - case cmp.Clte: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a < 1 AND a <= 2 => a < 1 - // a < 1 AND a <= 1 => a < 1 - return k1, nil - default: - // a < 2 AND a <= 1 => a <= 1 - return k2, nil - } - } - case cmp.Clte: - switch k2.op { - case cmp.Ceq: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a <= 1 AND a = 2 => always false - return _emptyEvaluator, nil - default: - return k2, nil - } - case cmp.Cne: - // FIXME: a <= 1 AND a <> 2 - return k1, nil - case cmp.Clt: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a <= 1 AND a < 2 => a <= 1 - return k1, nil - default: - // a <= 3 AND a < 3 => a < 3 - // a <= 3 AND a < 1 => a < 1 - return k2, nil - } - case cmp.Clte: - switch misc.Compare(k2.v, k1.v) { - case -1: - // a <= 2 AND a <= 1 => a <= 1 - return k2, nil - default: - // a <= 2 AND a <= 3 => a <= 2 - // a <= 2 AND a <= 2 => a <= 2 - return k1, nil - } - case cmp.Cgt: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a <= 1 AND a > 1 => always false - return _emptyEvaluator, nil - default: - rangeMode = -1 - } - case cmp.Cgte: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a <= 1 AND a >= 2 => always false - return _emptyEvaluator, nil - case 0: - // a <= 1 AND a >= 1 => a = 1 - return NewKeyed(k1.k, cmp.Ceq, k1.v), nil - default: - rangeMode = -1 - } - } - case cmp.Cgt: - switch k2.op { - case cmp.Ceq: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a > 1 AND a = 2 => a = 2 - return k2, nil - default: - // a > 2 AND a = 2 => always false - // a > 2 AND a = 1 => always false - return _emptyEvaluator, nil - } - case cmp.Cne: - // FIXME: a > 1 AND a <> 2 - return k1, nil - case cmp.Cgt: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a > 3 AND a > 4 => a > 4 - return k2, nil - default: - // a > 3 AND a > 2 => a > 3 - // a > 3 AND a > 3 => a > 3 - return k1, nil - } - case cmp.Cgte: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a > 1 AND a >= 2 => a >= 2 - return k2, nil - default: - // a > 1 AND a >= 1 => a > 1 - // a > 3 AND a >= 2 => a > 3 - return k1, nil - } - case cmp.Clt: - switch misc.Compare(k2.v, k1.v) { - case -1, 0: - // a > 2 AND b < 1 => always false - // a > 2 AND b < 2 => always false - return _emptyEvaluator, nil - default: - rangeMode = 1 - } - case cmp.Clte: - switch misc.Compare(k2.v, k1.v) { - case -1, 0: - // a > 2 AND b <= 1 => always false - // a > 2 AND b <= 2 => always false - return _emptyEvaluator, nil - default: - rangeMode = 1 - } - } - case cmp.Cgte: - switch k2.op { - case cmp.Ceq: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a >= 1 AND a = 2 => a = 2 - // a >= 1 AND a = 1 => a = 1 - return k2, nil - default: - // a >= 2 AND a = 1 => always false - return _emptyEvaluator, nil - } - case cmp.Cne: - // FIXME: a >= 1 AND a <> 2 - return k1, nil - case cmp.Cgt: - switch misc.Compare(k2.v, k1.v) { - case 1, 0: - // a >= 1 AND a > 2 => a > 2 - // a >= 1 AND a > 1 => a > 1 - return k2, nil - default: - // a >= 2 AND a > 1 => a >= 2 - return k1, nil - } - case cmp.Cgte: - switch misc.Compare(k2.v, k1.v) { - case 1: - // a >= 1 AND a >= 2 => a >= 2 - return k2, nil - default: - // a >= 2 AND a >= 1 => a >= 2 - // a >= 2 AND a >= 2 => a >= 2 - return k1, nil - } - case cmp.Clt: - switch misc.Compare(k2.v, k1.v) { - case -1, 0: - // a >= 3 AND a < 2 => always false - // a >= 3 AND a < 3 => always false - return _emptyEvaluator, nil - default: - rangeMode = 1 - } - case cmp.Clte: - switch misc.Compare(k2.v, k1.v) { - case -1: - // a >= 3 AND a <= 2 => always false - return _emptyEvaluator, nil - case 0: - return NewKeyed(k1.k, cmp.Ceq, k1.v), nil - default: - rangeMode = 1 - } - } - } - - switch rangeMode { - case 1: - return processRange(vtab, k1, k2) - case -1: - return processRange(vtab, k2, k1) - } - } else if vtab.HasColumn(k1.k) && vtab.HasColumn(k2.k) { - // SKIP: multiple sharding keys, goto slow path - } else if vtab.HasColumn(k1.k) { - return k1, nil - } else if vtab.HasColumn(k2.k) { - return k2, nil - } else { - return _noopEvaluator, nil - } - } - - // slow path - v1, err := first.Eval(vtab) - if err != nil { - return nil, err - } - v2, err := second.Eval(vtab) - if err != nil { - return nil, err - } - - if v1 == nil && v2 == nil { - return _noopEvaluator, nil - } - - merged := rule.IntersectionShards(v1, v2) - - if merged.Len() < 1 { - return _emptyEvaluator, nil - } - - return (*staticEvaluator)(merged), nil -} - -func NewKeyed(key string, op cmp.Comparison, value proto.Value) *KeyedEvaluator { - return &KeyedEvaluator{ - k: key, - op: op, - v: value, - } -} diff --git a/pkg/runtime/rule/evaluator_test.go b/pkg/runtime/rule/evaluator_test.go deleted file mode 100644 index ff285a6f..00000000 --- a/pkg/runtime/rule/evaluator_test.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - "fmt" - "strconv" - "testing" -) - -import ( - "github.com/stretchr/testify/assert" -) - -import ( - "github.com/arana-db/arana/pkg/proto" - "github.com/arana-db/arana/pkg/proto/rule" - "github.com/arana-db/arana/pkg/runtime/cmp" -) - -const ( - fakeDB = "fake_db" - fakeTable = "fake_table" -) - -type simpleModComputer int - -func (s simpleModComputer) Compute(value interface{}) (int, error) { - n, err := strconv.ParseInt(fmt.Sprintf("%v", value), 10, 64) - if err != nil { - return 0, err - } - return int(n) % int(s), nil -} - -func makeTestRule(mod int) *rule.Rule { - var ( - vt rule.VTable - topo rule.Topology - ) - var s []int - for i := 0; i < mod; i++ { - s = append(s, i) - } - topo.SetTopology(0, s...) // single database, multiple tables - topo.SetRender(func(_ int) string { - return fakeDB - }, func(i int) string { - return fmt.Sprintf("%s_%04d", fakeTable, i) - }) - - vt.SetTopology(&topo) - - sm := &rule.ShardMetadata{ - Stepper: rule.Stepper{ - N: 1, - U: rule.Unum, - }, - Computer: simpleModComputer(mod), - } - - vt.SetShardMetadata("uid", nil, sm) - - var ru rule.Rule - ru.SetVTable(fakeTable, &vt) - return &ru -} - -func TestEvaluator_Eval(t *testing.T) { - var ( - ru4 = makeTestRule(4) - ru8 = makeTestRule(8) - ) - t.Run("Basic", func(t *testing.T) { - vtab, _ := ru8.VTable(fakeTable) - - a := NewKeyed("uid", cmp.Cgte, proto.NewValueInt64(10)).ToLogical() - b := NewKeyed("uid", cmp.Clte, proto.NewValueInt64(13)).ToLogical() - c := a.And(b) - - v, err := Eval(c, vtab) - assert.NoError(t, err) - res, err := v.Eval(vtab) - assert.NoError(t, err) - // 2,3,4,5 - t.Log("result:", res) - }) - - t.Run("LogicTuning", func(t *testing.T) { - vtab, _ := ru4.VTable(fakeTable) - - // "select * from `tb_user` where (id > 1 or uid = 10003 ) and (id > 1 or uid = 10004 or uid = 10005)"; - k1 := NewKeyed("id", cmp.Cgt, proto.NewValueInt64(1)).ToLogical() - k2 := NewKeyed("uid", cmp.Ceq, proto.NewValueInt64(10003)).ToLogical() - k3 := NewKeyed("id", cmp.Cgt, proto.NewValueInt64(1)).ToLogical() - k4 := NewKeyed("uid", cmp.Ceq, proto.NewValueInt64(10004)).ToLogical() - k5 := NewKeyed("uid", cmp.Ceq, proto.NewValueInt64(10005)).ToLogical() - - l := k1.Or(k2).And(k3.Or(k4).Or(k5)) - - // should be optimized to 'id > 1' - // (id > 1 or uid = 10003 ) and (id > 1 or uid = 10004 or uid = 10005) - // ( id > 1 OR ( uid = 10003 AND uid = 10005 ) OR ( uid = 10003 AND uid = 10004 ) ) - // id > 1 - v, err := Eval(l, vtab) - assert.NoError(t, err) - assert.Equal(t, "id > 1", v.(fmt.Stringer).String()) - }) - - t.Run("AlwayFalse", func(t *testing.T) { - vtab, _ := ru4.VTable(fakeTable) - - l1 := NewKeyed("uid", cmp.Cgte, proto.NewValueInt64(4)).ToLogical() - l2 := NewKeyed("uid", cmp.Cgte, proto.NewValueInt64(7)).ToLogical() - l3 := NewKeyed("uid", cmp.Clt, proto.NewValueInt64(8)).ToLogical() - - l := l1.And(l2).And(l3) // always false: uid >= 4 AND uid >= 7 AND uid < 8 - - v, err := Eval(l, vtab) - assert.NoError(t, err) - t.Logf("%s => %s", l.ToString("AND", "OR"), v) - }) -} diff --git a/pkg/runtime/rule/iterator.go b/pkg/runtime/rule/iterator.go deleted file mode 100644 index 3f12bf19..00000000 --- a/pkg/runtime/rule/iterator.go +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - "github.com/arana-db/arana/pkg/proto/rule" -) - -var ( - _ rule.Range = (*singleRange)(nil) - _ rule.Range = (*sliceRange)(nil) -) - -type sliceRange struct { - cur int - values []interface{} -} - -func (s *sliceRange) HasNext() bool { - return s.cur < len(s.values) -} - -func (s *sliceRange) Next() interface{} { - ret := s.values[s.cur] - s.cur++ - return ret -} - -type singleRange struct { - n uint8 - value interface{} -} - -func (s *singleRange) HasNext() bool { - return s.n == 0 -} - -func (s *singleRange) Next() interface{} { - if s.n == 0 { - s.n += 1 - } - return s.value -} - -type filterRange struct { - inner rule.Range - filter func(next interface{}) bool - next interface{} -} - -func (f *filterRange) HasNext() bool { - if !f.inner.HasNext() { - f.inner = nil - return false - } - next := f.inner.Next() - if f.filter(next) { - f.next = next - return true - } - return f.HasNext() -} - -func (f *filterRange) Next() interface{} { - return f.next -} - -// Multiple wraps multiple values as an Iterator. -func Multiple(values ...interface{}) rule.Range { - return &sliceRange{ - values: values, - } -} - -// Single wraps a single value as an Iterator. -func Single(value interface{}) rule.Range { - return &singleRange{ - value: value, - } -} - -// Filter wraps an existing Iterator with a filter. -func Filter(src rule.Range, predicate func(interface{}) bool) rule.Range { - return &filterRange{ - inner: src, - filter: predicate, - } -} diff --git a/pkg/runtime/rule/iterator_test.go b/pkg/runtime/rule/iterator_test.go deleted file mode 100644 index c379542d..00000000 --- a/pkg/runtime/rule/iterator_test.go +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - "testing" -) - -import ( - "github.com/golang/mock/gomock" - - "github.com/stretchr/testify/assert" -) - -import ( - "github.com/arana-db/arana/pkg/proto/rule" - "github.com/arana-db/arana/testdata" -) - -func TestFilter(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockRange := testdata.NewMockRange(ctrl) - type args struct { - src rule.Range - predicate func(interface{}) bool - } - tests := []struct { - name string - args args - want rule.Range - }{ - { - "TestFilter", - args{mockRange, nil}, - &filterRange{inner: mockRange, filter: nil}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, Filter(tt.args.src, tt.args.predicate), "Filter(%v, %v)", tt.args.src, tt.args.predicate) - }) - } -} - -func TestMultiple(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - slice := make([]interface{}, 0) - for i := 0; i < 3; i++ { - mockRange := testdata.NewMockRange(ctrl) - slice = append(slice, mockRange) - } - type args struct { - values []interface{} - } - tests := []struct { - name string - args args - want rule.Range - }{ - {"TestMultiple", args{slice}, &sliceRange{0, slice}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, Multiple(tt.args.values...), "Multiple(%v)", tt.args.values...) - }) - } -} - -func TestSingle(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockRange := testdata.NewMockRange(ctrl) - type args struct { - value interface{} - } - tests := []struct { - name string - args args - want rule.Range - }{ - {"TestSingle", args{mockRange}, &singleRange{0, mockRange}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, Single(tt.args.value), "Single(%v)", tt.args.value) - }) - } -} - -func Test_filterRange_HasNext(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - filter := func(interface{}) bool { return true } - mockRange := testdata.NewMockRange(ctrl) - mockRange.EXPECT().HasNext().Return(true) - mockRange.EXPECT().Next().Return(filter) - type fields struct { - inner rule.Range - filter func(next interface{}) bool - next interface{} - } - tests := []struct { - name string - fields fields - want bool - }{ - {"HasNext", fields{mockRange, filter, nil}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - f := &filterRange{ - inner: tt.fields.inner, - filter: tt.fields.filter, - next: tt.fields.next, - } - assert.Equalf(t, tt.want, f.HasNext(), "HasNext()") - }) - } -} - -func Test_filterRange_Next(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockRange := testdata.NewMockRange(ctrl) - filter := func(interface{}) bool { return true } - type fields struct { - inner rule.Range - filter func(next interface{}) bool - next interface{} - } - tests := []struct { - name string - fields fields - want interface{} - }{ - {"Next", fields{mockRange, filter, "filter_range_next"}, "filter_range_next"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - f := &filterRange{ - inner: tt.fields.inner, - filter: tt.fields.filter, - next: tt.fields.next, - } - assert.Equalf(t, tt.want, f.Next(), "Next()") - }) - } -} - -func Test_singleRange_HasNext(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - mockRange := testdata.NewMockRange(ctrl) - type fields struct { - n uint8 - value interface{} - } - tests := []struct { - name string - fields fields - want bool - }{ - {"HasNext", fields{0, mockRange}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := &singleRange{ - n: tt.fields.n, - value: tt.fields.value, - } - assert.Equalf(t, tt.want, s.HasNext(), "HasNext()") - }) - } -} - -func Test_singleRange_Next(t *testing.T) { - type fields struct { - n uint8 - value interface{} - } - tests := []struct { - name string - fields fields - want interface{} - }{ - {"Next", fields{0, "single_range_next"}, "single_range_next"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := &singleRange{ - n: tt.fields.n, - value: tt.fields.value, - } - assert.Equalf(t, tt.want, s.Next(), "Next()") - }) - } -} - -func Test_sliceRange_HasNext(t *testing.T) { - value := make([]interface{}, 0) - value = append(value, "a") - value = append(value, "b") - type fields struct { - cur int - values []interface{} - } - tests := []struct { - name string - fields fields - want bool - }{ - {"HasNext_1", fields{0, value}, true}, - {"HasNext_2", fields{2, value}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := &sliceRange{ - cur: tt.fields.cur, - values: tt.fields.values, - } - assert.Equalf(t, tt.want, s.HasNext(), "HasNext()") - }) - } -} - -func Test_sliceRange_Next(t *testing.T) { - value := make([]interface{}, 0) - value = append(value, "a") - value = append(value, "b") - type fields struct { - cur int - values []interface{} - } - tests := []struct { - name string - fields fields - want interface{} - }{ - {"Next_1", fields{0, value}, "a"}, - {"Next_2", fields{1, value}, "b"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := &sliceRange{ - cur: tt.fields.cur, - values: tt.fields.values, - } - assert.Equalf(t, tt.want, s.Next(), "Next()") - }) - } -} diff --git a/pkg/runtime/rule/route.go b/pkg/runtime/rule/route.go deleted file mode 100644 index 76b59fa3..00000000 --- a/pkg/runtime/rule/route.go +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - "github.com/pkg/errors" -) - -import ( - "github.com/arana-db/arana/pkg/proto/rule" - "github.com/arana-db/arana/pkg/runtime/cmp" -) - -type Matcher interface { - Eval() (rule.Range, error) -} - -type baseExpMatcher rule.VTable - -func (bem *baseExpMatcher) vtab() *rule.VTable { - return (*rule.VTable)(bem) -} - -func (bem *baseExpMatcher) innerEval(c *cmp.Comparative) (rule.Range, error) { - k := c.Key() - - // 非sharding键 - dbMetadata, tbMetadata, ok := bem.vtab().GetShardMetadata(k) - if !ok { - return nil, nil - } - - value, err := c.Value() - if err != nil { - return nil, errors.Wrap(err, "eval failed:") - } - - var md *rule.ShardMetadata - - if tbMetadata != nil { - md = tbMetadata - } else { - md = dbMetadata - } - - switch c.Comparison() { - case cmp.Ceq: - return Single(value), nil - case cmp.Cgt: - after, err := md.Stepper.After(value) - if err != nil { - return nil, errors.WithStack(err) - } - return md.Stepper.Ascend(after, md.Steps) - case cmp.Cgte: - return md.Stepper.Ascend(value, md.Steps) - case cmp.Clt: - before, err := md.Stepper.Before(value) - if err != nil { - return nil, errors.WithStack(err) - } - return md.Stepper.Descend(before, md.Steps) - case cmp.Clte: - return md.Stepper.Descend(value, md.Steps) - case cmp.Cne: - return nil, nil - default: - return nil, errors.Errorf("unsupported comparison %s", c.Comparison()) - } -} - -type cmpExpMatcher struct { - *baseExpMatcher - c *cmp.Comparative -} - -func (c *cmpExpMatcher) Eval() (rule.Range, error) { - if c.c == nil { - return nil, nil - } - return c.innerEval(c.c) -} - -func Route(vt *rule.VTable, c *cmp.Comparative) (Matcher, error) { - mat := &cmpExpMatcher{ - baseExpMatcher: (*baseExpMatcher)(vt), - c: c, - } - return mat, nil -} - -func MatchTables(vt *rule.VTable, column string, it rule.Range) (*rule.Shards, error) { - if it == nil { - return nil, nil - } - var values []interface{} - for it.HasNext() { - values = append(values, it.Next()) - } - - if len(values) < 1 { - return rule.NewShards(), nil - } - - ret := rule.NewShards() - for _, value := range values { - dbIdx, tbIdx, err := vt.Shard(column, value) - if err != nil { - return nil, err - } - ret.Add(dbIdx, tbIdx) - } - - return ret, nil -} diff --git a/pkg/runtime/rule/shard.go b/pkg/runtime/rule/shard.go deleted file mode 100644 index 8b07714c..00000000 --- a/pkg/runtime/rule/shard.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - "crypto/md5" - "fmt" - "hash/crc32" - "strconv" -) - -import ( - gxhash "github.com/dubbogo/gost/hash" - gxmath "github.com/dubbogo/gost/math" - - "github.com/pkg/errors" -) - -import ( - "github.com/arana-db/arana/pkg/proto/rule" -) - -const ( - ModShard ShardType = "modShard" - HashMd5Shard ShardType = "hashMd5Shard" - HashCrc32Shard ShardType = "hashCrc32Shard" - HashBKDRShard ShardType = "hashBKDRShard" - ScriptExpr ShardType = "scriptExpr" -) - -var shardMap = map[ShardType]ShardComputerFunc{ - ModShard: NewModShard, - HashMd5Shard: NewHashMd5Shard, - HashCrc32Shard: NewHashCrc32Shard, - HashBKDRShard: NewHashBKDRShard, -} - -func ShardFactory(shardType ShardType, shardNum int) (shardStrategy rule.ShardComputer, err error) { - if shardNum <= 0 { - return nil, errors.New("shardNum is invalid") - } - if f, ok := shardMap[shardType]; ok { - return f(shardNum), nil - } - return nil, errors.New("do not have this shardType") -} - -type ( - ShardType string - ShardComputerFunc func(shardNum int) rule.ShardComputer -) - -type modShard struct { - shardNum int -} - -func NewModShard(shardNum int) rule.ShardComputer { - return modShard{shardNum} -} - -func (mod modShard) Compute(value interface{}) (int, error) { - n, err := strconv.ParseInt(fmt.Sprintf("%v", value), 10, 64) - if err != nil { - return 0, err - } - return int(gxmath.AbsInt64(n)) % mod.shardNum, nil -} - -// FNV32, FNV32a, FNV64, FNV64a, MD5, SHA1, SHA256, SHA512, murmur3, crc32, crc64, adler -type hashCrc32Shard struct { - shardNum int -} - -func NewHashCrc32Shard(shardNum int) rule.ShardComputer { - return hashCrc32Shard{shardNum} -} - -func (h hashCrc32Shard) Compute(value interface{}) (int, error) { - s := fmt.Sprintf("%v", value) - return int(crc32.ChecksumIEEE([]byte(s))) % h.shardNum, nil -} - -type hashMd5Shard struct { - shardNum int -} - -func NewHashMd5Shard(shardNum int) rule.ShardComputer { - return hashMd5Shard{shardNum} -} - -func (m hashMd5Shard) Compute(value interface{}) (int, error) { - s := fmt.Sprintf("%v", value) - - h := uint64(0) - byteHash1 := md5.Sum([]byte(s)) - - shardNum := m.shardNum - bytesNum := 1 - for ; bytesNum < len(byteHash1); bytesNum++ { - shardNum >>= 8 - y := shardNum & 0xFF - if y == 0 { - break - } - } - - for i := 0; i < bytesNum; i++ { - h <<= 8 - h |= uint64(byteHash1[i]) & 0xFF - } - return int(h % uint64(m.shardNum)), nil -} - -type hashBKDRShard struct { - shardNum int -} - -func NewHashBKDRShard(shardNum int) rule.ShardComputer { - return hashBKDRShard{shardNum} -} - -func (m hashBKDRShard) Compute(value interface{}) (int, error) { - s := fmt.Sprintf("%v", value) - return int(gxmath.AbsInt32(gxhash.BKDRHash(s))) % m.shardNum, nil -} diff --git a/pkg/runtime/rule/shard_script.go b/pkg/runtime/rule/shard_script.go deleted file mode 100644 index e0061259..00000000 --- a/pkg/runtime/rule/shard_script.go +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - "runtime" - "strings" -) - -import ( - "github.com/dop251/goja" - - "github.com/pkg/errors" -) - -import ( - "github.com/arana-db/arana/pkg/proto/rule" -) - -var _ rule.ShardComputer = (*jsShardComputer)(nil) - -const ( - _jsEntrypoint = "__compute__" // shard method name - _jsValueName = "$value" // variable name of column in sharding script -) - -type jsShardComputer struct { - // runtime is not thread-safe, wrap as leaky buffer. - // please see https://go.dev/doc/effective_go#leaky_buffer - freelist chan *goja.Runtime - script string -} - -// NewJavascriptShardComputer returns a shard computer which is based on Javascript. -func NewJavascriptShardComputer(script string) (rule.ShardComputer, error) { - script = wrapScript(script) - - vm, err := createVM(script) - if err != nil { - return nil, errors.Wrapf(err, "failed to create javascript shard computer") - } - - ret := &jsShardComputer{ - freelist: make(chan *goja.Runtime, runtime.NumCPU()*2), - script: script, - } - ret.freelist <- vm - - return ret, nil -} - -func (j *jsShardComputer) Compute(value interface{}) (int, error) { - vm, err := j.getVM() - if err != nil { - return 0, err - } - - defer func() { - j.putVM(vm) - }() - - fn, _ := goja.AssertFunction(vm.Get(_jsEntrypoint)) - res, err := fn(goja.Undefined(), vm.ToValue(value)) - if err != nil { - return 0, errors.WithStack(err) - } - - return int(res.ToInteger()), nil -} - -func (j *jsShardComputer) getVM() (*goja.Runtime, error) { - select { - case next := <-j.freelist: - return next, nil - default: - return createVM(j.script) - } -} - -func (j *jsShardComputer) putVM(vm *goja.Runtime) { - select { - case j.freelist <- vm: - default: - } -} - -func wrapScript(script string) string { - var sb strings.Builder - - sb.Grow(32 + len(_jsEntrypoint) + len(_jsValueName) + len(script)) - - sb.WriteString("function ") - sb.WriteString(_jsEntrypoint) - sb.WriteString("(") - sb.WriteString(_jsValueName) - sb.WriteString(") {\n") - - if strings.Contains(script, "return ") { - sb.WriteString(script) - } else { - sb.WriteString("return ") - sb.WriteString(script) - } - - sb.WriteString("\n}") - - return sb.String() -} - -func createVM(script string) (*goja.Runtime, error) { - vm := goja.New() - if _, err := vm.RunString(script); err != nil { - return nil, errors.WithStack(err) - } - - // TODO: add prelude functions, includes some hash/utils - - return vm, nil -} diff --git a/pkg/runtime/rule/shard_test.go b/pkg/runtime/rule/shard_test.go deleted file mode 100644 index e07e70e1..00000000 --- a/pkg/runtime/rule/shard_test.go +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package rule - -import ( - "reflect" - "testing" -) - -import ( - "github.com/stretchr/testify/assert" -) - -func TestShardFactory(t *testing.T) { - shardTable := []struct { - in ShardType - wantName string - wantErr error - }{ - {ModShard, string(ModShard), nil}, - {HashMd5Shard, string(HashMd5Shard), nil}, - {HashCrc32Shard, string(HashCrc32Shard), nil}, - {HashBKDRShard, string(HashBKDRShard), nil}, - } - - shard, err := ShardFactory("xxx", 0) - assert.Equal(t, nil, shard) - assert.Equal(t, "shardNum is invalid", err.Error()) - - shard, err = ShardFactory("xxx", 1) - assert.Equal(t, nil, shard) - assert.Equal(t, "do not have this shardType", err.Error()) - - for i := 0; i < len(shardTable); i++ { - shard, err := ShardFactory(shardTable[i].in, 1) - assert.Equal(t, shardTable[i].wantName, reflect.TypeOf(shard).Name()) - assert.Equal(t, shardTable[i].wantErr, err) - } -} - -func TestModShard(t *testing.T) { - shardTable := []struct { - Mod int - in, want int - }{ - {7, -1, 1}, - {7, 3, 3}, - {7, 13, 6}, - {7, 14, 0}, - } - for i := 0; i < len(shardTable); i++ { - shard, err := ShardFactory(ModShard, shardTable[i].Mod) - if err != nil { - t.Error(err) - } - out, err := shard.Compute(shardTable[i].in) - if err != nil { - t.Error(err) - } - assert.Equal(t, shardTable[i].want, out) - } -} - -func TestMd5Shard(t *testing.T) { - shardTable := []struct { - Mod int - in string - want int - }{ - {7, "1", 0}, - {7, "abc", 4}, - {7, "DZ20201212", 1}, - } - for i := 0; i < len(shardTable); i++ { - shard, err := ShardFactory(HashMd5Shard, shardTable[i].Mod) - if err != nil { - t.Fatal(err) - } - out, err := shard.Compute(shardTable[i].in) - if err != nil { - t.Error(err) - } - assert.Equal(t, shardTable[i].want, out) - } -} - -func TestCrc32Shard(t *testing.T) { - shardTable := []struct { - Mod int - in string - want int - }{ - {7, "1", 2}, - {7, "abc", 5}, - {7, "DZ20201212", 3}, - } - for i := 0; i < len(shardTable); i++ { - shard, err := ShardFactory(HashCrc32Shard, shardTable[i].Mod) - if err != nil { - t.Fatal(err) - } - out, err := shard.Compute(shardTable[i].in) - if err != nil { - t.Error(err) - } - assert.Equal(t, shardTable[i].want, out) - } -} - -func TestBKDRShard(t *testing.T) { - shardTable := []struct { - Mod int - in string - want int - }{ - {7, "1", 0}, - {7, "abc", 6}, - {7, "DZ20201212", 5}, - } - for i := 0; i < len(shardTable); i++ { - shard, err := ShardFactory(HashBKDRShard, shardTable[i].Mod) - if err != nil { - t.Fatal(err) - } - out, err := shard.Compute(shardTable[i].in) - if err != nil { - t.Error(err) - } - assert.Equal(t, shardTable[i].want, out) - } -} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 457a6cce..4af3cb40 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -44,6 +44,7 @@ import ( errors2 "github.com/arana-db/arana/pkg/mysql/errors" "github.com/arana-db/arana/pkg/proto" "github.com/arana-db/arana/pkg/proto/hint" + _ "github.com/arana-db/arana/pkg/runtime/builtin" rcontext "github.com/arana-db/arana/pkg/runtime/context" _ "github.com/arana-db/arana/pkg/runtime/function" "github.com/arana-db/arana/pkg/runtime/namespace" @@ -93,9 +94,7 @@ func Unload(tenant, schema string) error { return nil } -var ( - _ proto.DB = (*AtomDB)(nil) -) +var _ proto.DB = (*AtomDB)(nil) type AtomDB struct { mu sync.Mutex diff --git a/pkg/runtime/transaction/fault_decision.go b/pkg/runtime/transaction/fault_decision.go index a3455577..8d5eafd9 100644 --- a/pkg/runtime/transaction/fault_decision.go +++ b/pkg/runtime/transaction/fault_decision.go @@ -29,7 +29,6 @@ type TxFaultDecisionExecutor struct { // Run Core logic of the decision -making decision -making at the bottom of the affairs func (bm *TxFaultDecisionExecutor) Run() { - } func (bm *TxFaultDecisionExecutor) scanUnFinishTxLog() ([]TrxLog, error) { @@ -37,13 +36,10 @@ func (bm *TxFaultDecisionExecutor) scanUnFinishTxLog() ([]TrxLog, error) { } func (bm *TxFaultDecisionExecutor) handlePreparing(tx TrxLog) { - } func (bm *TxFaultDecisionExecutor) handleCommitting(tx TrxLog) { - } func (bm *TxFaultDecisionExecutor) handleAborting(tx TrxLog) { - } diff --git a/pkg/runtime/transaction/xa.go b/pkg/runtime/transaction/xa.go index 2a53fef3..aaef7c5d 100644 --- a/pkg/runtime/transaction/xa.go +++ b/pkg/runtime/transaction/xa.go @@ -29,9 +29,7 @@ import ( rcontext "github.com/arana-db/arana/pkg/runtime/context" ) -var ( - ErrorInvalidTxId = errors.New("invalid transaction id") -) +var ErrorInvalidTxId = errors.New("invalid transaction id") // StartXA do start xa transaction action func StartXA(ctx context.Context, bc *mysql.BackendConnection) (proto.Result, error) { diff --git a/pkg/security/tenant.go b/pkg/security/tenant.go index 04e61d52..7ecdeb87 100644 --- a/pkg/security/tenant.go +++ b/pkg/security/tenant.go @@ -88,7 +88,7 @@ func (st *simpleTenantManager) GetUser(tenant string, username string) (*config. func (st *simpleTenantManager) GetUsers(tenant string) ([]*config.User, bool) { st.RLock() defer st.RUnlock() - var users = []*config.User{} + users := []*config.User{} exist, ok := st.tenants[tenant] if !ok { return nil, false diff --git a/pkg/sequence/sequence.go b/pkg/sequence/sequence.go index 7050b7f0..fff7f603 100644 --- a/pkg/sequence/sequence.go +++ b/pkg/sequence/sequence.go @@ -39,7 +39,7 @@ func newSequenceManager() proto.SequenceManager { } } -// SequenceManager Uniform management of seqneuce manager +// SequenceManager uniform management of sequence manager type sequenceManager struct { lock sync.RWMutex tenants map[string]*tenantBucket diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index f448a849..82d698bc 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -34,7 +34,6 @@ var ( func IsEnableLocalMathCompu(enable bool) bool { _enableLocalMathCompuSync.Do(func() { _enableLocalMathCompu = enable - }) return _enableLocalMathCompu } diff --git a/pkg/util/math/compare.go b/pkg/util/math/compare.go new file mode 100644 index 00000000..c8da5e42 --- /dev/null +++ b/pkg/util/math/compare.go @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package math + +import ( + "golang.org/x/exp/constraints" +) + +type Number interface { + constraints.Integer | constraints.Float +} + +func Max[T Number](a, b T) T { + if a > b { + return a + } + return b +} + +func Min[T Number](a, b T) T { + if a < b { + return a + } + return b +} diff --git a/pkg/util/misc/misc.go b/pkg/util/misc/misc.go index 8a20dfab..11fa8104 100644 --- a/pkg/util/misc/misc.go +++ b/pkg/util/misc/misc.go @@ -56,3 +56,39 @@ func TryClose(i interface{}) error { } return nil } + +func ReverseSlice[T any](input []T) { + if len(input) < 2 { + return + } + for i, j := 0, len(input)-1; i < j; { + input[i], input[j] = input[j], input[i] + i++ + j-- + } +} + +// CartesianProduct compute cartesian product. +func CartesianProduct[T any](inputs [][]T) [][]T { + var ( + res [][]T + rec []T + ) + cartesianProductHelper[T](inputs, &res, &rec, 0) + return res +} + +func cartesianProductHelper[T any](input [][]T, res *[][]T, rec *[]T, index int) { + if index < len(input) { + for _, v := range input[index] { + *rec = append(*rec, v) + cartesianProductHelper(input, res, rec, index+1) + *rec = (*rec)[:index] + } + return + } + tmp := make([]T, len(input)) + copy(tmp, *rec) + *res = append(*res, tmp) + *rec = (*rec)[:index] +} diff --git a/scripts/sharding.sql b/scripts/sharding.sql index d36edda0..e538f6b3 100644 --- a/scripts/sharding.sql +++ b/scripts/sharding.sql @@ -23,454 +23,6 @@ CREATE DATABASE IF NOT EXISTS employees_0003 CHARACTER SET utf8mb4 COLLATE utf8m CREATE DATABASE IF NOT EXISTS employees_0000_r CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0000` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0001` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0002` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0003` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0004` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0005` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0006` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0007` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0008` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0009` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0010` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0011` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0012` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0013` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0014` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0015` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0016` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0017` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0018` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0019` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0020` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0021` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0022` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0023` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0024` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0025` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0026` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0027` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0028` ( `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, `uid` BIGINT(20) UNSIGNED NOT NULL, @@ -486,182 +38,94 @@ CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0028` KEY `nickname` (`nickname`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0029` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0001` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0002` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0003` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0004` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0005` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0006` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`student_0007` LIKE `employees_0000`.`student_0000`; + +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0008` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0009` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0010` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0011` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0012` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0013` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0014` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`student_0015` LIKE `employees_0000`.`student_0000`; + +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0016` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0017` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0018` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0019` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0020` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0021` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0022` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`student_0023` LIKE `employees_0000`.`student_0000`; + +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0024` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0025` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0026` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0027` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0028` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0029` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0030` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0031` LIKE `employees_0000`.`student_0000`; + +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0000` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0001` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0002` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0003` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0004` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0005` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0006` LIKE `employees_0000`.`student_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0007` LIKE `employees_0000`.`student_0000`; -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0030` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +INSERT INTO employees_0000.student_0001(id,uid,name,score,nickname,gender,birth_year,created_at,modified_at) VALUES (1, 1, 'arana', 95, 'Awesome Arana', 0, 2021, NOW(), NOW()); -CREATE TABLE IF NOT EXISTS `employees_0003`.`student_0031` +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0000` ( `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', + `friend_id` BIGINT(20) UNSIGNED NOT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) + UNIQUE KEY `uk_uid_friend_id` (`uid`,`friend_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0001` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0002` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0003` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0004` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0005` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0006` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0000`.`friendship_0007` LIKE `employees_0000`.`friendship_0000`; -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0000` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0001` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0002` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0003` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0004` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0005` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0006` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE IF NOT EXISTS `employees_0000_r`.`student_0007` -( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, - `uid` BIGINT(20) UNSIGNED NOT NULL, - `name` VARCHAR(255) NOT NULL, - `score` DECIMAL(6,2) DEFAULT '0', - `nickname` VARCHAR(255) DEFAULT NULL, - `gender` TINYINT(4) NULL, - `birth_year` SMALLINT(5) UNSIGNED DEFAULT '0', - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `modified_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_uid` (`uid`), - KEY `nickname` (`nickname`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0008` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0009` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0010` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0011` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0012` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0013` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0014` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0001`.`friendship_0015` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0016` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0017` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0018` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0019` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0020` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0021` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0022` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0002`.`friendship_0023` LIKE `employees_0000`.`friendship_0000`; -INSERT INTO employees_0000.student_0001(id,uid,name,score,nickname,gender,birth_year,created_at,modified_at) VALUES (1, 1, 'arana', 95, 'Awesome Arana', 0, 2021, NOW(), NOW()); +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0024` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0025` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0026` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0027` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0028` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0029` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0030` LIKE `employees_0000`.`friendship_0000`; +CREATE TABLE IF NOT EXISTS `employees_0003`.`friendship_0031` LIKE `employees_0000`.`friendship_0000`; diff --git a/test/integration_test.go b/test/integration_test.go index e102ccd6..1364a9c1 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -643,8 +643,8 @@ func (s *IntegrationSuite) TestShowShardingTable() { t = s.T() ) - _, err := db.Query("show sharding table from employees") - assert.NoErrorf(t, err, "show sharding table from employees error: %v", err) + _, err := db.Query("show sharding table from student") + assert.NoError(t, err, "should execute 'show sharding table ...' correctly") } func (s *IntegrationSuite) TestShowDatabaseRules() { @@ -836,10 +836,11 @@ func (s *IntegrationSuite) TestMultipleHints() { } for _, it := range []tt{ - {"/*A! master */ /*A! fullscan */ SELECT * FROM student WHERE score > 100", nil, 0}, {"/*A! slave */ /*A! master */ /*A! fullscan */ SELECT id,name FROM student WHERE score > 100", nil, 0}, {"/*A! master */ /*A! direct */ SELECT * FROM student_0000 WHERE uid = ?", []interface{}{1}, 0}, - {"/*A! fullscan */ /*A! direct */ SELECT * FROM student WHERE uid in (?)", []interface{}{1}, 0}, + // TODO: fix hint of fullscan + //{"/*A! master */ /*A! fullscan */ SELECT * FROM student WHERE score > 100", nil, 0}, + //{"/*A! fullscan */ /*A! direct */ SELECT * FROM student WHERE uid in (?)", []interface{}{1}, 0}, } { t.Run(it.sql, func(t *testing.T) { // select from logical table @@ -1270,6 +1271,7 @@ func (s *IntegrationSuite) TestKill() { _, err = db.Query(fmt.Sprintf("KILL %s", data[row-1][0])) assert.NoError(t, err) } + func (s *IntegrationSuite) TestOptimizeLocalCompute() { var ( db = s.DB() diff --git a/testdata/fake_config.yaml b/testdata/fake_config.yaml index 59ddbe3c..85383ce2 100644 --- a/testdata/fake_config.yaml +++ b/testdata/fake_config.yaml @@ -64,14 +64,14 @@ data: type: snowflake option: db_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid expr: parseInt($value % 32 / 8) tbl_rules: - - column: uid - type: scriptExpr + - columns: + - name: uid + - step: 32 expr: $value % 32 - step: 32 topology: db_pattern: employees_${0000..0003} tbl_pattern: student_${0000..0031} diff --git a/testdata/mock_rule.go b/testdata/mock_rule.go index 36974297..2853a3c6 100644 --- a/testdata/mock_rule.go +++ b/testdata/mock_rule.go @@ -1,20 +1,3 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - // Code generated by MockGen. DO NOT EDIT. // Source: github.com/arana-db/arana/pkg/proto/rule (interfaces: ShardComputer) @@ -29,6 +12,10 @@ import ( gomock "github.com/golang/mock/gomock" ) +import ( + proto "github.com/arana-db/arana/pkg/proto" +) + // MockShardComputer is a mock of ShardComputer interface. type MockShardComputer struct { ctrl *gomock.Controller @@ -53,67 +40,34 @@ func (m *MockShardComputer) EXPECT() *MockShardComputerMockRecorder { } // Compute mocks base method. -func (m *MockShardComputer) Compute(arg0 interface{}) (int, error) { +func (m *MockShardComputer) Compute(arg0 ...proto.Value) (int, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Compute", arg0) + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Compute", varargs...) ret0, _ := ret[0].(int) ret1, _ := ret[1].(error) return ret0, ret1 } // Compute indicates an expected call of Compute. -func (mr *MockShardComputerMockRecorder) Compute(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compute", reflect.TypeOf((*MockShardComputer)(nil).Compute), arg0) -} - -// MockRange is a mock of Range interface. -type MockRange struct { - ctrl *gomock.Controller - recorder *MockRangeMockRecorder -} - -// MockRangeMockRecorder is the mock recorder for MockRange. -type MockRangeMockRecorder struct { - mock *MockRange -} - -// NewMockRange creates a new mock instance. -func NewMockRange(ctrl *gomock.Controller) *MockRange { - mock := &MockRange{ctrl: ctrl} - mock.recorder = &MockRangeMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRange) EXPECT() *MockRangeMockRecorder { - return m.recorder -} - -// HasNext mocks base method. -func (m *MockRange) HasNext() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HasNext") - ret0, _ := ret[0].(bool) - return ret0 -} - -// HasNext indicates an expected call of HasNext. -func (mr *MockRangeMockRecorder) HasNext() *gomock.Call { +func (mr *MockShardComputerMockRecorder) Compute(arg0 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasNext", reflect.TypeOf((*MockRange)(nil).HasNext)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compute", reflect.TypeOf((*MockShardComputer)(nil).Compute), arg0...) } -// Next mocks base method. -func (m *MockRange) Next() interface{} { +// Variables mocks base method. +func (m *MockShardComputer) Variables() []string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Next") - ret0, _ := ret[0].(interface{}) + ret := m.ctrl.Call(m, "Variables") + ret0, _ := ret[0].([]string) return ret0 } -// Next indicates an expected call of Next. -func (mr *MockRangeMockRecorder) Next() *gomock.Call { +// Variables indicates an expected call of Variables. +func (mr *MockShardComputerMockRecorder) Variables() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Next", reflect.TypeOf((*MockRange)(nil).Next)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Variables", reflect.TypeOf((*MockShardComputer)(nil).Variables)) }