From 5ef4f4d71d394641fa59abe991712da62c3d76b7 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 23 Jan 2024 22:32:53 +0000 Subject: [PATCH 1/3] Bump lyx --- go.mod | 2 +- go.sum | 4 ++ router/frontend_test.go | 2 +- router/qrouter/proxy_routing.go | 83 ++++++++++++++++++++++++++------- 4 files changed, 72 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 3e2812cd0..c46af540c 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/lib/pq v1.10.9 github.com/libp2p/go-reuseport v0.4.0 github.com/opentracing/opentracing-go v1.2.0 - github.com/pg-sharding/lyx v0.0.0-20240111083355-38b940962fe3 + github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 github.com/sevlyar/go-daemon v0.1.6 diff --git a/go.sum b/go.sum index de2acbf30..db83b6325 100644 --- a/go.sum +++ b/go.sum @@ -125,6 +125,10 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pg-sharding/lyx v0.0.0-20240111083355-38b940962fe3 h1:drqKxg58y9R3niC8T4wW6dqziPZnBosuAGC8X/eXLPc= github.com/pg-sharding/lyx v0.0.0-20240111083355-38b940962fe3/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= +github.com/pg-sharding/lyx v0.0.0-20240119072404-3d559dba6aa8 h1:s8f0Vo257gQvwS++K9RMkxKrgxsdVAXTtc5wJkqHvmg= +github.com/pg-sharding/lyx v0.0.0-20240119072404-3d559dba6aa8/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= +github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2 h1:Pja7QaJpBcflHtTUbIEFl/ZGJrk2vOmX6arh1IJwoIU= +github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/router/frontend_test.go b/router/frontend_test.go index bf1605892..5beb170d3 100644 --- a/router/frontend_test.go +++ b/router/frontend_test.go @@ -101,7 +101,7 @@ func TestFrontendSimple(t *testing.T) { qr.EXPECT().Route(gomock.Any(), &lyx.Select{ TargetList: []lyx.Node{ - &lyx.AExprConst{Value: "1"}, + &lyx.AExprIConst{Value: 1}, }, Where: &lyx.AExprEmpty{}, }, gomock.Any()).Return(routingstate.ShardMatchState{ diff --git a/router/qrouter/proxy_routing.go b/router/qrouter/proxy_routing.go index 87dd3708f..96f127323 100644 --- a/router/qrouter/proxy_routing.go +++ b/router/qrouter/proxy_routing.go @@ -100,13 +100,13 @@ func NewRoutingMetadataContext( } // TODO : unit tests -func (meta *RoutingMetadataContext) RecordConstExpr(resolvedRelation RelationFQN, colname string, expr *lyx.AExprConst) { +func (meta *RoutingMetadataContext) RecordConstExpr(resolvedRelation RelationFQN, colname string, expr string) { meta.rels[resolvedRelation] = append(meta.rels[resolvedRelation], colname) if _, ok := meta.exprs[resolvedRelation]; !ok { meta.exprs[resolvedRelation] = map[string]string{} } delete(meta.unparsed_columns, colname) - meta.exprs[resolvedRelation][colname] = expr.Value + meta.exprs[resolvedRelation][colname] = expr } // TODO : unit tests @@ -192,7 +192,7 @@ func (qr *ProxyQrouter) RouteKeyWithRanges(ctx context.Context, expr lyx.Node, m spqrlog.Zero.Debug().Str("key", string(meta.params[e.Number-1])).Str("hashed key", string(hashedKey)).Msg("applying hash function on key") return qr.DeparseKeyWithRangesInternal(ctx, string(hashedKey), meta) - case *lyx.AExprConst: + case *lyx.AExprSConst: hashedKey, err := hashfunction.ApplyHashFunction([]byte(e.Value), hf) if err != nil { return nil, err @@ -200,6 +200,15 @@ func (qr *ProxyQrouter) RouteKeyWithRanges(ctx context.Context, expr lyx.Node, m spqrlog.Zero.Debug().Str("key", e.Value).Str("hashed key", string(hashedKey)).Msg("applying hash function on key") return qr.DeparseKeyWithRangesInternal(ctx, string(hashedKey), meta) + case *lyx.AExprIConst: + val := fmt.Sprintf("%d", e.Value) + hashedKey, err := hashfunction.ApplyHashFunction([]byte(val), hf) + if err != nil { + return nil, err + } + + spqrlog.Zero.Debug().Int("key", e.Value).Str("hashed key", string(hashedKey)).Msg("applying hash function on key") + return qr.DeparseKeyWithRangesInternal(ctx, string(hashedKey), meta) default: return nil, ComplexQuery } @@ -225,7 +234,24 @@ func (qr *ProxyQrouter) routeByClause(ctx context.Context, expr lyx.Node, meta * switch rght := texpr.Right.(type) { case *lyx.ParamRef: // ignore - case *lyx.AExprConst: + case *lyx.AExprSConst: + alias, colname := lft.TableAlias, lft.ColName + + if !meta.CheckColumnRls(colname) { + spqrlog.Zero.Debug(). + Str("colname", colname). + Msg("skip column due no rule mathing") + continue + } + + resolvedRelation, err := meta.ResolveRelationByAlias(alias) + if err == nil { + // TBD: postpone routing from here to root of parsing tree + meta.RecordConstExpr(resolvedRelation, colname, rght.Value) + } else { + meta.unparsed_columns[colname] = struct{}{} + } + case *lyx.AExprIConst: alias, colname := lft.TableAlias, lft.ColName if !meta.CheckColumnRls(colname) { @@ -238,7 +264,8 @@ func (qr *ProxyQrouter) routeByClause(ctx context.Context, expr lyx.Node, meta * resolvedRelation, err := meta.ResolveRelationByAlias(alias) if err == nil { // TBD: postpone routing from here to root of parsing tree - meta.RecordConstExpr(resolvedRelation, colname, rght) + // maybe expimely inefficient. Will be fixed in SPQR-2.0 + meta.RecordConstExpr(resolvedRelation, colname, fmt.Sprintf("%d", rght.Value)) } else { meta.unparsed_columns[colname] = struct{}{} } @@ -247,7 +274,7 @@ func (qr *ProxyQrouter) routeByClause(ctx context.Context, expr lyx.Node, meta * if len(rght.List) != 0 { expr := rght.List[0] switch bexpr := expr.(type) { - case *lyx.AExprConst: + case *lyx.AExprSConst: alias, colname := lft.TableAlias, lft.ColName if !meta.CheckColumnRls(colname) { @@ -260,7 +287,25 @@ func (qr *ProxyQrouter) routeByClause(ctx context.Context, expr lyx.Node, meta * resolvedRelation, err := meta.ResolveRelationByAlias(alias) if err == nil { // TBD: postpone routing from here to root of parsing tree - meta.RecordConstExpr(resolvedRelation, colname, bexpr) + meta.RecordConstExpr(resolvedRelation, colname, bexpr.Value) + } else { + meta.unparsed_columns[colname] = struct{}{} + } + case *lyx.AExprIConst: + alias, colname := lft.TableAlias, lft.ColName + + if !meta.CheckColumnRls(colname) { + spqrlog.Zero.Debug(). + Str("colname", colname). + Msg("skip column due no rule mathing") + continue + } + + resolvedRelation, err := meta.ResolveRelationByAlias(alias) + if err == nil { + // TBD: postpone routing from here to root of parsing tree + // maybe expimely inefficient. Will be fixed in SPQR-2.0 + meta.RecordConstExpr(resolvedRelation, colname, fmt.Sprintf("%d", bexpr.Value)) } else { meta.unparsed_columns[colname] = struct{}{} } @@ -280,7 +325,7 @@ func (qr *ProxyQrouter) routeByClause(ctx context.Context, expr lyx.Node, meta * } case *lyx.ColumnRef: /* colref = colref case, skip */ - case *lyx.AExprConst: + case *lyx.AExprIConst, *lyx.AExprSConst, *lyx.AExprBConst, *lyx.AExprNConst: /* should not happend */ case *lyx.AExprEmpty: /*skip*/ @@ -541,8 +586,6 @@ func (qr *ProxyQrouter) deparseShardingMapping( Strs("statements", cols). Msg("deparsed insert statement columns") - meta.ValuesLists = stmt.Values - meta.InsertStmtCols = cols switch q := stmt.TableRef.(type) { case *lyx.RangeVar: @@ -552,12 +595,17 @@ func (qr *ProxyQrouter) deparseShardingMapping( } if selectStmt := stmt.SubSelect; selectStmt != nil { - spqrlog.Zero.Debug().Msg("routing insert stmt on select clause") - _ = qr.DeparseSelectStmt(ctx, selectStmt, meta) - /* try target list */ - spqrlog.Zero.Debug().Msg("routing insert stmt on target list") - /* this target list for some insert (...) sharding column */ - meta.TargetList = selectStmt.(*lyx.Select).TargetList + switch subS := selectStmt.(type) { + case *lyx.Select: + spqrlog.Zero.Debug().Msg("routing insert stmt on select clause") + _ = qr.DeparseSelectStmt(ctx, subS, meta) + /* try target list */ + spqrlog.Zero.Debug().Msg("routing insert stmt on target list") + /* this target list for some insert (...) sharding column */ + meta.TargetList = selectStmt.(*lyx.Select).TargetList + case *lyx.ValueClause: + meta.ValuesLists = subS.Values + } } return nil @@ -778,9 +826,10 @@ func (qr *ProxyQrouter) routeWithRules(ctx context.Context, stmt lyx.Node, sph s if e.Name == "current_schema" { any_routable = true } - case *lyx.AExprConst: + case *lyx.AExprIConst, *lyx.AExprSConst, *lyx.AExprNConst, *lyx.AExprBConst: // ok any_routable = true + case *lyx.ColumnRef: /* Step 1.4.8.2 - SELECT current_schema special case */ if e.ColName == "current_schema" { From 681648060eebe236ddce0c94217dd6c47c08fee0 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 23 Jan 2024 23:03:17 +0000 Subject: [PATCH 2/3] proper bump --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c46af540c..d3a454764 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/lib/pq v1.10.9 github.com/libp2p/go-reuseport v0.4.0 github.com/opentracing/opentracing-go v1.2.0 - github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2 + github.com/pg-sharding/lyx v0.0.0-20240123223048-f4c8715e77ed github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 github.com/sevlyar/go-daemon v0.1.6 diff --git a/go.sum b/go.sum index db83b6325..ca9ce73d7 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,8 @@ github.com/pg-sharding/lyx v0.0.0-20240119072404-3d559dba6aa8 h1:s8f0Vo257gQvwS+ github.com/pg-sharding/lyx v0.0.0-20240119072404-3d559dba6aa8/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2 h1:Pja7QaJpBcflHtTUbIEFl/ZGJrk2vOmX6arh1IJwoIU= github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= +github.com/pg-sharding/lyx v0.0.0-20240123223048-f4c8715e77ed h1:NNv0qFDlAFc9/crthqjFqUmYRW5SjGnvkxSiLx6hOwQ= +github.com/pg-sharding/lyx v0.0.0-20240123223048-f4c8715e77ed/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 9dc033f52e136248ee203d213e883cc7acf15b87 Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 24 Jan 2024 08:58:25 +0000 Subject: [PATCH 3/3] Bump correct lyx version --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d3a454764..e1b68d721 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/lib/pq v1.10.9 github.com/libp2p/go-reuseport v0.4.0 github.com/opentracing/opentracing-go v1.2.0 - github.com/pg-sharding/lyx v0.0.0-20240123223048-f4c8715e77ed + github.com/pg-sharding/lyx v0.0.0-20240124084632-a9f5aa89a6ac github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 github.com/sevlyar/go-daemon v0.1.6 diff --git a/go.sum b/go.sum index ca9ce73d7..17d155991 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2 h1:Pja7QaJpBcflHtT github.com/pg-sharding/lyx v0.0.0-20240123215906-b049786c6ee2/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= github.com/pg-sharding/lyx v0.0.0-20240123223048-f4c8715e77ed h1:NNv0qFDlAFc9/crthqjFqUmYRW5SjGnvkxSiLx6hOwQ= github.com/pg-sharding/lyx v0.0.0-20240123223048-f4c8715e77ed/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= +github.com/pg-sharding/lyx v0.0.0-20240124084632-a9f5aa89a6ac h1:iJcay9e0ibZYH1Bx/2tLHZsHYj5k5I480VOswOpfeGo= +github.com/pg-sharding/lyx v0.0.0-20240124084632-a9f5aa89a6ac/go.mod h1:2dPBQAhqv/30mhzj2yBXQkXhsGJQ8GhM+oWOfbGua58= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=