union-and-ci-uplifted (#428)

Summary:

- Support for `union` with exotic dataflow.
- Added robot test `Union of List And Details Dataflow View Works As Exemplified By AWS KMS Key Cloud Control`.
- Update parser.
- Add docker cleanup and debug information to docker build run.  This was required because robot is no longer cleaning up docker containers adequately.
- Adopt robot framework deprecation suggestion for `[Return]` -> `RETURN`.

.github/workflows/go.yml

@@ -909,6 +909,17 @@ jobs:
       if: success()
       timeout-minutes: 20
       run: |
+        echo "## Stray docker containers before postgres robot tests ##"
+        docker container ls --format "table {{.ID}}\t{{.Names}}\t{{.Ports}}" -a
+        echo "## End ##"
+        docker compose down
+        # shellcheck disable=SC2046
+        docker stop $(docker ps -a -q)
+        # shellcheck disable=SC2046
+        docker rm $(docker ps -a -q)
+        echo "## Stray docker containers after clean up commands ##"
+        docker container ls --format "table {{.ID}}\t{{.Names}}\t{{.Ports}}" -a
+        echo "## End ##"
         python cicd/python/build.py --robot-test --config='{ "variables":  { "EXECUTION_PLATFORM": "docker", "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": true, "SQL_BACKEND": "postgres_tcp" } }'
 
     - name: Output from mocked functional tests

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/stackql/go-sqlite3 v0.0.3-stackqlalpha02
 	github.com/stackql/go-suffix-map v0.0.1-alpha01
 	github.com/stackql/psql-wire v0.1.1-alpha07
-	github.com/stackql/stackql-parser v0.0.13-beta29
+	github.com/stackql/stackql-parser v0.0.13-beta30
 	github.com/stretchr/testify v1.8.4
 	github.com/xo/dburl v0.12.4
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8

go.sum

@@ -484,8 +484,8 @@ github.com/stackql/psql-wire v0.1.1-alpha07 h1:LQWVUlx4Bougk6dztDNG5tmXxpIVeeTSs
 github.com/stackql/psql-wire v0.1.1-alpha07/go.mod h1:a44Wd8kDC3irFLpGutarKDBqhJ/aqXlj1aMzO5bVJYg=
 github.com/stackql/readline v0.0.2-alpha05 h1:ID4QzGdplFBsrSnTuz8pvKzWw96JbrJg8fsLry2UriU=
 github.com/stackql/readline v0.0.2-alpha05/go.mod h1:OFAYOdXk/X4+5GYiDXFfaGrk+bCN6Qv0SYY5HNzD2E0=
-github.com/stackql/stackql-parser v0.0.13-beta29 h1:NBcAuGo+vNqRN+4GgEZMerfTV4wLyLnhlKCrSf5uoAo=
-github.com/stackql/stackql-parser v0.0.13-beta29/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ=
+github.com/stackql/stackql-parser v0.0.13-beta30 h1:+M5PJxC2lCO9pR8gxxQfWcpq2geAQN3R3l3K0ypWcYI=
+github.com/stackql/stackql-parser v0.0.13-beta30/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ=
 github.com/stackql/stackql-provider-registry v0.0.1-rc06 h1:MgroWOr0bSqjSTDGnXB0UoZGFXpW3SRtN0EFkzB8Rpo=
 github.com/stackql/stackql-provider-registry v0.0.1-rc06/go.mod h1:87rVxnS2aRASK20lBQgoYA0o7FSJTZBGGRaWFR7IDm4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

internal/stackql/astanalysis/earlyanalysis/first_passes.go

@@ -46,6 +46,56 @@ type InitialPassesScreener interface {
 	IsReadOnly() bool
 }
 
+type threeToFivePassAggregate interface {
+	GetAliasMap() parserutil.TableAliasMap
+	GetTables() sqlparser.TableExprs
+	GetColRefs() parserutil.ColTableMap
+	GetAliasedColumns() parserutil.TableExprMap
+	GetParameters() parserutil.ParameterMap
+}
+
+var (
+	_ threeToFivePassAggregate = &threeToFivePassAggregateImpl{}
+)
+
+func newThreeToFivePassAggregate(
+	firstPassVisitor astvisit.ParserTableExtractAstVisitor,
+	secondPassVisitor astvisit.ParserTableAliasPairingAstVisitor,
+	thirdPassVisitor astvisit.ParserPlaceholderParamAstVisitor,
+) threeToFivePassAggregate {
+	return &threeToFivePassAggregateImpl{
+		firstPassVisitor:  firstPassVisitor,
+		secondPassVisitor: secondPassVisitor,
+		thirdPassVisitor:  thirdPassVisitor,
+	}
+}
+
+type threeToFivePassAggregateImpl struct {
+	firstPassVisitor  astvisit.ParserTableExtractAstVisitor
+	secondPassVisitor astvisit.ParserTableAliasPairingAstVisitor
+	thirdPassVisitor  astvisit.ParserPlaceholderParamAstVisitor
+}
+
+func (tfa *threeToFivePassAggregateImpl) GetAliasMap() parserutil.TableAliasMap {
+	return tfa.firstPassVisitor.GetAliasMap()
+}
+
+func (tfa *threeToFivePassAggregateImpl) GetTables() sqlparser.TableExprs {
+	return tfa.firstPassVisitor.GetTables()
+}
+
+func (tfa *threeToFivePassAggregateImpl) GetColRefs() parserutil.ColTableMap {
+	return tfa.secondPassVisitor.GetColRefs()
+}
+
+func (tfa *threeToFivePassAggregateImpl) GetAliasedColumns() parserutil.TableExprMap {
+	return tfa.secondPassVisitor.GetAliasedColumns()
+}
+
+func (tfa *threeToFivePassAggregateImpl) GetParameters() parserutil.ParameterMap {
+	return tfa.thirdPassVisitor.GetParameters()
+}
+
 type InitialPassesScreenerAnalyzer interface {
 	Analyzer
 	InitialPassesScreener
@@ -128,6 +178,36 @@ func (sp *standardInitialPasses) Analyze(
 	return sp.initialPasses(statement, handlerCtx, tcc)
 }
 
+//nolint:unparam // future proofing
+func thirdToFifthPasses(
+	ast sqlparser.SQLNode, annotatedAST annotatedast.AnnotatedAst) (threeToFivePassAggregate, error) {
+	// Third pass AST analysis; extract parser table objects.
+	// Extracts:
+	//   - parser objects representing tables.
+	//   - mapping of string aliases to tables.
+	tVis := astvisit.NewTableExtractAstVisitor(annotatedAST)
+	tVis.Visit(ast) //nolint:errcheck // TODO: fix this
+
+	// Fourth pass AST analysis.
+	// Accepts slice of parser table objects
+	// extracted from previous analysis.
+	// Extracts:
+	//   - Col Refs; mapping columnar objects to tables.
+	//   - Alias Map; mapping the "TableName" objects
+	//     defining aliases to table objects.
+	aVis := astvisit.NewTableAliasAstVisitor(annotatedAST, tVis.GetTables())
+	aVis.Visit(ast) //nolint:errcheck // TODO: fix this
+
+	// Fifth pass AST analysis.
+	// Extracts:
+	//   - Columnar parameters with null values.
+	//     Useful for method matching.
+	//     Especially for "Insert" queries.
+	tpv := astvisit.NewPlaceholderParamAstVisitor(annotatedAST, "", false)
+	tpv.Visit(ast) //nolint:errcheck // TODO: fix this
+	return newThreeToFivePassAggregate(tVis, aVis, tpv), nil
+}
+
 //nolint:funlen,gocognit,gocyclo,cyclop // this is a large function abstracting plenty
 func (sp *standardInitialPasses) initialPasses(
 	statement sqlparser.Statement,
@@ -236,40 +316,21 @@ func (sp *standardInitialPasses) initialPasses(
 		}
 	}
 
-	// Third pass AST analysis; extract parser table objects.
-	// Extracts:
-	//   - parser objects representing tables.
-	//   - mapping of string aliases to tables.
-	tVis := astvisit.NewTableExtractAstVisitor(annotatedAST)
-	tVis.Visit(ast) //nolint:errcheck // TODO: fix this
-
-	// Fourth pass AST analysis.
-	// Accepts slice of parser table objects
-	// extracted from previous analysis.
-	// Extracts:
-	//   - Col Refs; mapping columnar objects to tables.
-	//   - Alias Map; mapping the "TableName" objects
-	//     defining aliases to table objects.
-	aVis := astvisit.NewTableAliasAstVisitor(annotatedAST, tVis.GetTables())
-	aVis.Visit(ast) //nolint:errcheck // TODO: fix this
-
-	// Fifth pass AST analysis.
-	// Extracts:
-	//   - Columnar parameters with null values.
-	//     Useful for method matching.
-	//     Especially for "Insert" queries.
-	tpv := astvisit.NewPlaceholderParamAstVisitor(annotatedAST, "", false)
-	tpv.Visit(ast) //nolint:errcheck // TODO: fix this
+	// Third to fifth pass AST analysis; extract parser table objects, col refs, and parameters.
+	threeToFiveAgg, err := thirdToFifthPasses(ast, annotatedAST)
+	if err != nil {
+		return err
+	}
 
 	pbi, err := planbuilderinput.NewPlanBuilderInput(
 		annotatedAST,
 		handlerCtx,
 		ast,
-		tVis.GetTables(),
-		aVis.GetAliasedColumns(),
-		tVis.GetAliasMap(),
-		aVis.GetColRefs(),
-		tpv.GetParameters(),
+		threeToFiveAgg.GetTables(),
+		threeToFiveAgg.GetAliasedColumns(),
+		threeToFiveAgg.GetAliasMap(),
+		threeToFiveAgg.GetColRefs(),
+		threeToFiveAgg.GetParameters(),
 		tcc.Clone(),
 	)
 	if err != nil {
@@ -362,7 +423,49 @@ func (sp *standardInitialPasses) initialPasses(
 		}
 		pbi = routeAnalyzer.GetPlanBuilderInput()
 	case *sqlparser.Union:
-		routeAnalyzer := routeanalysis.NewSelectRoutePass(node, pbi, whereParams)
+		lhsThreeToFiveAgg, passErr := thirdToFifthPasses(node.FirstStatement, annotatedAST)
+		if passErr != nil {
+			return passErr
+		}
+		lhsPbi, pbiErr := planbuilderinput.NewPlanBuilderInput(
+			annotatedAST,
+			handlerCtx,
+			node,
+			lhsThreeToFiveAgg.GetTables(),
+			lhsThreeToFiveAgg.GetAliasedColumns(),
+			lhsThreeToFiveAgg.GetAliasMap(),
+			lhsThreeToFiveAgg.GetColRefs(),
+			lhsThreeToFiveAgg.GetParameters(),
+			tcc.Clone(),
+		)
+		if pbiErr != nil {
+			return pbiErr
+		}
+		rhsPbi := lhsPbi
+		for _, stmt := range node.UnionSelects {
+			var nextPbi planbuilderinput.PlanBuilderInput
+			rhsThreeToFiveAgg, nextPassErr := thirdToFifthPasses(stmt, annotatedAST)
+			if nextPassErr != nil {
+				return nextPassErr
+			}
+			nextPbi, pbiErr = planbuilderinput.NewPlanBuilderInput(
+				annotatedAST,
+				handlerCtx,
+				stmt.Statement,
+				rhsThreeToFiveAgg.GetTables(),
+				rhsThreeToFiveAgg.GetAliasedColumns(),
+				rhsThreeToFiveAgg.GetAliasMap(),
+				rhsThreeToFiveAgg.GetColRefs(),
+				rhsThreeToFiveAgg.GetParameters(),
+				tcc.CloneAndIncrementInsertID(),
+			)
+			if pbiErr != nil {
+				return pbiErr
+			}
+			rhsPbi.WithNext(nextPbi)
+			rhsPbi = nextPbi
+		}
+		routeAnalyzer := routeanalysis.NewSelectRoutePass(node, lhsPbi, whereParams)
 		err = routeAnalyzer.RoutePass()
 		if err != nil {
 			return err

internal/stackql/astanalysis/routeanalysis/select_route_analysis.go

@@ -1,6 +1,8 @@
 package routeanalysis
 
 import (
+	"fmt"
+
 	"github.com/stackql/stackql-parser/go/vt/sqlparser"
 	"github.com/stackql/stackql/internal/stackql/astanalysis/selectmetadata"
 	"github.com/stackql/stackql/internal/stackql/astvisit"
@@ -57,7 +59,7 @@ func (sp *standardSelectRoutePass) RoutePass() error {
 
 	pbi := sp.inputPbi.Clone()
 
-	counters := pbi.GetTxnCtrlCtrs()
+	// counters := pbi.GetTxnCtrlCtrs()
 
 	switch n := sp.node.(type) {
 	case *sqlparser.Select:
@@ -69,34 +71,25 @@ func (sp *standardSelectRoutePass) RoutePass() error {
 		sp.outputPbi = pbi
 		return err
 	case *sqlparser.Union:
-		lhsPbi, err := planbuilderinput.NewPlanBuilderInput(
-			pbi.GetAnnotatedAST(),
-			sp.handlerCtx,
-			n.FirstStatement,
-			nil, nil, nil, nil, nil, counters)
-		if err != nil {
-			return err
-		}
-		routePass := NewSelectRoutePass(n.FirstStatement, lhsPbi, sp.parentWhereParams)
-		err = routePass.RoutePass()
+		routePass := NewSelectRoutePass(n.FirstStatement, pbi, sp.parentWhereParams)
+		err := routePass.RoutePass()
 		if err != nil {
 			return err
 		}
 		lhsPGInternalOnly := routePass.IsPGInternalOnly()
 		// TODO: eventualy accomodate sharing pg native stuff to
 		//       mix and match with stackql stuff.
 		var rhsNonPGInternalDetected bool
+		rhsPbi := pbi
+		var hasPbi bool
 		for _, s := range n.UnionSelects {
-			ctrClone := counters.CloneAndIncrementInsertID()
-			sPbi, err := planbuilderinput.NewPlanBuilderInput( //nolint:govet // defer cosmetics
-				sp.inputPbi.GetAnnotatedAST(),
-				sp.handlerCtx,
-				s.Statement,
-				nil, nil, nil, nil, nil, ctrClone)
-			if err != nil {
-				return err
+			rhsPbi, hasPbi = rhsPbi.Next()
+			if !hasPbi {
+				return fmt.Errorf("no more PBIs for union selects")
 			}
-			routePass := NewSelectRoutePass(s.Statement, sPbi, sp.parentWhereParams) //nolint:govet // intentional shadow
+			// ctrClone := counters.CloneAndIncrementInsertID()
+			// rhsPbi.SetTxnCtrlCtrs(ctrClone)
+			routePass := NewSelectRoutePass(s.Statement, rhsPbi, sp.parentWhereParams) //nolint:govet // intentional shadow
 			err = routePass.RoutePass()
 			if err != nil {
 				return err

internal/stackql/planbuilderinput/plan_builder_input.go

@@ -58,6 +58,9 @@ type PlanBuilderInput interface {
 	WithTableRouteVisitor(tableRouteVisitor router.TableRouteAstVisitor) PlanBuilderInput
 	SetReadOnly(bool)
 	IsReadOnly() bool
+	Next() (PlanBuilderInput, bool)
+	WithNext(PlanBuilderInput)
+	SetTxnCtrlCtrs(tcc internaldto.TxnControlCounters)
 }
 
 type StandardPlanBuilderInput struct {
@@ -80,6 +83,7 @@ type StandardPlanBuilderInput struct {
 	prepStmtOffset           int
 	isCreateMaterializedView bool
 	rawQuery                 string
+	next                     PlanBuilderInput
 }
 
 func NewPlanBuilderInput(
@@ -137,6 +141,14 @@ func newPlanBuilderInput(
 	return rv
 }
 
+func (pbi *StandardPlanBuilderInput) Next() (PlanBuilderInput, bool) {
+	return pbi.next, pbi.next != nil
+}
+
+func (pbi *StandardPlanBuilderInput) WithNext(next PlanBuilderInput) {
+	pbi.next = next
+}
+
 func (pbi *StandardPlanBuilderInput) Clone() PlanBuilderInput {
 	clonedPbi := newPlanBuilderInput(
 		pbi.annotatedAST,
@@ -152,6 +164,7 @@ func (pbi *StandardPlanBuilderInput) Clone() PlanBuilderInput {
 	clonedPbi.SetPrepStmtOffset(pbi.prepStmtOffset)
 	clonedPbi.SetReadOnly(pbi.IsReadOnly())
 	clonedPbi.SetCreateMaterializedView(pbi.isCreateMaterializedView)
+	clonedPbi.WithNext(pbi.next)
 	return clonedPbi
 }
 
@@ -267,6 +280,10 @@ func (pbi *StandardPlanBuilderInput) GetTxnCtrlCtrs() internaldto.TxnControlCoun
 	return pbi.tcc
 }
 
+func (pbi *StandardPlanBuilderInput) SetTxnCtrlCtrs(tcc internaldto.TxnControlCounters) {
+	pbi.tcc = tcc
+}
+
 func (pbi *StandardPlanBuilderInput) GetPlaceholderParams() parserutil.ParameterMap {
 	return pbi.paramsPlaceheld
 }

internal/stackql/primitivegenerator/statement_analyzer.go

@@ -165,32 +165,35 @@ func (pb *standardPrimitiveGenerator) analyzeUnion(
 
 	pb.prepStmtOffset = ctx.GetCtrlColumnRepeats()
 
+	currentPbi := pbi
+	var currentPbiExists bool
+
 	for _, rhsStmt := range node.UnionSelects {
 		i++
 		leaf, err := pb.PrimitiveComposer.GetSymTab().NewLeaf(i)
 		if err != nil {
 			return err
 		}
+		currentPbi, currentPbiExists = currentPbi.Next()
+		if !currentPbiExists {
+			return fmt.Errorf("no more PBIs for union selects in analysis of union statement")
+		}
 		pChild := pb.AddChildPrimitiveGenerator(rhsStmt.Statement, leaf).WithPrepStmtOffset(pb.prepStmtOffset)
 		pChild.SetElideRead(true)
 		ctrClone = ctrClone.CloneAndIncrementInsertID()
-		sPbi, err := planbuilderinput.NewPlanBuilderInput(
-			pbi.GetAnnotatedAST(),
-			handlerCtx,
-			rhsStmt.Statement,
-			nil, nil, nil, nil, nil, ctrClone)
+
 		if err != nil {
 			return err
 		}
-		sPbi.SetIsTccSetAheadOfTime(true)
-		sPbi.SetPrepStmtOffset(pb.prepStmtOffset)
-		err = pChild.AnalyzeSelectStatement(sPbi)
+		currentPbi.SetIsTccSetAheadOfTime(true)
+		currentPbi.SetPrepStmtOffset(pb.prepStmtOffset)
+		err = pChild.AnalyzeSelectStatement(currentPbi)
 		if err != nil {
 			return err
 		}
 		ctx := pChild.GetPrimitiveComposer().GetSelectPreparedStatementCtx()
 		ctx.SetKind(rhsStmt.Type)
-		ctx.SetGCCtrlCtrs(ctrClone)
+		// ctx.SetGCCtrlCtrs(ctrClone)
 		selectStatementContexts = append(selectStatementContexts, ctx)
 		pb.prepStmtOffset += ctx.GetCtrlColumnRepeats()
 		// unionSelectCtx