MB-32638 Fix scope of CTE to cover setop queries

Previous implementation of CTE associated CTE with a subselect, which
is analogous to how LET is associated with a subselect. One issue
with this approach is that when a CTE is defined in a setop query
(e.g. UNION) the CTE is only visible from the first arm of the UNION,
which is not how CTE is defined in the SQL standard.

We now "raise" the scope of the CTE to be associated with a fullselect
rather than a subselect, which is consistent with the SQL standard.

In addition, in SQL standard the CTE (WITH clause) is only allowed
to be specified for the top-most query, it is NOT allowed in a query
that is under a setop operation (e.g. UNION). However, since
  1. Previous implementation does allow WITH clause under a setop
     operation
  2. Postgres also support this behavior (although Oracle follows
     the SQL standard and gives error for such queries)
we will also support WITH clause in a query under setop operations.
This means you could have "nested" WITH clause, and the scope
(or visibility) of each WITH clause is defined by where it is
specified in the query.

Change-Id: Ie93eb33056d268c898ad33b91a175182dccf5e8f
Reviewed-on: https://review.couchbase.org/c/query/+/175119
Reviewed-by: Sitaram Vemulapalli <sitaram.vemulapalli@couchbase.com>
Reviewed-by: Marco Greco <marco.greco@couchbase.com>
Tested-by: Bingjie Miao <bingjie.miao@couchbase.com>

algebra/select.go

@@ -29,19 +29,22 @@ type Select struct {
 	statementBase
 
 	subresult  Subresult             `json:"subresult"`
+	with       expression.Bindings   `json:"with"`
 	order      *Order                `json:"order"`
 	offset     expression.Expression `json:"offset"`
 	limit      expression.Expression `json:"limit"`
 	correlated bool                  `json:"correlated"`
+	setop      bool                  `json:"setop"`
 }
 
 /*
 The function NewSelect returns a pointer to the Select struct
 by assigning the input attributes to the fields of the struct.
 */
-func NewSelect(subresult Subresult, order *Order, offset, limit expression.Expression) *Select {
+func NewSelect(subresult Subresult, with expression.Bindings, order *Order, offset, limit expression.Expression) *Select {
 	rv := &Select{
 		subresult: subresult,
+		with:      with,
 		order:     order,
 		offset:    offset,
 		limit:     limit,
@@ -90,6 +93,10 @@ func (this *Select) MapExpressions(mapper expression.Mapper) (err error) {
 		return
 	}
 
+	if this.with != nil {
+		err = this.with.MapExpressions(mapper)
+	}
+
 	if this.order != nil {
 		err = this.order.MapExpressions(mapper)
 	}
@@ -114,6 +121,10 @@ func (this *Select) MapExpressions(mapper expression.Mapper) (err error) {
 func (this *Select) Expressions() expression.Expressions {
 	exprs := this.subresult.Expressions()
 
+	if this.with != nil {
+		exprs = append(exprs, this.with.Expressions()...)
+	}
+
 	if this.order != nil {
 		exprs = append(exprs, this.order.Expressions()...)
 	}
@@ -140,6 +151,10 @@ func (this *Select) Privileges() (*auth.Privileges, errors.Error) {
 
 	exprs := make(expression.Expressions, 0, 16)
 
+	if this.with != nil {
+		exprs = append(exprs, this.with.Expressions()...)
+	}
+
 	if this.order != nil {
 		exprs = append(exprs, this.order.Expressions()...)
 	}
@@ -169,7 +184,13 @@ func (this *Select) Privileges() (*auth.Privileges, errors.Error) {
    Representation as a N1QL string.
 */
 func (this *Select) String() string {
-	s := this.subresult.String()
+	var s string
+
+	if len(this.with) > 0 {
+		s += withBindings(this.with)
+	}
+
+	s += this.subresult.String()
 
 	if this.order != nil {
 		s += " " + this.order.String()
@@ -192,13 +213,25 @@ namely the subresult, order, limit and offset within a subquery.
 For the subresult of the subquery, call Formalize, for the order
 by clause call MapExpressions, for limit and offset call Accept.
 */
-func (this *Select) FormalizeSubquery(parent *expression.Formalizer) error {
-	if parent != nil {
+func (this *Select) FormalizeSubquery(parent *expression.Formalizer) (err error) {
+	if parent != nil && !this.setop {
 		withs := parent.SaveWiths()
 		defer parent.RestoreWiths(withs)
 	}
 
-	f, err := this.subresult.Formalize(parent)
+	var f *expression.Formalizer
+	if this.with != nil {
+		f = expression.NewFormalizer("", parent)
+		err = f.PushBindings(this.with, false)
+		if err != nil {
+			return err
+		}
+		f.SetWiths(this.with)
+	} else {
+		f = parent
+	}
+
+	f, err = this.subresult.Formalize(f)
 	if err != nil {
 		return err
 	}
@@ -305,6 +338,24 @@ func (this *Select) SetCorrelated() {
 	this.correlated = true
 }
 
+/*
+this.setop indicates whether the Select is under a set operation (UNION/INTERSECT/EXCEPT)
+*/
+func (this *Select) IsUnderSetOp() bool {
+	return this.setop
+}
+
+func (this *Select) SetUnderSetOp() {
+	this.setop = true
+}
+
+/*
+Returns the With clause in the select statement.
+*/
+func (this *Select) With() expression.Bindings {
+	return this.with
+}
+
 func (this *Select) OptimHints() *OptimHints {
 	return this.subresult.OptimHints()
 }

algebra/select_sub.go

@@ -23,7 +23,6 @@ group and projection, map to the FromTerm, let clause, group by
 and select clause respectively.
 */
 type Subselect struct {
-	with       expression.Bindings   `json:"with"`
 	from       FromTerm              `json:"from"`
 	let        expression.Bindings   `json:"let"`
 	where      expression.Expression `json:"where"`
@@ -37,12 +36,11 @@ type Subselect struct {
 /*
 Constructor.
 */
-func NewSubselect(with expression.Bindings, from FromTerm, let expression.Bindings,
+func NewSubselect(from FromTerm, let expression.Bindings,
 	where expression.Expression, group *Group, window WindowTerms,
 	projection *Projection, optimHints *OptimHints) *Subselect {
 
 	return &Subselect{
-		with:       with,
 		from:       from,
 		let:        let,
 		where:      where,
@@ -76,24 +74,12 @@ group and projections, calls Map to map the where
 expressions and calls PushBindings for the let clause.
 */
 func (this *Subselect) Formalize(parent *expression.Formalizer) (f *expression.Formalizer, err error) {
-	if this.with != nil {
-		f = expression.NewFormalizer("", parent)
-		err = f.PushBindings(this.with, false)
-		if err != nil {
-			return nil, err
-		}
-		f.SetWiths(this.with)
-	}
-
 	if this.from != nil {
-		if f == nil {
-			f = parent
-		}
-		f, err = this.from.Formalize(f)
+		f, err = this.from.Formalize(parent)
 		if err != nil {
 			return nil, err
 		}
-	} else if f == nil {
+	} else {
 		f = expression.NewFormalizer("", parent)
 	}
 
@@ -247,10 +233,6 @@ func (this *Subselect) Privileges() (*auth.Privileges, errors.Error) {
 		exprs = append(exprs, this.let.Expressions()...)
 	}
 
-	if this.with != nil {
-		exprs = append(exprs, this.with.Expressions()...)
-	}
-
 	if this.where != nil {
 		exprs = append(exprs, this.where)
 	}
@@ -282,13 +264,7 @@ func (this *Subselect) Privileges() (*auth.Privileges, errors.Error) {
    Representation as a N1QL string.
 */
 func (this *Subselect) String() string {
-	var s string
-
-	if len(this.with) > 0 {
-		s += withBindings(this.with)
-	}
-
-	s += "select " + this.projection.String()
+	s := "select " + this.projection.String()
 
 	if this.from != nil {
 		s += " from " + this.from.String()
@@ -320,14 +296,6 @@ func (this *Subselect) SetCorrelated() {
 	this.correlated = true
 }
 
-/*
-Returns the let field that represents the With
-clause in the subselect statement.
-*/
-func (this *Subselect) With() expression.Bindings {
-	return this.with
-}
-
 /*
 Returns a FromTerm that represents the From clause
 in the subselect statement.

execution/with.go

@@ -48,6 +48,10 @@ func (this *With) PlanOp() plan.Operator {
 	return this.plan
 }
 
+func (this *With) IsParallel() bool {
+	return this.child.IsParallel()
+}
+
 func (this *With) RunOnce(context *Context, parent value.Value) {
 	this.once.Do(func() {
 		defer context.Recover(&this.base) // Recover from any panic

expression/formalizer.go

@@ -566,15 +566,6 @@ func (this *Formalizer) SetWiths(withs Bindings) {
 	}
 }
 
-func (this *Formalizer) SetPermanentWiths(withs Bindings) {
-	if this.withs == nil {
-		this.withs = make(map[string]bool, len(withs))
-	}
-	for _, b := range withs {
-		this.withs[b.Variable()] = true
-	}
-}
-
 func (this *Formalizer) SaveWiths() map[string]bool {
 	withs := this.withs
 	this.withs = make(map[string]bool, len(withs))

parser/n1ql/n1ql.y

@@ -419,7 +419,7 @@ tokOffset        int
 %type <expr>             on_keys on_key
 %type <indexRefs>        index_refs
 %type <indexRef>         index_ref
-%type <bindings>         opt_let let opt_with
+%type <bindings>         opt_let let with
 %type <expr>             opt_where where opt_filter
 %type <group>            opt_group group
 %type <bindings>         opt_letting letting
@@ -830,17 +830,32 @@ set_transaction_isolation
 fullselect:
 select_terms opt_order_by
 {
-    $$ = algebra.NewSelect($1, $2, nil, nil) /* OFFSET precedes LIMIT */
+    $$ = algebra.NewSelect($1, nil, $2, nil, nil) /* OFFSET precedes LIMIT */
 }
 |
 select_terms opt_order_by limit opt_offset
 {
-    $$ = algebra.NewSelect($1, $2, $4, $3) /* OFFSET precedes LIMIT */
+    $$ = algebra.NewSelect($1, nil, $2, $4, $3) /* OFFSET precedes LIMIT */
 }
 |
 select_terms opt_order_by offset opt_limit
 {
-    $$ = algebra.NewSelect($1, $2, $3, $4) /* OFFSET precedes LIMIT */
+    $$ = algebra.NewSelect($1, nil, $2, $3, $4) /* OFFSET precedes LIMIT */
+}
+|
+with select_terms opt_order_by
+{
+    $$ = algebra.NewSelect($2, $1, $3, nil, nil) /* OFFSET precedes LIMIT */
+}
+|
+with select_terms opt_order_by limit opt_offset
+{
+    $$ = algebra.NewSelect($2, $1, $3, $5, $4) /* OFFSET precedes LIMIT */
+}
+|
+with select_terms opt_order_by offset opt_limit
+{
+    $$ = algebra.NewSelect($2, $1, $3, $4, $5) /* OFFSET precedes LIMIT */
 }
 ;
 
@@ -882,36 +897,42 @@ select_terms EXCEPT ALL select_term
 |
 subquery_expr UNION select_term
 {
+    $1.Select().SetUnderSetOp()
     left_term := algebra.NewSelectTerm($1.Select())
     $$ = algebra.NewUnion(left_term, $3)
 }
 |
 subquery_expr UNION ALL select_term
 {
+    $1.Select().SetUnderSetOp()
     left_term := algebra.NewSelectTerm($1.Select())
     $$ = algebra.NewUnionAll(left_term, $4)
 }
 |
 subquery_expr INTERSECT select_term
 {
+    $1.Select().SetUnderSetOp()
     left_term := algebra.NewSelectTerm($1.Select())
     $$ = algebra.NewIntersect(left_term, $3)
 }
 |
 subquery_expr INTERSECT ALL select_term
 {
+    $1.Select().SetUnderSetOp()
     left_term := algebra.NewSelectTerm($1.Select())
     $$ = algebra.NewIntersectAll(left_term, $4)
 }
 |
 subquery_expr EXCEPT select_term
 {
+    $1.Select().SetUnderSetOp()
     left_term := algebra.NewSelectTerm($1.Select())
     $$ = algebra.NewExcept(left_term, $3)
 }
 |
 subquery_expr EXCEPT ALL select_term
 {
+    $1.Select().SetUnderSetOp()
     left_term := algebra.NewSelectTerm($1.Select())
     $$ = algebra.NewExceptAll(left_term, $4)
 }
@@ -925,6 +946,8 @@ subselect
 |
 subquery_expr
 {
+    // all current uses of select_term is under setop
+    $1.Select().SetUnderSetOp()
     $$ = algebra.NewSelectTerm($1.Select())
 }
 ;
@@ -936,16 +959,16 @@ select_from
 ;
 
 from_select:
-opt_with from opt_let opt_where opt_group opt_window_clause SELECT opt_optim_hints projection
+from opt_let opt_where opt_group opt_window_clause SELECT opt_optim_hints projection
 {
-    $$ = algebra.NewSubselect($1, $2, $3, $4, $5, $6, $9, $8)
+    $$ = algebra.NewSubselect($1, $2, $3, $4, $5, $8, $7)
 }
 ;
 
 select_from:
-opt_with SELECT opt_optim_hints projection opt_from opt_let opt_where opt_group opt_window_clause
+SELECT opt_optim_hints projection opt_from opt_let opt_where opt_group opt_window_clause
 {
-    $$ = algebra.NewSubselect($1, $5, $6, $7, $8, $9, $4, $3)
+    $$ = algebra.NewSubselect($4, $5, $6, $7, $8, $3, $2)
 }
 ;
 
@@ -1637,10 +1660,7 @@ alias EQ expr
  *
  *************************************************/
 
-opt_with:
-/* empty */
-{ $$ = nil }
-|
+with:
 WITH with_list
 {
     $$ = $2

planner/build_select.go

@@ -95,6 +95,18 @@ func (this *builder) VisitSelect(stmt *algebra.Select) (interface{}, error) {
 		return nil, err
 	}
 
+	with := stmt.With()
+	if with != nil {
+		cost := OPT_COST_NOT_AVAIL
+		cardinality := OPT_CARD_NOT_AVAIL
+		size := OPT_SIZE_NOT_AVAIL
+		frCost := OPT_COST_NOT_AVAIL
+		if this.useCBO {
+			cost, cardinality, size, frCost = getWithCost(sub.(plan.Operator), with)
+		}
+		sub = plan.NewWith(with, sub.(plan.Operator), cost, cardinality, size, frCost)
+	}
+
 	if stmtOrder == nil && stmtOffset == nil && stmtLimit == nil {
 		return sub, nil
 	}