Merge #52730

52730: sql: new EXPLAIN infrastructure r=RaduBerinde a=RaduBerinde

#### opt: store columns in OpaqueMetadata

Store the result columns of opaque operators in their metadata instead
of returning it separately.

Release note: None

#### opt: separate CreateTable/CreateTableAs exec operators

Separate the two variants in the exec factory so that we don't have to
pass around a `nil` node.

Release note: None

#### sql: new EXPLAIN infrastructure

This change reimplements the default `(PLAN)` variant of EXPLAIN.

Instead of visiting a plan node tree to get EXPLAIN information, we
record the exact `exec.Factory` calls that were produced. We do this
using an `exec.ExplainFactory` which is an extension of
`exec.Factory`.

This factory wraps a regular `exec.Factory` and forwards all factory
calls to it (while also recording all arguments). The result is that
for each `exec.Node` there is a corresponding `*explain.Node`. The
factory also provides a way for the execbuilder to annotate nodes with
more information, which will be initially used for row count and cost
(not exposed in this PR).

The implementation relies on optgen for the factory methods and
per-operation identifiers and structs.

I tried to avoid major changes to the EXPLAIN output; I left a lot of
TODOs for improvements that I plan to make in follow-up changes. Some
egregious issues were fixed in separate PRs. Still, there are various
differences; the notable ones are:
 - for simple projections, we now show a `project` node without any
   rendering expressions.
 - in some cases we see a new `project` that just rearranges columns.
   I will address this in a follow-up change.
 - window function information is no longer duplicated.
 - expressions for inverted filters and joins now show the column
   names instead of unnamed references like `@2`.
 - we no longer see a `count` node on top of mutations.

Release note (sql change): various improvements to EXPLAIN.

#### sql: EXPLAIN test updates

Separate commit for explain test changes.

Release note: None

Co-authored-by: Radu Berinde <radu@cockroachlabs.com>

Makefile

@@ -917,7 +917,8 @@ OPTGEN_TARGETS = \
 	pkg/sql/opt/norm/factory.og.go \
 	pkg/sql/opt/rule_name.og.go \
 	pkg/sql/opt/rule_name_string.go \
-	pkg/sql/opt/exec/factory.og.go
+	pkg/sql/opt/exec/factory.og.go \
+	pkg/sql/opt/exec/explain/explain_factory.og.go
 
 go-targets-ccl := \
 	$(COCKROACH) $(COCKROACHSHORT) \
@@ -1621,6 +1622,9 @@ pkg/sql/opt/norm/factory.og.go: $(optgen-defs) $(optgen-norm-rules) bin/optgen
 pkg/sql/opt/exec/factory.og.go: $(optgen-defs) $(optgen-exec-defs) bin/optgen
 	optgen -out $@ execfactory $(optgen-exec-defs)
 
+pkg/sql/opt/exec/explain/explain_factory.og.go: $(optgen-defs) $(optgen-exec-defs) bin/optgen
+	optgen -out $@ execexplain $(optgen-exec-defs)
+
 # Format non-generated .cc and .h files in libroach using clang-format.
 .PHONY: c-deps-fmt
 c-deps-fmt:

pkg/cli/cli_test.go

@@ -1369,15 +1369,16 @@ func Example_misc_table() {
 	//     hai
 	// (1 row)
 	// sql --format=table -e explain select s, 'foo' from t.t
-	//     tree    |    field     | description
-	// ------------+--------------+--------------
-	//             | distribution | full
-	//             | vectorized   | false
-	//   render    |              |
-	//    └── scan |              |
-	//             | table        | t@primary
-	//             | spans        | FULL SCAN
-	// (6 rows)
+	//        tree      |    field     | description
+	// -----------------+--------------+--------------
+	//                  | distribution | full
+	//                  | vectorized   | false
+	//   project        |              |
+	//    └── render    |              |
+	//         └── scan |              |
+	//                  | table        | t@primary
+	//                  | spans        | FULL SCAN
+	// (7 rows)
 }
 
 func Example_cert() {

pkg/sql/distsql_physical_planner.go

@@ -2669,7 +2669,7 @@ func (dsp *DistSQLPlanner) wrapPlan(planCtx *PlanningCtx, n planNode) (*Physical
 	if err := walkPlan(planCtx.ctx, n, planObserver{
 		enterNode: func(ctx context.Context, nodeName string, plan planNode) (bool, error) {
 			switch plan.(type) {
-			case *explainDistSQLNode, *explainPlanNode, *explainVecNode:
+			case *explainDistSQLNode, *explainPlanNode, *explainVecNode, *explainNewPlanNode:
 				// Don't continue recursing into explain nodes - they need to be left
 				// alone since they handle their own planning later.
 				return false, nil

pkg/sql/distsql_spec_exec_factory.go

@@ -609,10 +609,10 @@ func (e *distSQLSpecExecFactory) ConstructSetOp(
 }
 
 func (e *distSQLSpecExecFactory) ConstructSort(
-	input exec.Node, ordering sqlbase.ColumnOrdering, alreadyOrderedPrefix int,
+	input exec.Node, ordering exec.OutputOrdering, alreadyOrderedPrefix int,
 ) (exec.Node, error) {
 	physPlan, plan := getPhysPlan(input)
-	e.dsp.addSorters(physPlan, ordering, alreadyOrderedPrefix)
+	e.dsp.addSorters(physPlan, sqlbase.ColumnOrdering(ordering), alreadyOrderedPrefix)
 	// Since addition of sorters doesn't change any properties of the physical
 	// plan, we don't need to update any of those.
 	return plan, nil
@@ -777,6 +777,12 @@ func (e *distSQLSpecExecFactory) ConstructExplain(
 	return makePlanMaybePhysical(physPlan, []planNode{explainNode}), nil
 }
 
+func (e *distSQLSpecExecFactory) ConstructExplainPlan(
+	options *tree.ExplainOptions, buildFn exec.BuildPlanForExplainFn,
+) (exec.Node, error) {
+	return nil, unimplemented.NewWithIssue(47473, "experimental opt-driven distsql planning: explain plan")
+}
+
 func (e *distSQLSpecExecFactory) ConstructShowTrace(
 	typ tree.ShowTraceType, compact bool,
 ) (exec.Node, error) {
@@ -854,6 +860,12 @@ func (e *distSQLSpecExecFactory) ConstructDeleteRange(
 }
 
 func (e *distSQLSpecExecFactory) ConstructCreateTable(
+	schema cat.Schema, ct *tree.CreateTable,
+) (exec.Node, error) {
+	return nil, unimplemented.NewWithIssue(47473, "experimental opt-driven distsql planning: create table")
+}
+
+func (e *distSQLSpecExecFactory) ConstructCreateTableAs(
 	input exec.Node, schema cat.Schema, ct *tree.CreateTable,
 ) (exec.Node, error) {
 	return nil, unimplemented.NewWithIssue(47473, "experimental opt-driven distsql planning: create table")

pkg/sql/exec_factory_util.go

@@ -298,7 +298,7 @@ func constructVirtualScan(
 	// Virtual indexes never provide a legitimate ordering, so we have to make
 	// sure to sort if we have a required ordering.
 	if len(reqOrdering) != 0 {
-		n, err = ef.ConstructSort(n, sqlbase.ColumnOrdering(reqOrdering), 0)
+		n, err = ef.ConstructSort(n, reqOrdering, 0)
 		if err != nil {
 			return nil, err
 		}

pkg/sql/explain_plan_new.go

@@ -0,0 +1,107 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package sql
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
+	"github.com/cockroachdb/cockroach/pkg/sql/opt/cat"
+	"github.com/cockroachdb/cockroach/pkg/sql/opt/exec"
+	"github.com/cockroachdb/cockroach/pkg/sql/opt/exec/explain"
+	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
+	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
+)
+
+// explainNewPlanNode implements EXPLAIN (PLAN); it produces the output of
+// EXPLAIN given an explain.Plan.
+//
+// TODO(radu): move and rename this once explainPlanNode is removed.
+type explainNewPlanNode struct {
+	flags explain.Flags
+	plan  *explain.Plan
+	run   explainNewPlanNodeRun
+
+	columns sqlbase.ResultColumns
+}
+
+type explainNewPlanNodeRun struct {
+	results *valuesNode
+}
+
+func (e *explainNewPlanNode) startExec(params runParams) error {
+	ob := explain.NewOutputBuilder(e.flags)
+
+	realPlan := e.plan.WrappedPlan.(*planTop)
+	distribution, willVectorize := explainGetDistributedAndVectorized(params, &realPlan.planComponents)
+	ob.AddField("distribution", distribution.String())
+	ob.AddField("vectorized", fmt.Sprintf("%t", willVectorize))
+
+	spanFormatFn := func(table cat.Table, index cat.Index, scanParams exec.ScanParams) string {
+		var tabDesc *sqlbase.ImmutableTableDescriptor
+		var idxDesc *descpb.IndexDescriptor
+		if table.IsVirtualTable() {
+			tabDesc = table.(*optVirtualTable).desc
+			idxDesc = index.(*optVirtualIndex).desc
+		} else {
+			tabDesc = table.(*optTable).desc
+			idxDesc = index.(*optIndex).desc
+		}
+		spans, err := generateScanSpans(params.p, tabDesc, idxDesc, scanParams)
+		if err != nil {
+			return err.Error()
+		}
+		// skip is how many fields to skip when pretty-printing spans.
+		// Usually 2, but can be 4 when running EXPLAIN from a tenant since there
+		// will be an extra tenant prefix and ID. For example:
+		//  - /51/1/1 is a key read as a system tenant where the first two values
+		//    are the table ID and the index ID.
+		//  - /Tenant/10/51/1/1 is a key read as a non-system tenant where the first
+		//    four values are the special tenant prefix byte and tenant ID, followed
+		//    by the table ID and the index ID.
+		skip := 2
+		if !params.p.ExecCfg().Codec.ForSystemTenant() {
+			skip = 4
+		}
+		return sqlbase.PrettySpans(idxDesc, spans, skip)
+	}
+
+	if err := explain.Emit(e.plan, ob, spanFormatFn); err != nil {
+		return err
+	}
+
+	v := params.p.newContainerValuesNode(e.columns, 0)
+	for _, row := range ob.BuildExplainRows() {
+		if _, err := v.rows.AddRow(params.ctx, row); err != nil {
+			return err
+		}
+	}
+	e.run.results = v
+
+	return nil
+}
+
+func (e *explainNewPlanNode) Next(params runParams) (bool, error) { return e.run.results.Next(params) }
+func (e *explainNewPlanNode) Values() tree.Datums                 { return e.run.results.Values() }
+
+func (e *explainNewPlanNode) Close(ctx context.Context) {
+	e.plan.Root.WrappedNode().(planNode).Close(ctx)
+	for i := range e.plan.Subqueries {
+		e.plan.Subqueries[i].Root.(*explain.Node).WrappedNode().(planNode).Close(ctx)
+	}
+	for i := range e.plan.Checks {
+		e.plan.Checks[i].WrappedNode().(planNode).Close(ctx)
+	}
+	if e.run.results != nil {
+		e.run.results.Close(ctx)
+	}
+}

pkg/sql/logictest/testdata/logic_test/alter_primary_key

@@ -500,12 +500,10 @@ t  CREATE TABLE public.t (
 query TTT
 SELECT * FROM [EXPLAIN INSERT INTO t VALUES (4, 5, 6)] OFFSET 2
 ----
-count                  ·            ·
- └── insert fast path  ·            ·
-·                      into         t(x, y, z, crdb_internal_z_shard_5, crdb_internal_y_shard_10)
-·                      strategy     inserter
-·                      auto commit  ·
-·                      size         7 columns, 1 row
+insert fast path  ·            ·
+·                 into         t(x, y, z, crdb_internal_z_shard_5, crdb_internal_y_shard_10)
+·                 auto commit  ·
+·                 size         7 columns, 1 row
 
 # Ensure that all of the indexes have been rewritten.
 query IT

pkg/sql/logictest/testdata/logic_test/inverted_filter_geospatial_explain_local

@@ -29,15 +29,15 @@ EXPLAIN SELECT k, k_plus_one FROM geo_table2 WHERE ST_Intersects('POINT(3.0 3.0)
 ----
 ·                                    distribution     local
 ·                                    vectorized       false
-render                               ·                ·
+project                              ·                ·
  └── filter                          ·                ·
       │                              filter           st_intersects('010100000000000000000008400000000000000840', geom)
       └── index join                 ·                ·
            │                         table            geo_table2@primary
            │                         key columns      k, k_plus_one
-           └── render                ·                ·
+           └── project               ·                ·
                 └── inverted filter  ·                ·
-                     │               inverted column  2
+                     │               inverted column  geom
                      │               num spans        31
                      └── scan        ·                ·
 ·                                    table            geo_table2@geom_index
@@ -48,15 +48,15 @@ EXPLAIN SELECT k, k_plus_one FROM geo_table2 WHERE ST_DFullyWithin('POINT(3.0 3.
 ----
 ·                                    distribution     local
 ·                                    vectorized       false
-render                               ·                ·
+project                              ·                ·
  └── filter                          ·                ·
       │                              filter           st_dfullywithin('010100000000000000000008400000000000000840', geom, 1.0)
       └── index join                 ·                ·
            │                         table            geo_table2@primary
            │                         key columns      k, k_plus_one
-           └── render                ·                ·
+           └── project               ·                ·
                 └── inverted filter  ·                ·
-                     │               inverted column  2
+                     │               inverted column  geom
                      │               num spans        20
                      └── scan        ·                ·
 ·                                    table            geo_table2@geom_index