release-21.2: sql, opt: support hint to disallow full scan, update coster to avoid full scans

docs/generated/eventlog.md

@@ -315,7 +315,7 @@ is directly or indirectly a member of the admin role) executes a query.
 | `Age` | Age of the query in milliseconds. | no |
 | `NumRetries` | Number of retries, when the txn was reretried automatically by the server. | no |
 | `FullTableScan` | Whether the query contains a full table scan. | no |
-| `FullIndexScan` | Whether the query contains a full secondary index scan. | no |
+| `FullIndexScan` | Whether the query contains a full secondary index scan of a non-partial index. | no |
 | `TxnCounter` | The sequence number of the SQL transaction inside its session. | no |
 
 ### `sensitive_table_access`
@@ -349,7 +349,7 @@ a table marked as audited.
 | `Age` | Age of the query in milliseconds. | no |
 | `NumRetries` | Number of retries, when the txn was reretried automatically by the server. | no |
 | `FullTableScan` | Whether the query contains a full table scan. | no |
-| `FullIndexScan` | Whether the query contains a full secondary index scan. | no |
+| `FullIndexScan` | Whether the query contains a full secondary index scan of a non-partial index. | no |
 | `TxnCounter` | The sequence number of the SQL transaction inside its session. | no |
 
 ## SQL Execution Log
@@ -390,7 +390,7 @@ and the cluster setting `sql.trace.log_statement_execute` is set.
 | `Age` | Age of the query in milliseconds. | no |
 | `NumRetries` | Number of retries, when the txn was reretried automatically by the server. | no |
 | `FullTableScan` | Whether the query contains a full table scan. | no |
-| `FullIndexScan` | Whether the query contains a full secondary index scan. | no |
+| `FullIndexScan` | Whether the query contains a full secondary index scan of a non-partial index. | no |
 | `TxnCounter` | The sequence number of the SQL transaction inside its session. | no |
 
 ## SQL Logical Schema Changes
@@ -1935,7 +1935,7 @@ set to a non-zero value, AND
 | `Age` | Age of the query in milliseconds. | no |
 | `NumRetries` | Number of retries, when the txn was reretried automatically by the server. | no |
 | `FullTableScan` | Whether the query contains a full table scan. | no |
-| `FullIndexScan` | Whether the query contains a full secondary index scan. | no |
+| `FullIndexScan` | Whether the query contains a full secondary index scan of a non-partial index. | no |
 | `TxnCounter` | The sequence number of the SQL transaction inside its session. | no |
 
 ### `txn_rows_read_limit`
@@ -2054,7 +2054,7 @@ the "slow query" condition.
 | `Age` | Age of the query in milliseconds. | no |
 | `NumRetries` | Number of retries, when the txn was reretried automatically by the server. | no |
 | `FullTableScan` | Whether the query contains a full table scan. | no |
-| `FullIndexScan` | Whether the query contains a full secondary index scan. | no |
+| `FullIndexScan` | Whether the query contains a full secondary index scan of a non-partial index. | no |
 | `TxnCounter` | The sequence number of the SQL transaction inside its session. | no |
 
 ### `txn_rows_read_limit_internal`
@@ -2231,7 +2231,7 @@ contains common SQL event/execution details.
 | `Age` | Age of the query in milliseconds. | no |
 | `NumRetries` | Number of retries, when the txn was reretried automatically by the server. | no |
 | `FullTableScan` | Whether the query contains a full table scan. | no |
-| `FullIndexScan` | Whether the query contains a full secondary index scan. | no |
+| `FullIndexScan` | Whether the query contains a full secondary index scan of a non-partial index. | no |
 | `TxnCounter` | The sequence number of the SQL transaction inside its session. | no |
 
 ## Zone config events

docs/generated/sql/bnf/stmt_block.bnf

@@ -990,6 +990,7 @@ unreserved_keyword ::=
 	| 'NORMAL'
 	| 'NO_INDEX_JOIN'
 	| 'NO_ZIGZAG_JOIN'
+	| 'NO_FULL_SCAN'
 	| 'NOCREATEDB'
 	| 'NOCREATELOGIN'
 	| 'NOCANCELQUERY'
@@ -2772,6 +2773,7 @@ index_flags_param ::=
 	'FORCE_INDEX' '=' index_name
 	| 'NO_INDEX_JOIN'
 	| 'NO_ZIGZAG_JOIN'
+	| 'NO_FULL_SCAN'
 
 opt_asc_desc ::=
 	'ASC'

pkg/sql/explain_bundle.go

@@ -507,6 +507,8 @@ func (c *stmtEnvCollector) PrintSessionSettings(w io.Writer) error {
 		return sessiondatapb.VectorizeExecMode(n).String()
 	}
 
+	// TODO(rytaft): Keeping this list up to date is a challenge. Consider just
+	// printing all session settings.
 	relevantSettings := []struct {
 		sessionSetting string
 		clusterSetting settings.WritableSetting
@@ -517,6 +519,11 @@ func (c *stmtEnvCollector) PrintSessionSettings(w io.Writer) error {
 		{sessionSetting: "optimizer_use_histograms", clusterSetting: optUseHistogramsClusterMode, convFunc: boolToOnOff},
 		{sessionSetting: "optimizer_use_multicol_stats", clusterSetting: optUseMultiColStatsClusterMode, convFunc: boolToOnOff},
 		{sessionSetting: "locality_optimized_partitioned_index_scan", clusterSetting: localityOptimizedSearchMode, convFunc: boolToOnOff},
+		{sessionSetting: "prefer_lookup_joins_for_fks", clusterSetting: preferLookupJoinsForFKs, convFunc: boolToOnOff},
+		{sessionSetting: "intervalstyle_enabled", clusterSetting: intervalStyleEnabled, convFunc: boolToOnOff},
+		{sessionSetting: "datestyle_enabled", clusterSetting: dateStyleEnabled, convFunc: boolToOnOff},
+		{sessionSetting: "disallow_full_table_scans", clusterSetting: disallowFullTableScans, convFunc: boolToOnOff},
+		{sessionSetting: "large_full_scan_rows", clusterSetting: largeFullScanRows},
 		{sessionSetting: "distsql", clusterSetting: DistSQLClusterExecMode, convFunc: distsqlConv},
 		{sessionSetting: "vectorize", clusterSetting: VectorizeClusterMode, convFunc: vectorizeConv},
 	}

pkg/sql/logictest/testdata/logic_test/select

@@ -728,21 +728,45 @@ statement ok
 SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false
 
 statement ok
-CREATE TABLE t_disallow_scans(a INT, b INT, INDEX b_idx(b));
+CREATE TABLE t_disallow_scans(a INT, b INT, INDEX b_idx(b), INDEX b_partial(b) WHERE a > 0);
 
 statement ok
 SELECT * FROM t_disallow_scans
 
+# First disable full scans with a hint.
+statement error pq: could not produce a query plan conforming to the NO_FULL_SCAN hint
+SELECT * FROM t_disallow_scans@{NO_FULL_SCAN}
+
+statement error pq: could not produce a query plan conforming to the NO_FULL_SCAN hint
+SELECT * FROM t_disallow_scans@{FORCE_INDEX=b_idx,NO_FULL_SCAN}
+
+statement error pq: could not produce a query plan conforming to the NO_FULL_SCAN hint
+SELECT * FROM t_disallow_scans@{FORCE_INDEX=b_partial,NO_FULL_SCAN} WHERE a > 0
+
+statement ok
+SELECT * FROM t_disallow_scans@{NO_FULL_SCAN} WHERE a > 0
+
+statement ok
+SELECT * FROM t_disallow_scans@{FORCE_INDEX=b_idx,NO_FULL_SCAN} WHERE b > 0
+
+statement ok
+SELECT * FROM t_disallow_scans@{FORCE_INDEX=b_partial,NO_FULL_SCAN} WHERE a > 0 AND b = 1
+
+# Now disable full scans with the session variable.
 statement ok
 SET disallow_full_table_scans = true;
 
 # Full scans are prohibited because we don't have stats on the table.
 statement error pq: query `SELECT \* FROM t_disallow_scans` contains a full table/index scan which is explicitly disallowed
 SELECT * FROM t_disallow_scans
 
-statement error pq: query `SELECT \* FROM t_disallow_scans@b_idx` contains a full table/index scan which is explicitly disallowed
+statement error pq: index "b_idx" cannot be used for this query\nHINT: try overriding the `disallow_full_table_scans`
 SELECT * FROM t_disallow_scans@b_idx
 
+# Full scans of partial indexes are allowed.
+statement ok
+SELECT * FROM t_disallow_scans@b_partial WHERE a > 0
+
 # Disable 'large_full_scan_rows' heuristic and inject the stats in order to
 # test 'disallow_full_table_scans' in isolation.
 statement ok
@@ -776,7 +800,7 @@ ALTER TABLE t_disallow_scans INJECT STATISTICS '[
 statement error pq: query `SELECT \* FROM t_disallow_scans` contains a full table/index scan which is explicitly disallowed
 SELECT * FROM t_disallow_scans
 
-statement error pq: query `SELECT \* FROM t_disallow_scans@b_idx` contains a full table/index scan which is explicitly disallowed
+statement error pq: index "b_idx" cannot be used for this query\nHINT: try overriding the `disallow_full_table_scans`
 SELECT * FROM t_disallow_scans@b_idx
 
 # Internal statements should still be allowed with this setting.
@@ -804,7 +828,7 @@ SET large_full_scan_rows = 2
 statement error pq: query `SELECT \* FROM t_disallow_scans` contains a full table/index scan which is explicitly disallowed
 SELECT * FROM t_disallow_scans
 
-statement error pq: query `SELECT \* FROM t_disallow_scans@b_idx` contains a full table/index scan which is explicitly disallowed
+statement error pq: index "b_idx" cannot be used for this query\nHINT: try overriding the `disallow_full_table_scans`
 SELECT * FROM t_disallow_scans@b_idx
 
 # Cleanup

pkg/sql/opt/exec/execbuilder/builder.go

@@ -92,8 +92,8 @@ type Builder struct {
 	ContainsFullTableScan bool
 
 	// ContainsFullIndexScan is set to true if the statement contains an
-	// unconstrained secondary index scan. This could be a full scan of any
-	// cardinality.
+	// unconstrained non-partial secondary index scan. This could be a full scan
+	// of any cardinality.
 	ContainsFullIndexScan bool
 
 	// ContainsLargeFullTableScan is set to true if the statement contains an
@@ -102,7 +102,7 @@ type Builder struct {
 	ContainsLargeFullTableScan bool
 
 	// ContainsLargeFullIndexScan is set to true if the statement contains an
-	// unconstrained secondary index scan estimated to read more than
+	// unconstrained non-partial secondary index scan estimated to read more than
 	// large_full_scan_rows (or without without available stats).
 	ContainsLargeFullIndexScan bool
 

pkg/sql/opt/exec/execbuilder/relational.go

@@ -534,8 +534,13 @@ func (b *Builder) scanParams(
 				idx.Name(),
 			)
 		default:
-			// This should never happen.
 			err = fmt.Errorf("index \"%s\" cannot be used for this query", idx.Name())
+			if b.evalCtx.SessionData().DisallowFullTableScans &&
+				(b.ContainsLargeFullTableScan || b.ContainsLargeFullIndexScan) {
+				err = errors.WithHint(err,
+					"try overriding the `disallow_full_table_scans` or increasing the `large_full_scan_rows` cluster/session settings",
+				)
+			}
 		}
 
 		return exec.ScanParams{}, opt.ColMap{}, err
@@ -645,24 +650,20 @@ func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) {
 		telemetry.Inc(sqltelemetry.PartialIndexScanUseCounter)
 	}
 
-	params, outputCols, err := b.scanParams(tab, &scan.ScanPrivate, scan.Relational(), scan.RequiredPhysical())
-	if err != nil {
-		return execPlan{}, err
-	}
-	res := execPlan{outputCols: outputCols}
-	root, err := b.factory.ConstructScan(
-		tab,
-		tab.Index(scan.Index),
-		params,
-		res.reqOrdering(scan),
-	)
-	if err != nil {
-		return execPlan{}, err
+	isUnfiltered := scan.IsUnfiltered(md)
+	if scan.Flags.NoFullScan {
+		// Normally a full scan of a partial index would be allowed with the
+		// NO_FULL_SCAN hint (isUnfiltered is false for partial indexes), but if the
+		// user has explicitly forced the partial index *and* used NO_FULL_SCAN, we
+		// disallow the full index scan.
+		if isUnfiltered || (scan.Flags.ForceIndex && scan.IsFullIndexScan(md)) {
+			return execPlan{}, fmt.Errorf("could not produce a query plan conforming to the NO_FULL_SCAN hint")
+		}
 	}
 
 	// Save if we planned a full table/index scan on the builder so that the
 	// planner can be made aware later. We only do this for non-virtual tables.
-	if !tab.IsVirtualTable() && scan.Constraint == nil && scan.InvertedConstraint == nil && !scan.HardLimit.IsSet() {
+	if !tab.IsVirtualTable() && isUnfiltered {
 		stats := scan.Relational().Stats
 		large := !stats.Available || stats.RowCount > b.evalCtx.SessionData().LargeFullScanRows
 		if scan.Index == cat.PrimaryIndex {
@@ -674,6 +675,21 @@ func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) {
 		}
 	}
 
+	params, outputCols, err := b.scanParams(tab, &scan.ScanPrivate, scan.Relational(), scan.RequiredPhysical())
+	if err != nil {
+		return execPlan{}, err
+	}
+	res := execPlan{outputCols: outputCols}
+	root, err := b.factory.ConstructScan(
+		tab,
+		tab.Index(scan.Index),
+		params,
+		res.reqOrdering(scan),
+	)
+	if err != nil {
+		return execPlan{}, err
+	}
+
 	res.root = root
 	return res, nil
 }

pkg/sql/opt/exec/execbuilder/testdata/delete

@@ -250,3 +250,70 @@ WHERE message LIKE '%DelRange%' OR message LIKE '%sending batch%'
 ----
 batch flow coordinator  DelRange /Table/57/1/5 - /Table/57/1/5/#
 dist sender send        r43: sending batch 1 DelRng, 1 EndTxn to (n1,s1):1
+
+statement ok
+CREATE TABLE xyz (
+  x INT PRIMARY KEY,
+  y INT,
+  z INT,
+  INDEX (y)
+)
+
+# Ensure that we can use a hint to avoid a full table scan.
+
+# Without the hint, we plan a full table scan.
+query T
+EXPLAIN (VERBOSE) DELETE FROM xyz WHERE (y > 0 AND y < 1000) OR (y > 2000 AND y < 3000) RETURNING z
+----
+distribution: local
+vectorized: true
+·
+• project
+│ columns: (z)
+│ estimated row count: 990 (missing stats)
+│
+└── • delete
+    │ columns: (x, z)
+    │ estimated row count: 990 (missing stats)
+    │ from: xyz
+    │ auto commit
+    │
+    └── • filter
+        │ columns: (x, y, z)
+        │ estimated row count: 990 (missing stats)
+        │ filter: ((y > 0) AND (y < 1000)) OR ((y > 2000) AND (y < 3000))
+        │
+        └── • scan
+              columns: (x, y, z)
+              estimated row count: 1,000 (missing stats)
+              table: xyz@primary
+              spans: FULL SCAN
+
+# With the hint, we use a constrained scan.
+query T
+EXPLAIN (VERBOSE) DELETE FROM xyz@{NO_FULL_SCAN} WHERE (y > 0 AND y < 1000) OR (y > 2000 AND y < 3000) RETURNING z
+----
+distribution: local
+vectorized: true
+·
+• project
+│ columns: (z)
+│ estimated row count: 990 (missing stats)
+│
+└── • delete
+    │ columns: (x, z)
+    │ estimated row count: 990 (missing stats)
+    │ from: xyz
+    │ auto commit
+    │
+    └── • index join
+        │ columns: (x, y, z)
+        │ estimated row count: 990 (missing stats)
+        │ table: xyz@primary
+        │ key columns: x
+        │
+        └── • scan
+              columns: (x, y)
+              estimated row count: 990 (missing stats)
+              table: xyz@xyz_y_idx
+              spans: /1-/1000 /2001-/3000