Better cardinality estimation for queries with many predicates (especially those not matched to any indices)

src/jrd/optimizer/InnerJoin.cpp

@@ -108,7 +108,6 @@ void InnerJoin::calculateStreamInfo()
 		innerStream->baseIndexes = candidate->indexes;
 		innerStream->baseUnique = candidate->unique;
 		innerStream->baseNavigated = candidate->navigated;
-		innerStream->baseConjuncts = candidate->conjuncts;
 
 		csb->csb_rpt[innerStream->number].deactivate();
 	}

src/jrd/optimizer/Optimizer.cpp

@@ -1218,6 +1218,39 @@ double Optimizer::getDependentSelectivity()
 }
 
 
+//
+// Estimate overall selectivity for a list of conjuncts.
+// Booleans are usually inter-dependent in practice and simple multiplication results to a very low selectivity value,
+// thus causing the stream cardinality being under-estimated. To avoid this, apply exponential backoff adjustment.
+// See also explanation in the middle of Retrieval::makeInversion().
+//
+
+double Optimizer::estimateSelectivity(const BooleanList& filters, double cardinality, unsigned priorConjuncts)
+{
+	// Get selectivities and order them
+	SortedArray<double, InlineStorage<double, OPT_STATIC_ITEMS> > selectivities;
+
+	for (const auto filter : filters)
+		selectivities.add(getSelectivity(filter));
+
+	auto selectivity = MAXIMUM_SELECTIVITY;
+
+	if (selectivities.hasData() && !priorConjuncts && cardinality)
+	{
+		// If the table is small enough, the hardcoded selectivity factors are causing
+		// too small resulting selectivity. Adjust the initial value to protect from this case.
+		const auto minSelectivity = MAXIMUM_SELECTIVITY / cardinality;
+		selectivity *= minSelectivity / selectivities.front();
+	}
+
+	// Apply exponential backoff
+	for (auto factor : selectivities)
+		selectivity *= applyBackoff(factor, priorConjuncts++);
+
+	return selectivity;
+}
+
+
 //
 // Prepare relation and its indices for optimization
 //
@@ -2970,6 +3003,7 @@ RecordSource* Optimizer::generateRetrieval(StreamType stream,
 	BoolExprNode* condition = nullptr;
 	Array<DbKeyRangeNode*> dbkeyRanges;
 	double scanSelectivity = MAXIMUM_SELECTIVITY;
+	double filterSelectivity = MAXIMUM_SELECTIVITY;
 
 	if (relation()->getExtFile())
 	{
@@ -3020,14 +3054,14 @@ RecordSource* Optimizer::generateRetrieval(StreamType stream,
 		// Persistent table
 		Retrieval retrieval(tdbb, this, stream, outerFlag, innerFlag,
 							(sortClause ? *sortClause : nullptr), false);
-		const auto candidate = retrieval.getInversion();
 
-		if (candidate)
+		if (const auto candidate = retrieval.getInversion())
 		{
 			inversion = candidate->inversion;
 			condition = candidate->condition;
 			dbkeyRanges.assign(candidate->dbkeyRanges);
 			scanSelectivity = candidate->matchSelectivity;
+			filterSelectivity = candidate->filterSelectivity;
 
 			// Just for safety sake, this condition must be already checked
 			// inside OptimizerRetrieval::matchOnIndexes()
@@ -3066,8 +3100,8 @@ RecordSource* Optimizer::generateRetrieval(StreamType stream,
 	// booleans.  When one is found, roll it into a final boolean and mark
 	// it used. If a computable boolean didn't match against an index then
 	// mark the stream to denote unmatched booleans.
+	BooleanList filters;
 	BoolExprNode* boolean = nullptr;
-	double filterSelectivity = MAXIMUM_SELECTIVITY;
 
 	for (auto iter = getConjuncts(outerFlag, innerFlag); iter.hasData(); ++iter)
 	{
@@ -3093,12 +3127,16 @@ RecordSource* Optimizer::generateRetrieval(StreamType stream,
 				}
 
 				if (!(iter & (CONJUNCT_MATCHED | CONJUNCT_JOINED)))
-					filterSelectivity *= getSelectivity(*iter);
+					filters.add(*iter);
 			}
 		}
 	}
 
-	if (!rsb)
+	if (rsb)
+	{
+		filterSelectivity = Optimizer::estimateSelectivity(filters, rsb->getCardinality());
+	}
+	else
 	{
 		if (inversion && condition)
 		{
@@ -3134,9 +3172,13 @@ RecordSource* Optimizer::generateRetrieval(StreamType stream,
 
 RecordSource* Optimizer::applyBoolean(RecordSource* rsb, ConjunctIterator& iter)
 {
-	double selectivity = MAXIMUM_SELECTIVITY;
-	if (const auto boolean = composeBoolean(iter, &selectivity))
+	BooleanList filters;
+
+	if (const auto boolean = composeBoolean(iter, filters))
+	{
+		const auto selectivity = estimateSelectivity(filters, rsb->getCardinality());
 		rsb = FB_NEW_POOL(getPool()) FilteredStream(csb, rsb, boolean, selectivity);
+	}
 
 	return rsb;
 }
@@ -3167,8 +3209,8 @@ RecordSource* Optimizer::applyLocalBoolean(RecordSource* rsb,
 
 RecordSource* Optimizer::applyResidualBoolean(RecordSource* rsb)
 {
+	BooleanList filters;
 	BoolExprNode* boolean = nullptr;
-	double selectivity = MAXIMUM_SELECTIVITY;
 
 	for (auto iter = getBaseConjuncts(); iter.hasData(); ++iter)
 	{
@@ -3178,15 +3220,17 @@ RecordSource* Optimizer::applyResidualBoolean(RecordSource* rsb)
 			iter |= CONJUNCT_USED;
 
 			if (!(iter & (CONJUNCT_MATCHED | CONJUNCT_JOINED)))
-				selectivity *= getSelectivity(*iter);
+				filters.add(*iter);
 		}
 	}
 
+	const auto selectivity = estimateSelectivity(filters, rsb->getCardinality());
+
 	return boolean ? FB_NEW_POOL(getPool()) FilteredStream(csb, rsb, boolean, selectivity) : rsb;
 }
 
 
-BoolExprNode* Optimizer::composeBoolean(ConjunctIterator& iter, double* selectivity)
+BoolExprNode* Optimizer::composeBoolean(ConjunctIterator& iter, BooleanList& filters)
 {
 	BoolExprNode* boolean = nullptr;
 
@@ -3199,8 +3243,8 @@ BoolExprNode* Optimizer::composeBoolean(ConjunctIterator& iter, double* selectiv
 			compose(getPool(), &boolean, iter);
 			iter |= CONJUNCT_USED;
 
-			if (!(iter & (CONJUNCT_MATCHED | CONJUNCT_JOINED)) && selectivity)
-				*selectivity *= getSelectivity(*iter);
+			if (!(iter & (CONJUNCT_MATCHED | CONJUNCT_JOINED)))
+				filters.add(*iter);
 		}
 	}
 

src/jrd/optimizer/Optimizer.h

@@ -43,6 +43,8 @@
 #include "../jrd/Statement.h"
 #include "../jrd/recsrc/RecordSource.h"
 
+#include <cmath>
+
 namespace Jrd {
 
 // AB: 2005-11-05
@@ -84,6 +86,9 @@ class SortNode;
 class River;
 class SortedStream;
 
+// List of booleans
+typedef Firebird::HalfStaticArray<BoolExprNode*, OPT_STATIC_ITEMS> BooleanList;
+
 
 //
 // River
@@ -275,7 +280,7 @@ class Optimizer final : public Firebird::PermanentStorage
 	{
 		// Conjunctions and their options
 		BoolExprNode* node;
-		unsigned flags;
+		unsigned flags = 0;
 	};
 
 	static constexpr unsigned CONJUNCT_USED		= 1;	// conjunct is used
@@ -445,23 +450,13 @@ class Optimizer final : public Firebird::PermanentStorage
 			}
 		}
 
-		// dimitr:
-		//
-		// Adjust to values similar to those used when the index selectivity is missing.
-		// The final value will be in the range [0.1 .. 0.5] that also matches the v3/v4 logic.
-		// This estimation is quite pessimistic but it seems to work better in practice,
-		// especially when multiple unmatchable booleans are used.
-
-		constexpr auto adjustment = DEFAULT_SELECTIVITY / REDUCE_SELECTIVITY_FACTOR_EQUALITY;
-		const auto selectivity = factor * adjustment;
+		if (!factor)
+			factor = DEFAULT_SELECTIVITY;
 
-		return MIN(selectivity, MAXIMUM_SELECTIVITY / 2);
+		return MIN(factor, MAXIMUM_SELECTIVITY);
 	}
 
-	static void adjustSelectivity(double& selectivity, double factor) noexcept
-	{
-		selectivity = MIN(selectivity * factor, MAXIMUM_SELECTIVITY);
-	}
+	static double estimateSelectivity(const BooleanList& filters, double cardinality = 0, unsigned priorConjuncts = 0);
 
 	double getDependentSelectivity();
 
@@ -485,6 +480,14 @@ class Optimizer final : public Firebird::PermanentStorage
 		return false;
 	}
 
+	static double applyBackoff(double selectivity, unsigned priorConjuncts)
+	{
+		for (unsigned i = 0; i < priorConjuncts; i++)
+			selectivity = std::sqrt(selectivity);
+
+		return selectivity;
+	}
+
 	static RecordSource* compile(thread_db* tdbb, CompilerScratch* csb, RseNode* rse)
 	{
 		bool firstRows = false;
@@ -560,13 +563,13 @@ class Optimizer final : public Firebird::PermanentStorage
 									ConjunctIterator& iter);
 	RecordSource* applyResidualBoolean(RecordSource* rsb);
 
-	BoolExprNode* composeBoolean(ConjunctIterator& iter,
-								 double* selectivity = nullptr);
+	BoolExprNode* composeBoolean(ConjunctIterator& iter, BooleanList& filters);
 
-	BoolExprNode* composeBoolean(double* selectivity = nullptr)
+	BoolExprNode* composeBoolean()
 	{
+		BooleanList filters;
 		auto iter = getBaseConjuncts();
-		return composeBoolean(iter, selectivity);
+		return composeBoolean(iter, filters);
 	}
 
 	bool checkEquiJoin(BoolExprNode* boolean);
@@ -653,8 +656,6 @@ enum segmentScanType {
 	segmentScanList
 };
 
-typedef Firebird::HalfStaticArray<BoolExprNode*, OPT_STATIC_ITEMS> BooleanList;
-
 struct IndexScratchSegment
 {
 	explicit IndexScratchSegment(MemoryPool& p)
@@ -714,11 +715,12 @@ typedef Firebird::ObjectsArray<IndexScratch> IndexScratchList;
 struct InversionCandidate
 {
 	explicit InversionCandidate(MemoryPool& p)
-		: conjuncts(p), matches(p), dbkeyRanges(p), dependentFromStreams(p)
+		: matches(p), filters(p), dbkeyRanges(p), dependentFromStreams(p)
 	{}
 
 	double selectivity = MAXIMUM_SELECTIVITY;
 	double matchSelectivity = MAXIMUM_SELECTIVITY;
+	double filterSelectivity = MAXIMUM_SELECTIVITY;
 	double cost = 0;
 	unsigned nonFullMatchedSegments = MAX_INDEX_SEGMENTS + 1;
 	unsigned matchedSegments = 0;
@@ -732,10 +734,19 @@ struct InversionCandidate
 	bool unique = false;
 	bool navigated = false;
 
-	BooleanList conjuncts;							// booleans referring our stream
 	BooleanList matches;							// booleans matched to any index
+	BooleanList filters;							// unmatched booleans referring our stream
 	Firebird::Array<DbKeyRangeNode*> dbkeyRanges;
 	SortedStreamList dependentFromStreams;
+
+	void applyFilters(double cardinality)
+	{
+		fb_assert(selectivity == matchSelectivity);
+		fb_assert(filterSelectivity == MAXIMUM_SELECTIVITY);
+		const auto matchCount = (unsigned) matches.getCount();
+		filterSelectivity = Optimizer::estimateSelectivity(filters, cardinality, matchCount);
+		selectivity *= filterSelectivity;
+	}
 };
 
 typedef Firebird::HalfStaticArray<InversionCandidate*, OPT_STATIC_ITEMS> InversionCandidateList;
@@ -865,7 +876,7 @@ class InnerJoin final : private Firebird::PermanentStorage
 	{
 	public:
 		StreamInfo(MemoryPool& p, StreamType num)
-			: number(num), baseConjuncts(p), indexedRelationships(p)
+			: number(num), indexedRelationships(p)
 		{}
 
 		bool isIndependent() const noexcept
@@ -912,7 +923,6 @@ class InnerJoin final : private Firebird::PermanentStorage
 		bool used = false;
 		unsigned previousExpectedStreams = 0;
 
-		BooleanList baseConjuncts;
 		IndexedRelationships indexedRelationships;
 	};