Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add runtime exclusion for ANY using subselect #4556

Merged
merged 1 commit into from Aug 25, 2022
Merged

Add runtime exclusion for ANY using subselect #4556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
36 changes: 31 additions & 5 deletions src/nodes/chunk_append/chunk_append.c
View file
Open in desktop
Expand Up @@ -114,10 +114,25 @@ ts_chunk_append_path_create(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht,
{
ListCell *lc_var;

/*
* check the param references a partitioning column of the hypertable
* otherwise we skip runtime exclusion
/* We have two types of exclusion:
*
* Parent exclusion fires if the entire hypertable can be excluded.
* This happens if doing things like joining against a parameter
* value that is an empty array or NULL. It doesn't happen often,
* but when it does, it speeds up the query immensely. It's also cheap
* to check for this condition as you check this once per hypertable
* at runtime.
*
* Child exclusion works by seeing if there is a contradiction between
* the chunks constraints and the expression on parameter values. For example,
* it can evaluate whether a time parameter from a subquery falls outside
* the range of the chunk. It is more widely applicable than the parent
* exclusion but is also more expensive to evaluate since you have to perform
* the check on every chunk. Child exclusion can only apply if one of the quals
* involves a partitioning column.
*
*/
path->runtime_exclusion_parent = true;
foreach (lc_var, pull_var_clause((Node *) rinfo->clause, 0))
{
Var *var = lfirst(lc_var);
Expand All @@ -130,12 +145,22 @@ ts_chunk_append_path_create(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht,
if (var->varno == rel->relid && var->varattno > 0 &&
ts_is_partitioning_column(ht, var->varattno))
{
path->runtime_exclusion = true;
path->runtime_exclusion_children = true;
break;
}
}
}
}
/*
* Our strategy is to use child exclusion if possible (if a partitioning
* column is used) and fall back to parent exclusion if we can't use child
* exclusion. Please note: there is no point to using both child and parent
* exclusion at the same time since child exclusion would always exclude
* the same chunks that parent exclusion would.
*/

if (path->runtime_exclusion_parent && path->runtime_exclusion_children)
path->runtime_exclusion_parent = false;

/*
* Make sure our subpath is either an Append or MergeAppend node
Expand Down Expand Up @@ -253,7 +278,8 @@ ts_chunk_append_path_create(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht,
if (!has_scan_childs)
{
path->startup_exclusion = false;
path->runtime_exclusion = false;
path->runtime_exclusion_parent = false;
path->runtime_exclusion_children = false;
}

path->cpath.custom_paths = nested_children;
Expand Down
3 changes: 2 additions & 1 deletion src/nodes/chunk_append/chunk_append.h
View file
Open in desktop
Expand Up @@ -15,7 +15,8 @@ typedef struct ChunkAppendPath
{
CustomPath cpath;
bool startup_exclusion;
bool runtime_exclusion;
bool runtime_exclusion_parent;
bool runtime_exclusion_children;
bool pushdown_limit;
int limit_tuples;
int first_partial_path;
Expand Down
132 changes: 94 additions & 38 deletions src/nodes/chunk_append/exec.c
View file
Open in desktop
Expand Up @@ -52,7 +52,8 @@ typedef struct ChunkAppendState

Oid ht_reloid;
bool startup_exclusion;
bool runtime_exclusion;
bool runtime_exclusion_parent;
bool runtime_exclusion_children;
bool runtime_initialized;
uint32 limit;

Expand All @@ -62,6 +63,8 @@ typedef struct ChunkAppendState
List *initial_constraints;
/* list of restrictinfo clauses indexed like initial_subplans */
List *initial_ri_clauses;
/* List of restrictinfo clauses on the parent hypertable */
List *initial_parent_clauses;

/* list of subplans after startup exclusion */
List *filtered_subplans;
Expand All @@ -79,7 +82,8 @@ typedef struct ChunkAppendState

/* number of loops and exclusions for EXPLAIN */
int runtime_number_loops;
int runtime_number_exclusions;
int runtime_number_exclusions_parent;
int runtime_number_exclusions_children;

LWLock *lock;
ParallelContext *pcxt;
Expand Down Expand Up @@ -140,11 +144,13 @@ ts_chunk_append_state_create(CustomScan *cscan)
state->initial_subplans = cscan->custom_plans;
state->initial_ri_clauses = lsecond(cscan->custom_private);
state->sort_options = lfourth(cscan->custom_private);
state->initial_parent_clauses = lfirst(list_nth_cell(cscan->custom_private, 4));

state->startup_exclusion = (bool) linitial_int(settings);
state->runtime_exclusion = (bool) lsecond_int(settings);
state->limit = lthird_int(settings);
state->first_partial_plan = lfourth_int(settings);
state->runtime_exclusion_parent = (bool) lsecond_int(settings);
state->runtime_exclusion_children = (bool) lthird_int(settings);
state->limit = lfourth_int(settings);
state->first_partial_plan = lfirst_int(list_nth_cell(settings, 4));

state->filtered_subplans = state->initial_subplans;
state->filtered_ri_clauses = state->initial_ri_clauses;
Expand Down Expand Up @@ -225,10 +231,10 @@ do_startup_exclusion(ChunkAppendState *state)
}

/*
* if this node does runtime exclusion we keep the constified
* if this node does runtime exclusion on the children we keep the constified
* expressions to save us some work during runtime exclusion
*/
if (state->runtime_exclusion)
if (state->runtime_exclusion_children)
{
List *const_ri_clauses = NIL;
foreach (lc, restrictinfos)
Expand Down Expand Up @@ -319,7 +325,7 @@ chunk_append_begin(CustomScanState *node, EState *estate, int eflags)
i++;
}

if (state->runtime_exclusion)
if (state->runtime_exclusion_parent || state->runtime_exclusion_children)
{
state->params = state->subplanstates[0]->plan->allParam;
/*
Expand All @@ -329,6 +335,30 @@ chunk_append_begin(CustomScanState *node, EState *estate, int eflags)
}
}

static bool
can_exclude_constraints_using_clauses(ChunkAppendState *state, List *constraints, List *clauses,
PlannerInfo root, PlanState *ps)
{
bool can_exclude;
ListCell *lc;
MemoryContext old = MemoryContextSwitchTo(state->exclusion_ctx);
List *restrictinfos = NIL;

foreach (lc, clauses)
{
RestrictInfo *ri = makeNode(RestrictInfo);
ri->clause = lfirst(lc);
restrictinfos = lappend(restrictinfos, ri);
}
restrictinfos = constify_restrictinfo_params(&root, ps->state, restrictinfos);

can_exclude = can_exclude_chunk(constraints, restrictinfos);

MemoryContextReset(state->exclusion_ctx);
MemoryContextSwitchTo(old);
return can_exclude;
}

/*
* build bitmap of valid subplans for runtime exclusion
*/
Expand All @@ -345,61 +375,76 @@ initialize_runtime_exclusion(ChunkAppendState *state)
.glob = &glob,
};

Assert(state->num_subplans == list_length(state->filtered_ri_clauses));

lc_clauses = list_head(state->filtered_ri_clauses);
lc_constraints = list_head(state->filtered_constraints);
state->runtime_initialized = true;

if (state->num_subplans == 0)
{
state->runtime_initialized = true;
return;
}

state->runtime_number_loops++;

if (state->runtime_exclusion_parent)
{
/* try to exclude all the chunks using the parents clauses.
* here, all constraints are true but exclusion can still
* happen because of things like ANY(empty set), and NULL
* inference
*/
if (can_exclude_constraints_using_clauses(state,
list_make1(makeBoolConst(true, false)),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So here we're passing it a list of constraints that consists of a single true value, to effectively make it check whether the initial_parent_clauses evaluate to true, given the plan parameter?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, except that we are checking whether initial_parent_clauses are always false (not true) for exclusion. But yes this is simply checking the parent clauses for internal contradiction given the evaluated parameter values

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this mean that parent clauses should evaluate to constant false after constify_restrictinfo_params? I wonder if we can wrirte the check explicitly like this, constify_restrictinfo_params and then check that every element is a false Const.

I mean, I just spent half an hour at it, and I still can't tell what predicate_refuted_by does if you pass clause = False, predicate = Null, and weak = true. And this is a fallback used when there are more than one predicate, which apparently isn't covered by the new tests.

state->initial_parent_clauses,
root,
&state->csstate.ss.ps))
{
state->runtime_number_exclusions_parent++;
return;
}
}

if (!state->runtime_exclusion_children)
{
for (i = 0; i < state->num_subplans; i++)
{
state->valid_subplans = bms_add_member(state->valid_subplans, i);
}
return;
}

Assert(state->num_subplans == list_length(state->filtered_ri_clauses));

lc_clauses = list_head(state->filtered_ri_clauses);
lc_constraints = list_head(state->filtered_constraints);

/*
* mark subplans as active/inactive in valid_subplans
*/
for (i = 0; i < state->num_subplans; i++)
{
PlanState *ps = state->subplanstates[i];
Scan *scan = ts_chunk_append_get_scan_plan(ps->plan);
List *restrictinfos = NIL;
ListCell *lc;

if (scan == NULL || scan->scanrelid == 0)
{
state->valid_subplans = bms_add_member(state->valid_subplans, i);
}
else
{
bool can_exclude;
MemoryContext old = MemoryContextSwitchTo(state->exclusion_ctx);

foreach (lc, lfirst(lc_clauses))
{
RestrictInfo *ri = makeNode(RestrictInfo);
ri->clause = lfirst(lc);
restrictinfos = lappend(restrictinfos, ri);
}
restrictinfos = constify_restrictinfo_params(&root, ps->state, restrictinfos);

can_exclude = can_exclude_chunk(lfirst(lc_constraints), restrictinfos);

MemoryContextReset(state->exclusion_ctx);
MemoryContextSwitchTo(old);
bool can_exclude = can_exclude_constraints_using_clauses(state,
lfirst(lc_constraints),
lfirst(lc_clauses),
root,
ps);

if (!can_exclude)
state->valid_subplans = bms_add_member(state->valid_subplans, i);
else
state->runtime_number_exclusions++;
state->runtime_number_exclusions_children++;
}

lc_clauses = lnext_compat(state->filtered_ri_clauses, lc_clauses);
lc_constraints = lnext_compat(state->filtered_constraints, lc_constraints);
}

state->runtime_initialized = true;
}

/*
Expand Down Expand Up @@ -464,7 +509,7 @@ get_next_subplan(ChunkAppendState *state, int last_plan)
if (last_plan == NO_MATCHING_SUBPLANS)
return NO_MATCHING_SUBPLANS;

if (state->runtime_exclusion)
if (state->runtime_exclusion_parent || state->runtime_exclusion_children)
{
if (!state->runtime_initialized)
initialize_runtime_exclusion(state);
Expand Down Expand Up @@ -611,7 +656,8 @@ chunk_append_rescan(CustomScanState *node)
/*
* detect changed params and reset runtime exclusion state
*/
if (state->runtime_exclusion && bms_overlap(node->ss.ps.chgParam, state->params))
if ((state->runtime_exclusion_parent || state->runtime_exclusion_children) &&
bms_overlap(node->ss.ps.chgParam, state->params))
{
bms_free(state->valid_subplans);
state->valid_subplans = NULL;
Expand Down Expand Up @@ -786,6 +832,8 @@ constify_param_mutator(Node *node, void *context)
{
ExprContext *econtext = GetPerTupleExprContext(estate);
ExecSetParamPlan(prm.execPlan, econtext);
// reload prm as it may have been changed by ExecSetParamPlan call above.
prm = estate->es_param_exec_vals[param->paramid];
}

if (prm.execPlan == NULL)
Expand Down Expand Up @@ -1010,17 +1058,25 @@ chunk_append_explain(CustomScanState *node, List *ancestors, ExplainState *es)
ExplainPropertyBool("Startup Exclusion", state->startup_exclusion, es);

if (es->verbose || es->format != EXPLAIN_FORMAT_TEXT)
ExplainPropertyBool("Runtime Exclusion", state->runtime_exclusion, es);
ExplainPropertyBool("Runtime Exclusion",
(state->runtime_exclusion_parent || state->runtime_exclusion_children),
es);

if (state->startup_exclusion)
ExplainPropertyInteger("Chunks excluded during startup",
NULL,
list_length(state->initial_subplans) - list_length(node->custom_ps),
es);

if (state->runtime_exclusion && state->runtime_number_loops > 0)
if (state->runtime_exclusion_parent && state->runtime_number_loops > 0)
{
int avg_excluded = state->runtime_number_exclusions_parent / state->runtime_number_loops;
ExplainPropertyInteger("Hypertables excluded during runtime", NULL, avg_excluded, es);
}

if (state->runtime_exclusion_children && state->runtime_number_loops > 0)
{
int avg_excluded = state->runtime_number_exclusions / state->runtime_number_loops;
int avg_excluded = state->runtime_number_exclusions_children / state->runtime_number_loops;
ExplainPropertyInteger("Chunks excluded during runtime", NULL, avg_excluded, es);
}
}
Expand Down
20 changes: 17 additions & 3 deletions src/nodes/chunk_append/planner.c
View file
Open in desktop
Expand Up @@ -89,6 +89,7 @@ ts_chunk_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path
List *clauses, List *custom_plans)
{
ListCell *lc_child;
List *parent_clauses = NIL;
List *chunk_ri_clauses = NIL;
List *chunk_rt_indexes = NIL;
List *sort_options = NIL;
Expand Down Expand Up @@ -244,7 +245,7 @@ ts_chunk_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path
* If we do either startup or runtime exclusion, we need to pass restrictinfo
* clauses into executor.
*/
if (capath->startup_exclusion || capath->runtime_exclusion)
if (capath->startup_exclusion || capath->runtime_exclusion_children)
{
foreach (lc_child, cscan->custom_plans)
{
Expand Down Expand Up @@ -272,20 +273,33 @@ ts_chunk_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path
chunk_rt_indexes = lappend_oid(chunk_rt_indexes, scan->scanrelid);
}
}

Assert(list_length(cscan->custom_plans) == list_length(chunk_ri_clauses));
Assert(list_length(chunk_ri_clauses) == list_length(chunk_rt_indexes));
}

/* pass down the parent clauses if doing parent exclusion */
if (capath->runtime_exclusion_parent)
{
ListCell *lc;
foreach (lc, clauses)
{
parent_clauses = lappend(parent_clauses, castNode(RestrictInfo, lfirst(lc))->clause);
}
}

if (capath->pushdown_limit && capath->limit_tuples > 0)
limit = capath->limit_tuples;

custom_private = list_make1(list_make4_int(capath->startup_exclusion,
capath->runtime_exclusion,
custom_private = list_make1(list_make5_int(capath->startup_exclusion,
capath->runtime_exclusion_parent,
capath->runtime_exclusion_children,
limit,
capath->first_partial_path));
custom_private = lappend(custom_private, chunk_ri_clauses);
custom_private = lappend(custom_private, chunk_rt_indexes);
custom_private = lappend(custom_private, sort_options);
custom_private = lappend(custom_private, parent_clauses);

cscan->custom_private = custom_private;

Expand Down