Skip to content

Commit 147ff71

Browse files
author
Alena Rybakina
committed
Add smart statement timeout for learning AQO.
AQO evaluates to have an enough time for training by the average integral error. If the integral error hasn't changed for comparing with error from the previous iteration, we increase the training time exponentially. Start value of smart statement timeout as equal as aqo_statement timeout or 0. If the user needs an upper limit on query execution time, he can set a vanilla statement timeout setting aqo_statement_timeout as 0. Smart statement timeout in this option is unenabled. The aqo.statement_timeout value is stored in aqo_queries. Initially it equals as 0 (default value). The user has the ability to specify the GUC aqo.statement_timeout and terminates the request by analogy with statement timeout.
1 parent 081c6a5 commit 147ff71

12 files changed

+270
-10
lines changed

‎aqo--1.3--1.4.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
\echo Use "ALTER EXTENSION aqo UPDATE TO '1.4'" to load this file. \quit
55

66
ALTER TABLE public.aqo_data ADD COLUMN reliability double precision [];
7+
ALTER TABLE public.aqo_queries ADD COLUMN smart_timeout bigint;
8+
ALTER TABLE public.aqo_queries ADD COLUMN count_increase_timeout bigint;
79

810
DROP FUNCTION public.top_error_queries(int);
911

‎aqo--1.4--1.5.sql‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ CREATE FUNCTION aqo_queries (
3030
OUT fs bigint,
3131
OUT learn_aqo boolean,
3232
OUT use_aqo boolean,
33-
OUT auto_tuning boolean
33+
OUT auto_tuning boolean,
34+
OUT smart_timeout bigint,
35+
OUT count_increase_timeout bigint
3436
)
3537
RETURNS SETOF record
3638
AS 'MODULE_PATHNAME', 'aqo_queries'

‎aqo.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ void _PG_init(void);
3434
/* Strategy of determining feature space for new queries. */
3535
int aqo_mode = AQO_MODE_CONTROLLED;
3636
bool force_collect_stat;
37+
int aqo_statement_timeout;
3738

3839
/*
3940
* Show special info in EXPLAIN mode.
@@ -47,6 +48,7 @@ bool force_collect_stat;
4748
*/
4849
bool aqo_show_hash;
4950
bool aqo_show_details;
51+
bool change_flex_timeout;
5052

5153
/* GUC variables */
5254
static const struct config_enum_entry format_options[] = {
@@ -305,6 +307,17 @@ _PG_init(void)
305307
NULL,
306308
NULL
307309
);
310+
DefineCustomIntVariable("aqo.statement_timeout",
311+
"Time limit on learning.",
312+
NULL,
313+
&aqo_statement_timeout,
314+
0,
315+
0, INT_MAX,
316+
PGC_USERSET,
317+
0,
318+
NULL,
319+
NULL,
320+
NULL);
308321

309322
prev_shmem_startup_hook = shmem_startup_hook;
310323
shmem_startup_hook = aqo_init_shmem;

‎aqo.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,8 +199,15 @@ typedef struct QueryContextData
199199

200200
instr_time start_execution_time;
201201
double planning_time;
202+
int64 smart_timeout;
203+
int64 count_increase_timeout;
202204
} QueryContextData;
203205

206+
/*
207+
* Indicator for using smart statement timeout for query
208+
*/
209+
extern bool change_flex_timeout;
210+
204211
struct StatEntry;
205212

206213
extern double predicted_ppi_rows;
@@ -249,6 +256,7 @@ extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook;
249256
extern ExplainOneNode_hook_type prev_ExplainOneNode_hook;
250257

251258
extern void ppi_hook(ParamPathInfo *ppi);
259+
extern int aqo_statement_timeout;
252260

253261
/* Hash functions */
254262
void get_eclasses(List *clauselist, int *nargs, int **args_hash,
@@ -297,5 +305,8 @@ extern void selectivity_cache_clear(void);
297305

298306
extern bool IsQueryDisabled(void);
299307

308+
extern bool update_query_timeout(uint64 queryid, int64 smart_timeout);
309+
extern double get_mean(double *elems, int nelems);
310+
300311
extern List *cur_classes;
301312
#endif

‎auto_tuning.c‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,15 @@
2727
*/
2828
double auto_tuning_convergence_error = 0.01;
2929

30-
static double get_mean(double *elems, int nelems);
3130
static double get_estimation(double *elems, int nelems);
3231
static bool is_stable(double *elems, int nelems);
3332
static bool converged_cq(double *elems, int nelems);
3433
static bool is_in_infinite_loop_cq(double *elems, int nelems);
3534

36-
3735
/*
3836
* Returns mean value of the array of doubles.
3937
*/
40-
static double
38+
double
4139
get_mean(double *elems, int nelems)
4240
{
4341
double sum = 0;

‎expected/smart_statement_timeout.out‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
DROP TABLE IF EXISTS a,b CASCADE;
2+
NOTICE: table "a" does not exist, skipping
3+
NOTICE: table "b" does not exist, skipping
4+
CREATE TABLE a (x1 int, x2 int, x3 int);
5+
INSERT INTO a (x1, x2, x3) SELECT mod(ival,4), mod(ival,10), mod(ival,10) FROM generate_series(1,100) As ival;
6+
CREATE TABLE b (y1 int, y2 int, y3 int);
7+
INSERT INTO b (y1, y2, y3) SELECT mod(ival + 1,4), mod(ival + 1,10), mod(ival + 1,10) FROM generate_series(1,100) As ival;
8+
CREATE EXTENSION IF NOT EXISTS aqo;
9+
SET aqo.join_threshold = 0;
10+
SET aqo.mode = 'learn';
11+
SET aqo.show_details = 'off';
12+
SET aqo.learn_statement_timeout = 'on';
13+
SET statement_timeout = 1500; -- [1.5s]
14+
SET aqo.statement_timeout = 500; -- [0.5s]
15+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
16+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. Timeout is 0
17+
NOTICE: [AQO] Time limit for execution of the statement was increased. Current timeout is 1
18+
count | count
19+
-------+-------
20+
62500 | 62500
21+
(1 row)
22+
23+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
24+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
25+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
26+
smart_timeout | count_increase_timeout
27+
---------------+------------------------
28+
1 | 1
29+
(1 row)
30+
31+
SET aqo.learn_statement_timeout = 'off';
32+
SET aqo.statement_timeout = 1000; -- [1s]
33+
INSERT INTO a (x1, x2, x3) SELECT mod(ival,20), mod(ival,10), mod(ival,10) FROM generate_series(1,1000) As ival;
34+
SET aqo.learn_statement_timeout = 'on';
35+
SET aqo.statement_timeout = 500; -- [0.5s]
36+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
37+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. Timeout is 1
38+
NOTICE: [AQO] Time limit for execution of the statement was increased. Current timeout is 6
39+
count | count
40+
--------+--------
41+
563300 | 562500
42+
(1 row)
43+
44+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
45+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
46+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
47+
smart_timeout | count_increase_timeout
48+
---------------+------------------------
49+
6 | 2
50+
(1 row)
51+
52+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
53+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. Timeout is 6
54+
NOTICE: [AQO] Time limit for execution of the statement was increased. Current timeout is 63
55+
count | count
56+
--------+--------
57+
563300 | 562500
58+
(1 row)
59+
60+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
61+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
62+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
63+
smart_timeout | count_increase_timeout
64+
---------------+------------------------
65+
63 | 3
66+
(1 row)
67+
68+
SET statement_timeout = 100; -- [0.1s]
69+
SET aqo.statement_timeout = 150;
70+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
71+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. Timeout is 63
72+
ERROR: canceling statement due to statement timeout
73+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
74+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
75+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
76+
smart_timeout | count_increase_timeout
77+
---------------+------------------------
78+
63 | 3
79+
(1 row)
80+
81+
SELECT 1 FROM aqo_reset();
82+
?column?
83+
----------
84+
1
85+
(1 row)
86+
87+
DROP TABLE a;
88+
DROP TABLE b;
89+
DROP EXTENSION aqo;

‎postprocessing.c‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ typedef struct
4444

4545
static double cardinality_sum_errors;
4646
static int cardinality_num_objects;
47+
static int64 max_timeout_value;
48+
static int64 growth_rate = 3;
4749

4850
/*
4951
* Store an AQO-related query data into the Query Environment structure.
@@ -625,15 +627,46 @@ aqo_timeout_handler(void)
625627
ctx.learn = query_context.learn_aqo;
626628
ctx.isTimedOut = true;
627629

628-
elog(NOTICE, "[AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data.");
630+
if (aqo_statement_timeout == 0)
631+
elog(NOTICE, "[AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data.");
632+
else
633+
elog(NOTICE, "[AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. Timeout is %ld", max_timeout_value);
634+
629635
learnOnPlanState(timeoutCtl.queryDesc->planstate, (void *) &ctx);
630636
MemoryContextSwitchTo(oldctx);
631637
}
632638

639+
/*
640+
* Function for updating smart statement timeout
641+
*/
642+
static int64
643+
increase_smart_timeout()
644+
{
645+
int64 smart_timeout_fin_time = (query_context.smart_timeout + 1) * pow(growth_rate, query_context.count_increase_timeout);
646+
647+
if (query_context.smart_timeout == max_timeout_value && !update_query_timeout(query_context.query_hash, smart_timeout_fin_time))
648+
elog(NOTICE, "[AQO] Timeout is not updated!");
649+
650+
return smart_timeout_fin_time;
651+
}
652+
633653
static bool
634654
set_timeout_if_need(QueryDesc *queryDesc)
635655
{
636-
TimestampTz fin_time;
656+
int64 fintime = (int64) get_timeout_finish_time(STATEMENT_TIMEOUT)-1;
657+
658+
if (aqo_learn_statement_timeout && aqo_statement_timeout > 0)
659+
{
660+
max_timeout_value = Min(query_context.smart_timeout, (int64) aqo_statement_timeout);
661+
if (max_timeout_value > fintime)
662+
{
663+
max_timeout_value = fintime;
664+
}
665+
}
666+
else
667+
{
668+
max_timeout_value = fintime;
669+
}
637670

638671
if (IsParallelWorker())
639672
/*
@@ -663,8 +696,7 @@ set_timeout_if_need(QueryDesc *queryDesc)
663696
else
664697
Assert(!get_timeout_active(timeoutCtl.id));
665698

666-
fin_time = get_timeout_finish_time(STATEMENT_TIMEOUT);
667-
enable_timeout_at(timeoutCtl.id, fin_time - 1);
699+
enable_timeout_at(timeoutCtl.id, (TimestampTz) max_timeout_value);
668700

669701
/* Save pointer to queryDesc to use at learning after a timeout interruption. */
670702
timeoutCtl.queryDesc = queryDesc;
@@ -720,6 +752,7 @@ aqo_ExecutorEnd(QueryDesc *queryDesc)
720752
instr_time endtime;
721753
EphemeralNamedRelation enr = get_ENR(queryDesc->queryEnv, PlanStateInfo);
722754
MemoryContext oldctx = MemoryContextSwitchTo(AQOLearnMemCtx);
755+
double error = .0;
723756

724757
cardinality_sum_errors = 0.;
725758
cardinality_num_objects = 0;
@@ -778,6 +811,16 @@ aqo_ExecutorEnd(QueryDesc *queryDesc)
778811
/* Store all learn data into the AQO service relations. */
779812
if (!query_context.adding_query && query_context.auto_tuning)
780813
automatical_query_tuning(query_context.query_hash, stat);
814+
815+
error = stat->est_error_aqo[stat->cur_stat_slot_aqo-1] - cardinality_sum_errors/(1 + cardinality_num_objects);
816+
817+
if ( aqo_learn_statement_timeout && aqo_statement_timeout > 0 && error >= 0.1)
818+
{
819+
int64 fintime = increase_smart_timeout();
820+
elog(NOTICE, "[AQO] Time limit for execution of the statement was increased. Current timeout is %ld", fintime);
821+
}
822+
823+
pfree(stat);
781824
}
782825
}
783826

‎preprocessing.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,8 @@ aqo_planner(Query *parse,
249249
elog(ERROR, "unrecognized mode in AQO: %d", aqo_mode);
250250
break;
251251
}
252+
query_context.count_increase_timeout = 0;
253+
query_context.smart_timeout = 0;
252254
}
253255
else /* Query class exists in a ML knowledge base. */
254256
{

‎regress_schedule‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@ test: unsupported
1212
test: clean_aqo_data
1313
test: parallel_workers
1414
test: plancache
15-
# Performance-dependent test. Can be ignored if executes in containers or on slow machines
15+
# Performance-dependent tests. Can be ignored if executes in containers or on slow machines
1616
ignore: statement_timeout
17+
ignore: smart_statement_timeout
1718
test: statement_timeout
1819
test: temp_tables
1920
test: top_queries
2021
test: relocatable
2122
test: look_a_like
2223
test: feature_subspace
24+
test: smart_statement_timeout

‎sql/smart_statement_timeout.sql‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
DROP TABLE IF EXISTS a,b CASCADE;
2+
CREATE TABLE a (x1 int, x2 int, x3 int);
3+
INSERT INTO a (x1, x2, x3) SELECT mod(ival,4), mod(ival,10), mod(ival,10) FROM generate_series(1,100) As ival;
4+
5+
CREATE TABLE b (y1 int, y2 int, y3 int);
6+
INSERT INTO b (y1, y2, y3) SELECT mod(ival + 1,4), mod(ival + 1,10), mod(ival + 1,10) FROM generate_series(1,100) As ival;
7+
8+
CREATE EXTENSION IF NOT EXISTS aqo;
9+
SET aqo.join_threshold = 0;
10+
SET aqo.mode = 'learn';
11+
SET aqo.show_details = 'off';
12+
SET aqo.learn_statement_timeout = 'on';
13+
SET statement_timeout = 1500; -- [1.5s]
14+
SET aqo.statement_timeout = 500; -- [0.5s]
15+
16+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
17+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
18+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
19+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
20+
21+
SET aqo.learn_statement_timeout = 'off';
22+
SET aqo.statement_timeout = 1000; -- [1s]
23+
INSERT INTO a (x1, x2, x3) SELECT mod(ival,20), mod(ival,10), mod(ival,10) FROM generate_series(1,1000) As ival;
24+
SET aqo.learn_statement_timeout = 'on';
25+
SET aqo.statement_timeout = 500; -- [0.5s]
26+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
27+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
28+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
29+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
30+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
31+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
32+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
33+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
34+
35+
SET statement_timeout = 100; -- [0.1s]
36+
SET aqo.statement_timeout = 150;
37+
SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;
38+
select smart_timeout, count_increase_timeout from aqo_queries, aqo_query_texts
39+
where query_text = 'SELECT count(a.x1),count(B.y1) FROM A a LEFT JOIN B ON a.x1 = B.y1 LEFT JOIN A a1 ON a1.x1 = B.y1;'
40+
and aqo_query_texts.queryid = aqo_queries.queryid limit 1;
41+
42+
SELECT 1 FROM aqo_reset();
43+
DROP TABLE a;
44+
DROP TABLE b;
45+
DROP EXTENSION aqo;

0 commit comments

Comments
 (0)