From 1b6a261ff002551498526ff7c5715b3973aaec98 Mon Sep 17 00:00:00 2001 From: Alena0704 Date: Tue, 23 Nov 2021 11:45:53 +0300 Subject: [PATCH 1/3] add query_hash as getting query_id --- aqo.c | 1 + preprocessing.c | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/aqo.c b/aqo.c index bd40b3dc..6f48196a 100644 --- a/aqo.c +++ b/aqo.c @@ -147,6 +147,7 @@ _PG_init(void) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("AQO module could be loaded only on startup."), errdetail("Add 'aqo' into the shared_preload_libraries list."))); + EnableQueryId(); DefineCustomEnumVariable("aqo.mode", "Mode of aqo usage.", diff --git a/preprocessing.c b/preprocessing.c index 30d6431c..e689329f 100644 --- a/preprocessing.c +++ b/preprocessing.c @@ -209,7 +209,17 @@ aqo_planner(Query *parse, } selectivity_cache_clear(); - query_context.query_hash = get_query_hash(parse, query_string); + /* + * TODO: this part of code ought to be clarified to clear understanding of + * such situation. + */ + if (parse->queryId == UINT64CONST(0)) + JumbleQuery(parse, query_string); + + Assert(parse->utilityStmt == NULL); + Assert(parse->queryId != UINT64CONST(0)); + + query_context.query_hash = (int) parse->queryId; if (query_is_deactivated(query_context.query_hash) || list_member_uint64(cur_classes,query_context.query_hash)) From b735d33f374092d72ae9167ecbbfc3603056db4d Mon Sep 17 00:00:00 2001 From: Alena0704 Date: Wed, 1 Dec 2021 15:05:38 +0300 Subject: [PATCH 2/3] add test for joint use pg_stat_statements and aqo --- Makefile | 5 ++- expected/aqo_fdw.out | 9 +++-- expected/gucs.out | 6 ++- t/002_stat_statements_with_aqo.pl | 65 +++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 t/002_stat_statements_with_aqo.pl diff --git a/Makefile b/Makefile index aedca207..70855867 100755 --- a/Makefile +++ b/Makefile @@ -26,9 +26,10 @@ REGRESS = aqo_disabled \ top_queries fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw -PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir) +stat_srcdir = $(top_srcdir)/contrib/pg_stat_statements +PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir) -I$(stat_srcdir) EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add -EXTRA_INSTALL = contrib/postgres_fdw +EXTRA_INSTALL = contrib/postgres_fdw contrib/pg_stat_statements DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql \ aqo--1.2--1.3.sql diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index 23cd2f3f..57214802 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -58,10 +58,11 @@ SELECT x FROM frgn WHERE x < 10; AQO not used Output: x Remote SQL: SELECT x FROM public.local WHERE ((x < 10)) + Query Identifier: -8486632260196378360 Using aqo: true AQO mode: LEARN JOINS: 0 -(7 rows) +(8 rows) EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants @@ -108,10 +109,11 @@ SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; Output: a.x, b.x Relations: (public.frgn a) INNER JOIN (public.frgn b) Remote SQL: SELECT r1.x, r2.x FROM (public.local r1 INNER JOIN public.local r2 ON (((r1.x = r2.x)))) + Query Identifier: -3803571735866573870 Using aqo: true AQO mode: LEARN JOINS: 0 -(8 rows) +(9 rows) -- TODO: Non-mergejoinable join condition. EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) @@ -135,10 +137,11 @@ SELECT * FROM frgn AS a, frgn AS b WHERE a.x 3; +print "start"; +my $node = PostgreSQL::Test::Cluster->new('profiling'); +$node->init; +print "create conf"; + +$node->append_conf('postgresql.conf', qq{ + aqo.mode = 'disabled' + aqo.profile_classes = -1 + aqo.profile_enable = 'true' + aqo.force_collect_stat = 'false' + aqo.log_ignorance = 'off' + log_statement = 'ddl' # reduce size of logs. + }); +# Test constants. +my $TRANSACTIONS = 100; +my $CLIENTS = 10; +my $THREADS = 10; +my $query_id; + +# General purpose variables. +my $res; +my $total_classes; +$node->start(); + # ERROR: AQO allow to load library only on startup +print "create extantion aqo"; +$node->psql('postgres', "CREATE EXTENSION aqo"); +$node->psql('postgres', "CREATE EXTENSION pg_stat_statements"); +print "create preload libraries"; +$node->append_conf('postgresql.conf', qq{shared_preload_libraries = 'aqo, pg_stat_statements'}); +$node->restart(); +$node->psql('postgres', "CREATE EXTENSION aqo"); +$node->psql('postgres', "CREATE EXTENSION pg_stat_statements"); +$node->psql('postgres', " + ALTER SYSTEM SET aqo.profile_enable = 'true'; + SELECT pg_reload_conf(); +"); + +$node->psql('postgres', "CREATE TABLE aqo_test0(a int, b int, c int, d int); +WITH RECURSIVE t(a, b, c, d) +AS ( + VALUES (0, 0, 0, 0) + UNION ALL + SELECT t.a + 1, t.b + 1, t.c + 1, t.d + 1 FROM t WHERE t.a < 2000 +) INSERT INTO aqo_test0 (SELECT * FROM t); +CREATE INDEX aqo_test0_idx_a ON aqo_test0 (a); +ANALYZE aqo_test0;"); +$node->psql('postgres', " + ALTER SYSTEM SET aqo.mode = 'controlled'; +"); +$res = $node->safe_psql('postgres', "SELECT * FROM aqo_test0"); +$res = $node->safe_psql('postgres', "SELECT count(*) FROM pg_stat_statements where query = 'SELECT * FROM aqo_test0'"); +is($res, 1); # The same query add in pg_stat_statements +$res = $node->safe_psql('postgres', "SELECT count(*) from aqo_query_texts where query_text = 'SELECT * FROM aqo_test0'"); +is($res, 0); +$query_id = $node->safe_psql('postgres', "SELECT queryid FROM pg_stat_statements where query = 'SELECT * FROM aqo_test0'"); +$res = $node->safe_psql('postgres', "insert into aqo_queries values ($query_id,'f','f',$query_id,'f')"); +$res = $node->safe_psql('postgres', "insert into aqo_query_texts values ($query_id,'SELECT * FROM aqo_test0')"); +$res = $node->safe_psql('postgres', "SELECT count(*) from aqo_query_texts where query_text = 'SELECT * FROM aqo_test0'"); +is($res, 1); +$node->stop(); \ No newline at end of file From 5e5450ed09732c334b108277ff2d6d851a36891f Mon Sep 17 00:00:00 2001 From: Alena0704 Date: Fri, 3 Dec 2021 07:12:23 +0300 Subject: [PATCH 3/3] add function for ignoring string with Query Identifier in aqo_gucs and aqo_fdw tests --- expected/aqo_fdw.out | 122 +++++++++++++++++++++++++++---------------- expected/gucs.out | 34 ++++++++---- sql/aqo_fdw.sql | 59 ++++++++++++++------- sql/gucs.sql | 25 +++++++-- 4 files changed, 164 insertions(+), 76 deletions(-) diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index 57214802..ec67743e 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -16,69 +16,102 @@ DO $d$ )$$; END; $d$; +-- +-- Returns string-by-string explain of a query. Made for removing some strings +-- from the explain output. +-- +CREATE OR REPLACE FUNCTION expln(query_string text default 'select * from table', verbose_p boolean default TRUE) RETURNS SETOF text AS $$ +BEGIN + IF verbose_p=TRUE THEN + RETURN QUERY EXECUTE format('EXPLAIN (ANALYZE, VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF) %s', query_string); + else + RETURN QUERY EXECUTE format('EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) %s', query_string); + END IF; + Return; +END; +$$ LANGUAGE PLPGSQL; CREATE USER MAPPING FOR PUBLIC SERVER loopback; CREATE TABLE local (x int); CREATE FOREIGN TABLE frgn(x int) SERVER loopback OPTIONS (table_name 'local'); INSERT INTO frgn (x) VALUES (1); ANALYZE local; -- Trivial foreign scan.s -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) -SELECT x FROM frgn; - QUERY PLAN ----------------------------------------------- - Foreign Scan on frgn (actual rows=1 loops=1) +SELECT str AS result +FROM expln('SELECT x FROM frgn;', TRUE) AS str +WHERE str NOT LIKE 'Query Identifier%'; + result +----------------------------------------------------- + Foreign Scan on public.frgn (actual rows=1 loops=1) AQO not used + Output: x + Remote SQL: SELECT x FROM public.local Using aqo: true AQO mode: LEARN JOINS: 0 -(5 rows) +(7 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) -SELECT x FROM frgn; - QUERY PLAN ----------------------------------------------- - Foreign Scan on frgn (actual rows=1 loops=1) +SELECT str AS result +FROM expln('SELECT x FROM frgn;', TRUE) AS str +WHERE str NOT LIKE 'Query Identifier%'; + result +----------------------------------------------------- + Foreign Scan on public.frgn (actual rows=1 loops=1) AQO: rows=1, error=0% + Output: x + Remote SQL: SELECT x FROM public.local Using aqo: true AQO mode: LEARN JOINS: 0 -(5 rows) +(7 rows) -- Push down base filters. Use verbose mode to see filters. -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)) -SELECT x FROM frgn WHERE x < 10; -ERROR: syntax error at or near ")" -LINE 1: ...LAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)) - ^ -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) -SELECT x FROM frgn WHERE x < 10; - QUERY PLAN +SELECT str AS result +FROM expln('SELECT x FROM frgn WHERE x < 10;', TRUE) AS str +WHERE str NOT LIKE 'Query Identifier%'; + result ----------------------------------------------------------- Foreign Scan on public.frgn (actual rows=1 loops=1) AQO not used Output: x Remote SQL: SELECT x FROM public.local WHERE ((x < 10)) - Query Identifier: -8486632260196378360 Using aqo: true AQO mode: LEARN JOINS: 0 -(8 rows) +(7 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) -SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants - QUERY PLAN ----------------------------------------------- - Foreign Scan on frgn (actual rows=0 loops=1) +SELECT str AS result +FROM expln('SELECT x FROM frgn WHERE x < 10;', TRUE) AS str +WHERE str NOT LIKE 'Query Identifier%'; + result +----------------------------------------------------------- + Foreign Scan on public.frgn (actual rows=1 loops=1) + AQO: rows=1, error=0% + Output: x + Remote SQL: SELECT x FROM public.local WHERE ((x < 10)) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(7 rows) + +SELECT str AS result +FROM expln('SELECT x FROM frgn WHERE x < -10;', TRUE) AS str +WHERE str NOT LIKE 'Query Identifier%'; -- AQO ignores constants + result +-------------------------------------------------------------- + Foreign Scan on public.frgn (actual rows=0 loops=1) AQO: rows=1, error=100% + Output: x + Remote SQL: SELECT x FROM public.local WHERE ((x < (-10))) Using aqo: true AQO mode: LEARN JOINS: 0 -(5 rows) +(7 rows) -- Trivial JOIN push-down. -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) -SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; - QUERY PLAN +SELECT str AS result +FROM expln('SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x;', FALSE) AS str +WHERE str NOT LIKE 'Query Identifier%'; + result ------------------------------------------------------------ Merge Join (actual rows=1 loops=1) AQO not used @@ -100,25 +133,26 @@ SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; JOINS: 0 (18 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) -SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; - QUERY PLAN +SELECT str AS result +FROM expln('SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x;', TRUE) AS str +WHERE str NOT LIKE 'Query Identifier%'; + result -------------------------------------------------------------------------------------------------------- Foreign Scan (actual rows=1 loops=1) AQO: rows=1, error=0% Output: a.x, b.x Relations: (public.frgn a) INNER JOIN (public.frgn b) Remote SQL: SELECT r1.x, r2.x FROM (public.local r1 INNER JOIN public.local r2 ON (((r1.x = r2.x)))) - Query Identifier: -3803571735866573870 Using aqo: true AQO mode: LEARN JOINS: 0 -(9 rows) +(8 rows) -- TODO: Non-mergejoinable join condition. -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) -SELECT * FROM frgn AS a, frgn AS b WHERE a.x