Skip to content

[SHRDM-762] extract info from foreign join #79

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 14, 2022
Merged

[SHRDM-762] extract info from foreign join #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9018a73
[SHRDM-762] extract info from foreign join
pyhalov Oct 7, 2022
65efa48
[SHRDM-762] add tests for partitioned tables with foreign partitions
pyhalov Oct 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 124 additions & 4 deletions expected/aqo_fdw.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,15 +110,15 @@ SELECT str FROM expln('
JOINS: 0
(6 rows)

-- TODO: Should learn on postgres_fdw nodes
-- Should learn on postgres_fdw nodes
SELECT str FROM expln('
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)
SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x;
') AS str WHERE str NOT LIKE '%Query Identifier%';
str
--------------------------------------------------------------------------------------------------------
Foreign Scan (actual rows=1 loops=1)
AQO not used
AQO: rows=1, error=0%
Output: a.x, b.x
Relations: (public.frgn a) INNER JOIN (public.frgn b)
Remote SQL: SELECT r1.x, r2.x FROM (public.local r1 INNER JOIN public.local r2 ON (((r1.x = r2.x))))
Expand All @@ -127,6 +127,122 @@ SELECT str FROM expln('
JOINS: 0
(8 rows)

CREATE TABLE local_a(aid int primary key, aval text);
CREATE TABLE local_b(bid int primary key, aid int references local_a(aid), bval text);
INSERT INTO local_a SELECT i, 'val_' || i FROM generate_series(1,100) i;
INSERT INTO local_b SELECT i, mod((i+random()*10)::numeric, 10) + 1, 'val_' || i FROM generate_series(1,1000) i;
ANALYZE local_a, local_b;
CREATE FOREIGN TABLE frgn_a(aid int, aval text) SERVER loopback OPTIONS (table_name 'local_a');
CREATE FOREIGN TABLE frgn_b(bid int, aid int, bval text) SERVER loopback OPTIONS (table_name 'local_b');
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from frgn_a AS a, frgn_b AS b
WHERE a.aid = b.aid AND b.bval like 'val%';
QUERY PLAN
-----------------------------------------------
Foreign Scan (actual rows=1000 loops=1)
AQO not used
Relations: (frgn_a a) INNER JOIN (frgn_b b)
Using aqo: true
AQO mode: LEARN
JOINS: 0
(6 rows)

EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from frgn_a AS a, frgn_b AS b
WHERE a.aid = b.aid AND b.bval like 'val%';
QUERY PLAN
-----------------------------------------------
Foreign Scan (actual rows=1000 loops=1)
AQO: rows=1000, error=0%
Relations: (frgn_a a) INNER JOIN (frgn_b b)
Using aqo: true
AQO mode: LEARN
JOINS: 0
(6 rows)

-- Partitioned join over foreign tables
set enable_partitionwise_join = on;
ALTER SERVER loopback OPTIONS (ADD fdw_tuple_cost '1.0');
CREATE TABLE local_main_p0(aid int, aval text);
CREATE TABLE local_main_p1(aid int, aval text);
CREATE TABLE main (aid int, aval text) PARTITION BY HASH(aid);
CREATE FOREIGN TABLE main_p0 PARTITION OF main FOR VALUES WITH (MODULUS 3, REMAINDER 0)
SERVER loopback OPTIONS (table_name 'local_main_p0');
CREATE FOREIGN TABLE main_p1 PARTITION OF main FOR VALUES WITH (MODULUS 3, REMAINDER 1)
SERVER loopback OPTIONS (table_name 'local_main_p1');
CREATE TABLE main_p2 PARTITION OF main FOR VALUES WITH (MODULUS 3, REMAINDER 2);
CREATE TABLE local_ref_p0(bid int, aid int, bval text);
CREATE TABLE local_ref_p1(bid int, aid int, bval text);
CREATE TABLE ref (bid int, aid int, bval text) PARTITION BY HASH(aid);
CREATE FOREIGN TABLE ref_p0 PARTITION OF ref FOR VALUES WITH (MODULUS 3, REMAINDER 0)
SERVER loopback OPTIONS (table_name 'local_ref_p0');
CREATE FOREIGN TABLE ref_p1 PARTITION OF ref FOR VALUES WITH (MODULUS 3, REMAINDER 1)
SERVER loopback OPTIONS (table_name 'local_ref_p1');
CREATE TABLE ref_p2 PARTITION OF ref FOR VALUES WITH (MODULUS 3, REMAINDER 2);
INSERT INTO main SELECT i, 'val_' || i FROM generate_series(1,100) i;
INSERT INTO ref SELECT i, mod(i, 10) + 1, 'val_' || i FROM generate_series(1,1000) i;
ANALYZE local_main_p0, local_main_p1, main_p2;
ANALYZE local_ref_p0, local_ref_p1, ref_p2;
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from main AS a, ref AS b
WHERE a.aid = b.aid AND b.bval like 'val%';
QUERY PLAN
--------------------------------------------------------------------
Append (actual rows=1000 loops=1)
AQO not used
-> Foreign Scan (actual rows=400 loops=1)
AQO not used
Relations: (main_p0 a_1) INNER JOIN (ref_p0 b_1)
-> Foreign Scan (actual rows=300 loops=1)
AQO not used
Relations: (main_p1 a_2) INNER JOIN (ref_p1 b_2)
-> Hash Join (actual rows=300 loops=1)
AQO not used
Hash Cond: (b_3.aid = a_3.aid)
-> Seq Scan on ref_p2 b_3 (actual rows=300 loops=1)
AQO not used
Filter: (bval ~~ 'val%'::text)
-> Hash (actual rows=38 loops=1)
Buckets: 1024 Batches: 1 Memory Usage: 10kB
-> Seq Scan on main_p2 a_3 (actual rows=38 loops=1)
AQO not used
Using aqo: true
AQO mode: LEARN
JOINS: 1
(21 rows)

EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from main AS a, ref AS b
WHERE a.aid = b.aid AND b.bval like 'val%';
QUERY PLAN
--------------------------------------------------------------------
Append (actual rows=1000 loops=1)
AQO not used
-> Foreign Scan (actual rows=400 loops=1)
AQO: rows=400, error=0%
Relations: (main_p0 a_1) INNER JOIN (ref_p0 b_1)
-> Foreign Scan (actual rows=300 loops=1)
AQO: rows=300, error=0%
Relations: (main_p1 a_2) INNER JOIN (ref_p1 b_2)
-> Hash Join (actual rows=300 loops=1)
AQO: rows=300, error=0%
Hash Cond: (b_3.aid = a_3.aid)
-> Seq Scan on ref_p2 b_3 (actual rows=300 loops=1)
AQO: rows=300, error=0%
Filter: (bval ~~ 'val%'::text)
-> Hash (actual rows=38 loops=1)
Buckets: 1024 Batches: 1 Memory Usage: 10kB
-> Seq Scan on main_p2 a_3 (actual rows=38 loops=1)
AQO: rows=38, error=0%
Using aqo: true
AQO mode: LEARN
JOINS: 1
(21 rows)

DROP TABLE main, local_main_p0, local_main_p1;
DROP TABLE ref, local_ref_p0, local_ref_p1;
ALTER SERVER loopback OPTIONS (DROP fdw_tuple_cost);
reset enable_partitionwise_join;
-- TODO: Non-mergejoinable join condition.
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * FROM frgn AS a, frgn AS b WHERE a.x<b.x;
Expand All @@ -147,7 +263,7 @@ SELECT str FROM expln('
str
--------------------------------------------------------------------------------------------------------
Foreign Scan (actual rows=0 loops=1)
AQO not used
AQO: rows=1, error=100%
Output: a.x, b.x
Relations: (public.frgn a) INNER JOIN (public.frgn b)
Remote SQL: SELECT r1.x, r2.x FROM (public.local r1 INNER JOIN public.local r2 ON (((r1.x < r2.x))))
Expand All @@ -158,8 +274,12 @@ SELECT str FROM expln('

DROP EXTENSION aqo CASCADE;
DROP EXTENSION postgres_fdw CASCADE;
NOTICE: drop cascades to 3 other objects
NOTICE: drop cascades to 5 other objects
DETAIL: drop cascades to server loopback
drop cascades to user mapping for public on server loopback
drop cascades to foreign table frgn
drop cascades to foreign table frgn_a
drop cascades to foreign table frgn_b
DROP TABLE local;
DROP TABLE local_b;
DROP TABLE local_a;
58 changes: 55 additions & 3 deletions path_utils.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
#include "aqo.h"
#include "hash.h"

#include "postgres_fdw.h"

/*
* Hook on creation of a plan node. We need to store AQO-specific data to
* support learning stage.
Expand Down Expand Up @@ -60,6 +62,31 @@ create_aqo_plan_node()
return node;
}


/* Ensure that it's postgres_fdw's foreign server oid */
static bool
is_postgres_fdw_server(Oid serverid)
{
ForeignServer *server;
ForeignDataWrapper *fdw;

if (!OidIsValid(serverid))
return false;

server = GetForeignServerExtended(serverid, FSV_MISSING_OK);
if (!server)
return false;

fdw = GetForeignDataWrapperExtended(server->fdwid, FDW_MISSING_OK);
if (!fdw || !fdw->fdwname)
return false;

if (strcmp(fdw->fdwname, "postgres_fdw") != 0)
return false;

return true;
}

/*
* Extract an AQO node from the plan private field.
* If no one node was found, return pointer to the default value or return NULL.
Expand Down Expand Up @@ -497,7 +524,8 @@ aqo_create_plan_hook(PlannerInfo *root, Path *src, Plan **dest)
return;

is_join_path = (src->type == T_NestPath || src->type == T_MergePath ||
src->type == T_HashPath);
src->type == T_HashPath ||
(src->type == T_ForeignPath && IS_JOIN_REL(src->parent)));

node = get_aqo_plan_node(plan, true);

Expand All @@ -513,8 +541,32 @@ aqo_create_plan_hook(PlannerInfo *root, Path *src, Plan **dest)

if (is_join_path)
{
node->clauses = aqo_get_clauses(root, ((JoinPath *) src)->joinrestrictinfo);
node->jointype = ((JoinPath *) src)->jointype;
if (IsA(src, ForeignPath))
{
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) src->parent->fdw_private;
List *restrictclauses = NIL;

if (!fpinfo)
return;

/* We have to ensure that this is postgres_fdw ForeignPath */
if (!is_postgres_fdw_server(src->parent->serverid))
return;

restrictclauses = list_concat(restrictclauses, fpinfo->joinclauses);
restrictclauses = list_concat(restrictclauses, fpinfo->remote_conds);
restrictclauses = list_concat(restrictclauses, fpinfo->local_conds);

node->clauses = aqo_get_clauses(root, restrictclauses);
node->jointype = fpinfo->jointype;

list_free(restrictclauses);
}
else
{
node->clauses = aqo_get_clauses(root, ((JoinPath *) src)->joinrestrictinfo);
node->jointype = ((JoinPath *) src)->jointype;
}
}
else if (IsA(src, AggPath))
/* Aggregation node must store grouping clauses. */
Expand Down
65 changes: 63 additions & 2 deletions sql/aqo_fdw.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,72 @@ SELECT str FROM expln('
SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x;
') AS str WHERE str NOT LIKE '%Sort Method%';

-- TODO: Should learn on postgres_fdw nodes
-- Should learn on postgres_fdw nodes
SELECT str FROM expln('
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)
SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x;
') AS str WHERE str NOT LIKE '%Query Identifier%';

CREATE TABLE local_a(aid int primary key, aval text);
CREATE TABLE local_b(bid int primary key, aid int references local_a(aid), bval text);
INSERT INTO local_a SELECT i, 'val_' || i FROM generate_series(1,100) i;
INSERT INTO local_b SELECT i, mod((i+random()*10)::numeric, 10) + 1, 'val_' || i FROM generate_series(1,1000) i;
ANALYZE local_a, local_b;

CREATE FOREIGN TABLE frgn_a(aid int, aval text) SERVER loopback OPTIONS (table_name 'local_a');
CREATE FOREIGN TABLE frgn_b(bid int, aid int, bval text) SERVER loopback OPTIONS (table_name 'local_b');

EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from frgn_a AS a, frgn_b AS b
WHERE a.aid = b.aid AND b.bval like 'val%';

EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from frgn_a AS a, frgn_b AS b
WHERE a.aid = b.aid AND b.bval like 'val%';

-- Partitioned join over foreign tables
set enable_partitionwise_join = on;
ALTER SERVER loopback OPTIONS (ADD fdw_tuple_cost '1.0');

CREATE TABLE local_main_p0(aid int, aval text);
CREATE TABLE local_main_p1(aid int, aval text);
CREATE TABLE main (aid int, aval text) PARTITION BY HASH(aid);

CREATE FOREIGN TABLE main_p0 PARTITION OF main FOR VALUES WITH (MODULUS 3, REMAINDER 0)
SERVER loopback OPTIONS (table_name 'local_main_p0');
CREATE FOREIGN TABLE main_p1 PARTITION OF main FOR VALUES WITH (MODULUS 3, REMAINDER 1)
SERVER loopback OPTIONS (table_name 'local_main_p1');
CREATE TABLE main_p2 PARTITION OF main FOR VALUES WITH (MODULUS 3, REMAINDER 2);

CREATE TABLE local_ref_p0(bid int, aid int, bval text);
CREATE TABLE local_ref_p1(bid int, aid int, bval text);
CREATE TABLE ref (bid int, aid int, bval text) PARTITION BY HASH(aid);

CREATE FOREIGN TABLE ref_p0 PARTITION OF ref FOR VALUES WITH (MODULUS 3, REMAINDER 0)
SERVER loopback OPTIONS (table_name 'local_ref_p0');
CREATE FOREIGN TABLE ref_p1 PARTITION OF ref FOR VALUES WITH (MODULUS 3, REMAINDER 1)
SERVER loopback OPTIONS (table_name 'local_ref_p1');
CREATE TABLE ref_p2 PARTITION OF ref FOR VALUES WITH (MODULUS 3, REMAINDER 2);

INSERT INTO main SELECT i, 'val_' || i FROM generate_series(1,100) i;
INSERT INTO ref SELECT i, mod(i, 10) + 1, 'val_' || i FROM generate_series(1,1000) i;

ANALYZE local_main_p0, local_main_p1, main_p2;
ANALYZE local_ref_p0, local_ref_p1, ref_p2;

EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from main AS a, ref AS b
WHERE a.aid = b.aid AND b.bval like 'val%';

EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * from main AS a, ref AS b
WHERE a.aid = b.aid AND b.bval like 'val%';

DROP TABLE main, local_main_p0, local_main_p1;
DROP TABLE ref, local_ref_p0, local_ref_p1;
ALTER SERVER loopback OPTIONS (DROP fdw_tuple_cost);
reset enable_partitionwise_join;

-- TODO: Non-mergejoinable join condition.
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
SELECT * FROM frgn AS a, frgn AS b WHERE a.x<b.x;
Expand All @@ -78,4 +138,5 @@ SELECT str FROM expln('
DROP EXTENSION aqo CASCADE;
DROP EXTENSION postgres_fdw CASCADE;
DROP TABLE local;

DROP TABLE local_b;
DROP TABLE local_a;