Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enable AO/AOCO insert to multiple files even enable_parallel is off #83

Merged
merged 3 commits into from
Jul 31, 2023
Merged

Enable AO/AOCO insert to multiple files even enable_parallel is off #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7cd24bb
Enable AO/AOCO insert to multiple files even enable_parallel is off.
avamingli Jul 7, 2023
3e3a125
Disable insert multiple files in Utility mode for AO/AOCS.
avamingli Jul 10, 2023
c088a88
Fix failed cases by gp_appendonly_insert_files.
avamingli Jul 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/backend/access/aocs/aocsam.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,7 @@ aocs_insert_init(Relation rel, int segno)
desc = (AOCSInsertDesc) palloc0(sizeof(AOCSInsertDescData));
desc->aoi_rel = rel;
desc->appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));
desc->insertMultiFiles = false;

/*
* Writers uses this since they have exclusive access to the lock acquired
Expand Down Expand Up @@ -1072,10 +1073,9 @@ aocs_insert_init(Relation rel, int segno)
(FileSegInfo *) desc->fsInfo, desc->lastSequence,
rel, segno, tupleDesc->natts, true);

/* should not enable insertMultiFiles if the table is created by own transaction */
desc->insertMultiFiles = enable_parallel &&
gp_appendonly_insert_files > 1 &&
!ShouldUseReservedSegno(rel, CHOOSE_MODE_WRITE);
/* Should not enable insertMultiFiles if the table is created by own transaction or in utility mode */
if (Gp_role != GP_ROLE_UTILITY)
desc->insertMultiFiles = gp_appendonly_insert_files > 1 && !ShouldUseReservedSegno(rel, CHOOSE_MODE_WRITE);
return desc;
}

Expand Down
8 changes: 4 additions & 4 deletions src/backend/access/appendonly/appendonlyam.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2664,6 +2664,7 @@ appendonly_insert_init(Relation rel, int segno)

aoInsertDesc->aoi_rel = rel;
aoInsertDesc->range = 0;
aoInsertDesc->insertMultiFiles = false;

/*
* We want to see an up-to-date view of the metadata. The target segment's
Expand Down Expand Up @@ -2851,10 +2852,9 @@ aoInsertDesc->appendOnlyMetaDataSnapshot, //CONCERN:Safe to assume all block dir
aoInsertDesc->fsInfo, aoInsertDesc->lastSequence,
rel, segno, 1, false);

/* should not enable insertMultiFiles if the table is created by own transaction */
aoInsertDesc->insertMultiFiles = enable_parallel &&
gp_appendonly_insert_files > 1 &&
!ShouldUseReservedSegno(rel, CHOOSE_MODE_WRITE);
/* Should not enable insertMultiFiles if the table is created by own transaction or in utility mode */
if (Gp_role != GP_ROLE_UTILITY)
aoInsertDesc->insertMultiFiles = gp_appendonly_insert_files > 1 && !ShouldUseReservedSegno(rel, CHOOSE_MODE_WRITE);
return aoInsertDesc;
}

Expand Down
2 changes: 1 addition & 1 deletion src/backend/utils/misc/guc_gp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3177,7 +3177,7 @@ struct config_int ConfigureNamesInt_gp[] =
NULL
},
&gp_appendonly_insert_files,
4, 0, 127,
4 /* CBDB_PARALLEL If default value is changed, set it in src/test/regress/GNUMakefile too, see details there */, 0, 127,
NULL, NULL, NULL
},

Expand Down
10 changes: 8 additions & 2 deletions src/test/isolation2/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,11 @@ clean distclean:
install: all gpdiff.pl gpstringsubs.pl

installcheck: install installcheck-parallel-retrieve-cursor
$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_schedule
(\
gpconfig -c gp_appendonly_insert_files -v 0; \
gpstop -u; \
$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_schedule \
)

installcheck-resgroup: install
$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_resgroup --dbname=isolation2resgrouptest --schedule=$(srcdir)/isolation2_resgroup_schedule
Expand All @@ -74,6 +78,8 @@ installcheck-parallel-retrieve-cursor: install
# Add a new rule for running installcheck with parallel mode enabled.
installcheck-cbdb-parallel: install
(\
export PGOPTIONS='-c optimizer=off -c enable_parallel=true -c gp_appendonly_insert_files=0'; \
gpconfig -c gp_appendonly_insert_files -v 0; \
gpstop -u; \
export PGOPTIONS='-c optimizer=off -c enable_parallel=true'; \
$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_schedule \
)
12 changes: 11 additions & 1 deletion src/test/regress/GNUmakefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,8 +204,18 @@ installcheck: installcheck-good
installcheck-small: all
$(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule $(EXTRA_TESTS)

# We have set default gp_appendonly_insert_files = 4 for user's underlying parallelism.
# But that will cause lots of diffs between non-parallel and parallel mode that we can't
# afford to modify them one by one, or create lots of answer files(and we didn't support that yet).
# Any tests related to ao parallel should set GUCs in their own sql files, like gp_parallel.
# Using gpconfig is a workaround for regression test for non-parallel and parallel mode.
# Same for src/test/isolation2/Makefile
installcheck-good: all twophase_pqexecparams hooktest query_info_hook_test
$(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule --schedule=$(srcdir)/greenplum_schedule $(EXTRA_TESTS)
(\
gpconfig -c gp_appendonly_insert_files -v 0; \
gpstop -u; \
$(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule --schedule=$(srcdir)/greenplum_schedule $(EXTRA_TESTS) \
)

installcheck-parallel: all
$(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule $(MAXCONNOPT) $(EXTRA_TESTS)
Expand Down
9 changes: 8 additions & 1 deletion src/test/regress/expected/gp_parallel.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ set force_parallel_mode = 0;
set optimizer = off;
create schema test_parallel;
set search_path to test_parallel;
-- set this to default in case regress change it by gpstop.
set gp_appendonly_insert_files = 4;
create table ao1(x int, y int) with(appendonly=true);
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
Expand All @@ -48,7 +50,8 @@ begin;
-- encourage use of parallel plans
set local min_parallel_table_scan_size = 0;
set local max_parallel_workers_per_gather = 4;
set local enable_parallel = true;
-- test insert into multiple files even enable_parallel is off.
set local enable_parallel = off;
-- insert multiple segfiles for parallel
set local gp_appendonly_insert_files = 4;
-- test appendonly table parallel
Expand All @@ -62,6 +65,7 @@ select segfilecount from pg_appendonly where relid = 'ao1'::regclass;
4
(1 row)

set local enable_parallel = on;
explain(costs off) select count(*) from ao1;
QUERY PLAN
--------------------------------------------------
Expand All @@ -79,6 +83,7 @@ select count(*) from ao1;
(1 row)

-- test aocs table parallel
set local enable_parallel = off;
insert into aocs1 select i, i from generate_series(1, 1200000) g(i);
analyze aocs1;
select segfilecount from pg_appendonly where relid = 'aocs1'::regclass;
Expand All @@ -87,6 +92,7 @@ select segfilecount from pg_appendonly where relid = 'aocs1'::regclass;
4
(1 row)

set local enable_parallel = on;
explain(costs off) select count(*) from aocs1;
QUERY PLAN
--------------------------------------------------
Expand Down Expand Up @@ -1698,5 +1704,6 @@ abort;
-- start_ignore
drop schema test_parallel cascade;
-- end_ignore
reset gp_appendonly_insert_files;
reset force_parallel_mode;
reset optimizer;
9 changes: 8 additions & 1 deletion src/test/regress/sql/gp_parallel.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ set optimizer = off;

create schema test_parallel;
set search_path to test_parallel;
-- set this to default in case regress change it by gpstop.
set gp_appendonly_insert_files = 4;

create table ao1(x int, y int) with(appendonly=true);
create table ao2(x int, y int) with(appendonly=true);
Expand All @@ -46,7 +48,8 @@ begin;
-- encourage use of parallel plans
set local min_parallel_table_scan_size = 0;
set local max_parallel_workers_per_gather = 4;
set local enable_parallel = true;
-- test insert into multiple files even enable_parallel is off.
set local enable_parallel = off;

-- insert multiple segfiles for parallel
set local gp_appendonly_insert_files = 4;
Expand All @@ -57,13 +60,16 @@ analyze ao1;
insert into ao2 select i%10, i from generate_series(1, 1200000) g(i);
analyze ao2;
select segfilecount from pg_appendonly where relid = 'ao1'::regclass;
set local enable_parallel = on;
explain(costs off) select count(*) from ao1;
select count(*) from ao1;

-- test aocs table parallel
set local enable_parallel = off;
insert into aocs1 select i, i from generate_series(1, 1200000) g(i);
analyze aocs1;
select segfilecount from pg_appendonly where relid = 'aocs1'::regclass;
set local enable_parallel = on;
explain(costs off) select count(*) from aocs1;
select count(*) from aocs1;

Expand Down Expand Up @@ -508,5 +514,6 @@ abort;
drop schema test_parallel cascade;
-- end_ignore

reset gp_appendonly_insert_files;
reset force_parallel_mode;
reset optimizer;
Loading