Add a performance measurement script for these new emaj_dump_changes_…

…group() and emaj_gen_sql_dump_changes_group() functions.
dalibo · Sep 14, 2023 · 6a365a0 · 6a365a0
1 parent 6051b95
commit 6a365a0
Show file tree

Hide file tree

Showing 4 changed files with 201 additions and 2 deletions.
diff --git a/docs/en/contributing.rst b/docs/en/contributing.rst
@@ -264,7 +264,8 @@ The scripts cover the following domains:
 
 * *log_overhead/pgbench.sh* evaluates the log mechanism overhead, using pgbench,
 * *large_group/large_group.sh* evaluates the behaviour of groups containing a large number of tables,
-* *rollback/rollback_perf.sh* evaluates the E-Maj rollback performances with different tables profiles.
+* *rollback/rollback_perf.sh* evaluates the E-Maj rollback performances with different tables profiles,
+* *dump_changes/dump_changes_perf.sh* measure the performances of changes dump operations, with various consolidation levels.
 
 For all these files, some variables have to be configured at the begining of the scripts.
 

diff --git a/docs/fr/contributing.rst b/docs/fr/contributing.rst
@@ -267,7 +267,8 @@ Les scripts couvrent les domaines suivants :
 
 * *log_overhead/pgbench.sh* évalue le surcoût du mécanisme de log, à l’aide de pgbench,
 * *large_group/large_group.sh* évalue le fonctionnement de groupes contenant un grand nombre de tables,
-* *rollback/rollback_perf.sh* évalue les performances des rollbacks E-Maj avec différents profils de tables.
+* *rollback/rollback_perf.sh* évalue les performances des rollbacks E-Maj avec différents profils de tables,
+* *dump_changes/dump_changes_perf.sh* mesure les performances des opérations de vidage des mises à jour, avec différents niveaux de consolidation.
 
 Pour chacun de ces fichiers, des variables sont à configurer en début de scripts,
 

diff --git a/tools/performance/README b/tools/performance/README
@@ -7,6 +7,7 @@ More precisely, it contains the following directories:
 - tools/performance/large_group, to measure the elapse time of main E-Maj functions with large tables groups (ie. containing a big number of tables)
 - tools/performance/log_overhead, to evaluate the overhead of the logging mechanism, using pgbench
 - tools/performance/rollback, to appreciate the rollback operations performance with various types of data updates.
+- tools/performance/dump_changes, to measure the changes dump operations performance with each consolidation level
 
 Each sub-directory contains a single shell script that needs to be customized.
 

diff --git a/tools/performance/dump_changes/dump_changes_perf.sh b/tools/performance/dump_changes/dump_changes_perf.sh
@@ -0,0 +1,196 @@
+#!/bin/sh
+# changes_consolidation_perf.sh: measure changes dump with consolidation performance
+#
+# Uses 2 tables: one with a short row but a large pkey, another with a large row and a short pkey
+
+# Define parameters
+
+export SCALEFACTOR=300       # each step creates 1000 * SCALEFACTOR rows
+
+export PGHOST=localhost
+export PGPORT=5416
+export PGUSER=postgres
+export PGDATABASE=regression
+
+dropdb $PGDATABASE
+createdb $PGDATABASE
+
+date
+
+rm -r /tmp/perf
+mkdir /tmp/perf
+
+# Execute the psql script
+psql -a -v p_scaleFactor=$SCALEFACTOR <<**PSQL1**
+
+\timing on
+\set ON_ERROR_STOP
+SET client_min_messages TO WARNING;
+
+-----------------------------------
+-- Create structures
+-----------------------------------
+-- install the E-Maj extension
+CREATE EXTENSION emaj CASCADE;
+
+create schema perfschema;
+set search_path = 'perfschema';
+
+-- Table perf1 has few columns but a large pkey
+create table perf1 (
+  c1    integer    not null,
+  c2    integer    not null,
+  c3    integer    not null,
+  c4    integer    not null,
+  c5    integer    not null,
+  c6    integer    not null,
+  c7    integer    not null,
+  c8    integer    not null,
+primary key (c1,c2,c3,c4,c5,c6,c7)
+);
+
+-- Table perf2 has few but large columns, a small pkey
+create table perf2 (
+  c1    integer    not null,
+  c2    text,
+  c3    text,
+  c4    text,
+primary key (c1)
+);
+create index on perf2(c2);
+
+select emaj.emaj_create_group('perf1',true);
+select emaj.emaj_create_group('perf2',true);
+
+select emaj.emaj_assign_table('perfschema','perf1','perf1');
+select emaj.emaj_assign_table('perfschema','perf2','perf2');
+
+select emaj.emaj_start_group(group_name,'init') from emaj.emaj_group where group_name like 'perf%';
+
+vacuum analyze;
+checkpoint;
+
+-----------------------------------------------------------
+-- Measure changes dump and logged rollbacks on table perf1
+-----------------------------------------------------------
+
+--> inserts only
+select emaj.emaj_stop_group('perf1');
+truncate table perf1;
+insert into perf1 select i, 2, 3, 4, 5, 6, 7, 8 from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf1','init');
+insert into perf1 select c1, 0, c3, c4, c5, c6, c7, c8 from perf1 where c1 % 10 = 0;
+select emaj.emaj_set_mark_group('perf1','end');
+vacuum analyze perf1;
+vacuum analyze emaj_perfschema.perf1_log;
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+--> updates only
+select emaj.emaj_stop_group('perf1');
+truncate table perf1;
+insert into perf1 select i, 2, 3, 4, 5, 6, 7, 8 from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf1','init');
+update perf1 set c2 = 0 where c1 % 10 = 1;
+select emaj.emaj_set_mark_group('perf1','end');
+vacuum analyze perf1;
+vacuum analyze emaj_perfschema.perf1_log;
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+--> deletes only
+select emaj.emaj_stop_group('perf1');
+truncate table perf1;
+insert into perf1 select i, 2, 3, 4, 5, 6, 7, 8 from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf1','init');
+delete from perf1 where c1 % 10 = 2;
+select emaj.emaj_set_mark_group('perf1','end');
+vacuum analyze perf1;
+vacuum analyze emaj_perfschema.perf1_log;
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+--> inserts, updates and deletes mix
+select emaj.emaj_stop_group('perf1');
+truncate table perf1;
+insert into perf1 select i, 2, 3, 4, 5, 6, 7, 8 from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf1','init');
+insert into perf1 select c1, 0, c3, c4, c5, c6, c7, c8 from perf1 where c1 % 10 = 0;
+update perf1 set c2 = 0 where c1 % 10 = 1;
+delete from perf1 where c1 % 10 = 2;
+select emaj.emaj_set_mark_group('perf1','end');
+vacuum analyze perf1;
+vacuum analyze emaj_perfschema.perf1_log;
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf1','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+select emaj.emaj_logged_rollback_group('perf1','init');
+
+-----------------------------------------------------------
+-- Measure changes dump and logged rollbacks on table perf2
+-----------------------------------------------------------
+
+--> inserts only
+select emaj.emaj_stop_group('perf2');
+truncate table perf2;
+insert into perf2 select i, rpad('2',300), rpad('3',300), rpad('4',300) from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf2','init');
+insert into perf2 select -c1, c2, c3, c4 from perf2 where c1 % 10 = 0;
+select emaj.emaj_set_mark_group('perf2','end');
+vacuum analyze perf2;
+vacuum analyze emaj_perfschema.perf2_log;
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+--> updates only
+select emaj.emaj_stop_group('perf2');
+truncate table perf2;
+insert into perf2 select i, rpad('2',300), rpad('3',300), rpad('4',300) from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf2','init');
+update perf2 set c2 = '' where c1 % 10 = 1;
+select emaj.emaj_set_mark_group('perf2','end');
+vacuum analyze perf2;
+vacuum analyze emaj_perfschema.perf2_log;
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+--> deletes only
+select emaj.emaj_stop_group('perf2');
+truncate table perf2;
+insert into perf2 select i, rpad('2',300), rpad('3',300), rpad('4',300) from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf2','init');
+delete from perf2 where c1 % 10 = 2;
+select emaj.emaj_set_mark_group('perf2','end');
+vacuum analyze perf2;
+vacuum analyze emaj_perfschema.perf2_log;
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+--> inserts, updates and deletes mix
+select emaj.emaj_stop_group('perf2');
+truncate table perf2;
+insert into perf2 select i, rpad('2',300), rpad('3',300), rpad('4',300) from generate_series (1, 1000*:p_scaleFactor) i;
+select emaj.emaj_start_group('perf2','init');
+insert into perf2 select -c1, c2, c3, c4 from perf2 where c1 % 10 = 0;
+update perf2 set c2 = '' where c1 % 10 = 1;
+delete from perf2 where c1 % 10 = 2;
+select emaj.emaj_set_mark_group('perf2','end');
+vacuum analyze perf2;
+vacuum analyze emaj_perfschema.perf2_log;
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=NONE',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=PARTIAL',NULL,'/tmp/perf');
+select emaj.emaj_dump_changes_group('perf2','init','end','CONSOLIDATION=FULL',NULL,'/tmp/perf');
+
+select emaj.emaj_logged_rollback_group('perf2','init');
+
+**PSQL1**
+rm -r /tmp/perf
+
+date