-
Notifications
You must be signed in to change notification settings - Fork 4.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Flush logical slots to disk during a shutdown checkpoint if required.
It's entirely possible for a logical slot to have a confirmed_flush LSN higher than the last value saved on disk while not being marked as dirty. Currently, it is not a major problem but a later patch adding support for the upgrade of slots relies on that value being properly flushed to disk. It can also help avoid processing the same transactions again in some boundary cases after the clean shutdown and restart. Say, we process some transactions for which we didn't send anything downstream (the changes got filtered) but the confirm_flush LSN is updated due to keepalives. As we don't flush the latest value of confirm_flush LSN, it may lead to processing the same changes again without this patch. The approach taken by this patch has been suggested by Ashutosh Bapat. Author: Vignesh C, Julien Rouhaud, Kuroda Hayato Reviewed-by: Amit Kapila, Dilip Kumar, Michael Paquier, Ashutosh Bapat, Peter Smith, Hou Zhijie Discussion: http://postgr.es/m/CAA4eK1JzJagMmb_E8D4au=GYQkxox0AfNBm1FbP7sy7t4YWXPQ@mail.gmail.com Discussion: http://postgr.es/m/TYAPR01MB58664C81887B3AF2EB6B16E3F5939@TYAPR01MB5866.jpnprd01.prod.outlook.com
- Loading branch information
Amit Kapila
committed
Sep 14, 2023
1 parent
a2e0d5e
commit e0b2eed
Showing
5 changed files
with
145 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
|
|
||
| # Copyright (c) 2023, PostgreSQL Global Development Group | ||
|
|
||
| # Test logical replication slots are always flushed to disk during a shutdown | ||
| # checkpoint. | ||
|
|
||
| use strict; | ||
| use warnings; | ||
|
|
||
| use PostgreSQL::Test::Cluster; | ||
| use PostgreSQL::Test::Utils; | ||
| use Test::More; | ||
|
|
||
| sub compare_confirmed_flush | ||
| { | ||
| my ($node, $confirmed_flush_from_log) = @_; | ||
|
|
||
| # Fetch Latest checkpoint location from the control file | ||
| my ($stdout, $stderr) = | ||
| run_command([ 'pg_controldata', $node->data_dir ]); | ||
| my @control_data = split("\n", $stdout); | ||
| my $latest_checkpoint = undef; | ||
| foreach (@control_data) | ||
| { | ||
| if ($_ =~ /^Latest checkpoint location:\s*(.*)$/mg) | ||
| { | ||
| $latest_checkpoint = $1; | ||
| last; | ||
| } | ||
| } | ||
| die "Latest checkpoint location not found in control file\n" | ||
| unless defined($latest_checkpoint); | ||
|
|
||
| # Is it same as the value read from log? | ||
| ok( $latest_checkpoint eq $confirmed_flush_from_log, | ||
| "Check that the slot's confirmed_flush LSN is the same as the latest_checkpoint location" | ||
| ); | ||
|
|
||
| return; | ||
| } | ||
|
|
||
| # Initialize publisher node | ||
| my $node_publisher = PostgreSQL::Test::Cluster->new('pub'); | ||
| $node_publisher->init(allows_streaming => 'logical'); | ||
| # Avoid checkpoint during the test, otherwise, the latest checkpoint location | ||
| # will change. | ||
| $node_publisher->append_conf( | ||
| 'postgresql.conf', q{ | ||
| checkpoint_timeout = 1h | ||
| autovacuum = off | ||
| }); | ||
| $node_publisher->start; | ||
|
|
||
| # Create subscriber node | ||
| my $node_subscriber = PostgreSQL::Test::Cluster->new('sub'); | ||
| $node_subscriber->init(allows_streaming => 'logical'); | ||
| $node_subscriber->start; | ||
|
|
||
| # Create tables | ||
| $node_publisher->safe_psql('postgres', "CREATE TABLE test_tbl (id int)"); | ||
| $node_subscriber->safe_psql('postgres', "CREATE TABLE test_tbl (id int)"); | ||
|
|
||
| # Insert some data | ||
| $node_publisher->safe_psql('postgres', | ||
| "INSERT INTO test_tbl VALUES (generate_series(1, 5));"); | ||
|
|
||
| # Setup logical replication | ||
| my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; | ||
| $node_publisher->safe_psql('postgres', | ||
| "CREATE PUBLICATION pub FOR ALL TABLES"); | ||
| $node_subscriber->safe_psql('postgres', | ||
| "CREATE SUBSCRIPTION sub CONNECTION '$publisher_connstr' PUBLICATION pub" | ||
| ); | ||
|
|
||
| $node_subscriber->wait_for_subscription_sync($node_publisher, 'sub'); | ||
|
|
||
| my $result = | ||
| $node_subscriber->safe_psql('postgres', "SELECT count(*) FROM test_tbl"); | ||
|
|
||
| is($result, qq(5), "check initial copy was done"); | ||
|
|
||
| my $offset = -s $node_publisher->logfile; | ||
|
|
||
| # Restart the publisher to ensure that the slot will be flushed if required | ||
| $node_publisher->restart(); | ||
|
|
||
| # Wait until the walsender creates decoding context | ||
| $node_publisher->wait_for_log( | ||
| qr/Streaming transactions committing after ([A-F0-9]+\/[A-F0-9]+), reading WAL from ([A-F0-9]+\/[A-F0-9]+)./, | ||
| $offset); | ||
|
|
||
| # Extract confirmed_flush from the logfile | ||
| my $log_contents = slurp_file($node_publisher->logfile, $offset); | ||
| $log_contents =~ | ||
| qr/Streaming transactions committing after ([A-F0-9]+\/[A-F0-9]+), reading WAL from ([A-F0-9]+\/[A-F0-9]+)./ | ||
| or die "could not get confirmed_flush_lsn"; | ||
|
|
||
| # Ensure that the slot's confirmed_flush LSN is the same as the | ||
| # latest_checkpoint location. | ||
| compare_confirmed_flush($node_publisher, $1); | ||
|
|
||
| done_testing(); |