Permalink
Browse files

Logical replication

- Add PUBLICATION catalogs and DDL
- Add SUBSCRIPTION catalog and DDL
- Define logical replication protocol and output plugin
- Add logical replication workers

From: Petr Jelinek <petr@2ndquadrant.com>
Reviewed-by: Steve Singer <steve@ssinger.info>
Reviewed-by: Andres Freund <andres@anarazel.de>
Reviewed-by: Erik Rijkers <er@xs4all.nl>
Reviewed-by: Peter Eisentraut <peter.eisentraut@2ndquadrant.com>
  • Loading branch information...
petere committed Jan 19, 2017
1 parent ba61a04 commit 665d1fad99e7b11678b0d5fa24d2898424243cd6
Showing with 13,354 additions and 95 deletions.
  1. +309 −0 doc/src/sgml/catalogs.sgml
  2. +41 −0 doc/src/sgml/config.sgml
  3. +1 −0 doc/src/sgml/filelist.sgml
  4. +1 −1 doc/src/sgml/func.sgml
  5. +396 −0 doc/src/sgml/logical-replication.sgml
  6. +74 −0 doc/src/sgml/monitoring.sgml
  7. +1 −0 doc/src/sgml/postgres.sgml
  8. +721 −0 doc/src/sgml/protocol.sgml
  9. +6 −0 doc/src/sgml/ref/allfiles.sgml
  10. +139 −0 doc/src/sgml/ref/alter_publication.sgml
  11. +139 −0 doc/src/sgml/ref/alter_subscription.sgml
  12. +206 −0 doc/src/sgml/ref/create_publication.sgml
  13. +176 −0 doc/src/sgml/ref/create_subscription.sgml
  14. +107 −0 doc/src/sgml/ref/drop_publication.sgml
  15. +110 −0 doc/src/sgml/ref/drop_subscription.sgml
  16. +21 −0 doc/src/sgml/ref/pg_dump.sgml
  17. +28 −0 doc/src/sgml/ref/psql-ref.sgml
  18. +6 −0 doc/src/sgml/reference.sgml
  19. +1 −0 src/Makefile
  20. +2 −0 src/backend/access/transam/xact.c
  21. +4 −3 src/backend/catalog/Makefile
  22. +57 −0 src/backend/catalog/aclchk.c
  23. +6 −2 src/backend/catalog/catalog.c
  24. +24 −0 src/backend/catalog/dependency.c
  25. +219 −0 src/backend/catalog/objectaddress.c
  26. +457 −0 src/backend/catalog/pg_publication.c
  27. +11 −0 src/backend/catalog/pg_shdepend.c
  28. +207 −0 src/backend/catalog/pg_subscription.c
  29. +25 −0 src/backend/catalog/system_views.sql
  30. +4 −4 src/backend/commands/Makefile
  31. +10 −0 src/backend/commands/alter.c
  32. +17 −0 src/backend/commands/dbcommands.c
  33. +28 −0 src/backend/commands/define.c
  34. +4 −0 src/backend/commands/dropcmds.c
  35. +8 −0 src/backend/commands/event_trigger.c
  36. +754 −0 src/backend/commands/publicationcmds.c
  37. +643 −0 src/backend/commands/subscriptioncmds.c
  38. +12 −0 src/backend/commands/tablecmds.c
  39. +1 −1 src/backend/executor/Makefile
  40. +2 −1 src/backend/executor/execMain.c
  41. +553 −0 src/backend/executor/execReplication.c
  42. +78 −0 src/backend/nodes/copyfuncs.c
  43. +73 −0 src/backend/nodes/equalfuncs.c
  44. +255 −7 src/backend/parser/gram.y
  45. +22 −1 src/backend/postmaster/bgworker.c
  46. +6 −0 src/backend/postmaster/pgstat.c
  47. +9 −0 src/backend/postmaster/postmaster.c
  48. +194 −23 src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
  49. +2 −2 src/backend/replication/logical/Makefile
  50. +759 −0 src/backend/replication/logical/launcher.c
  51. +637 −0 src/backend/replication/logical/proto.c
  52. +489 −0 src/backend/replication/logical/relation.c
  53. +1,429 −0 src/backend/replication/logical/worker.c
  54. +32 −0 src/backend/replication/pgoutput/Makefile
  55. +596 −0 src/backend/replication/pgoutput/pgoutput.c
  56. +15 −5 src/backend/replication/walreceiver.c
  57. +3 −0 src/backend/storage/ipc/ipci.c
  58. +2 −0 src/backend/storage/lmgr/lwlocknames.txt
  59. +83 −0 src/backend/tcop/utility.c
  60. +30 −5 src/backend/utils/cache/inval.c
  61. +128 −8 src/backend/utils/cache/relcache.c
  62. +69 −0 src/backend/utils/cache/syscache.c
  63. +13 −0 src/backend/utils/misc/guc.c
  64. +1 −0 src/backend/utils/misc/postgresql.conf.sample
  65. +12 −0 src/bin/pg_dump/common.c
  66. +3 −0 src/bin/pg_dump/pg_backup.h
  67. +6 −1 src/bin/pg_dump/pg_backup_archiver.c
  68. +464 −0 src/bin/pg_dump/pg_dump.c
  69. +45 −1 src/bin/pg_dump/pg_dump.h
  70. +19 −1 src/bin/pg_dump/pg_dump_sort.c
  71. +3 −0 src/bin/pg_dump/pg_restore.c
  72. +66 −10 src/bin/pg_dump/t/002_pg_dump.pl
  73. +16 −0 src/bin/psql/command.c
  74. +292 −0 src/bin/psql/describe.c
  75. +9 −0 src/bin/psql/describe.h
  76. +2 −0 src/bin/psql/help.c
  77. +49 −4 src/bin/psql/tab-complete.c
  78. +3 −0 src/include/catalog/dependency.h
  79. +18 −0 src/include/catalog/indexing.h
  80. +6 −0 src/include/catalog/pg_proc.h
  81. +104 −0 src/include/catalog/pg_publication.h
  82. +52 −0 src/include/catalog/pg_publication_rel.h
  83. +83 −0 src/include/catalog/pg_subscription.h
  84. +1 −0 src/include/commands/defrem.h
  85. +28 −0 src/include/commands/publicationcmds.h
  86. +27 −0 src/include/commands/subscriptioncmds.h
  87. +17 −0 src/include/executor/executor.h
  88. +5 −0 src/include/nodes/nodes.h
  89. +51 −0 src/include/nodes/parsenodes.h
  90. +3 −0 src/include/parser/kwlist.h
  91. +3 −1 src/include/pgstat.h
  92. +27 −0 src/include/replication/logicallauncher.h
  93. +104 −0 src/include/replication/logicalproto.h
  94. +43 −0 src/include/replication/logicalrelation.h
  95. +17 −0 src/include/replication/logicalworker.h
  96. +29 −0 src/include/replication/pgoutput.h
  97. +49 −11 src/include/replication/walreceiver.h
  98. +62 −0 src/include/replication/worker_internal.h
  99. +2 −1 src/include/storage/sinval.h
  100. +4 −0 src/include/utils/acl.h
  101. +2 −0 src/include/utils/inval.h
  102. +6 −0 src/include/utils/rel.h
  103. +6 −0 src/include/utils/relcache.h
  104. +6 −0 src/include/utils/syscache.h
  105. +1 −1 src/test/Makefile
  106. +12 −1 src/test/perl/PostgresNode.pm
  107. +156 −0 src/test/regress/expected/publication.out
  108. +18 −0 src/test/regress/expected/rules.out
  109. +3 −0 src/test/regress/expected/sanity_check.out
  110. +66 −0 src/test/regress/expected/subscription.out
  111. +3 −0 src/test/regress/parallel_schedule
  112. +2 −0 src/test/regress/serial_schedule
  113. +82 −0 src/test/regress/sql/publication.sql
  114. +44 −0 src/test/regress/sql/subscription.sql
  115. +2 −0 src/test/subscription/.gitignore
  116. +22 −0 src/test/subscription/Makefile
  117. +16 −0 src/test/subscription/README
  118. +188 −0 src/test/subscription/t/001_rep_changes.pl
  119. +539 −0 src/test/subscription/t/002_types.pl
View
@@ -240,6 +240,16 @@
<entry>functions and procedures</entry>
</row>
<row>
<entry><link linkend="catalog-pg-publication"><structname>pg_publication</structname></link></entry>
<entry>publications for logical replication</entry>
</row>
<row>
<entry><link linkend="catalog-pg-publication-rel"><structname>pg_publication_rel</structname></link></entry>
<entry>relation to publication mapping</entry>
</row>
<row>
<entry><link linkend="catalog-pg-range"><structname>pg_range</structname></link></entry>
<entry>information about range types</entry>
@@ -285,6 +295,11 @@
<entry>planner statistics</entry>
</row>
<row>
<entry><link linkend="catalog-pg-subscription"><structname>pg_subscription</structname></link></entry>
<entry>logical replication subscriptions</entry>
</row>
<row>
<entry><link linkend="catalog-pg-tablespace"><structname>pg_tablespace</structname></link></entry>
<entry>tablespaces within this database cluster</entry>
@@ -5271,6 +5286,137 @@
</sect1>
<sect1 id="catalog-pg-publication">
<title><structname>pg_publication</structname></title>
<indexterm zone="catalog-pg-publication">
<primary>pg_publication</primary>
</indexterm>
<para>
The catalog <structname>pg_publication</structname> contains all
publications created in the database. For more on publications see
<xref linkend="logical-replication-publication">.
</para>
<table>
<title><structname>pg_publication</structname> Columns</title>
<tgroup cols="4">
<thead>
<row>
<entry>Name</entry>
<entry>Type</entry>
<entry>References</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry><structfield>oid</structfield></entry>
<entry><type>oid</type></entry>
<entry></entry>
<entry>Row identifier (hidden attribute; must be explicitly selected)</entry>
</row>
<row>
<entry><structfield>pubname</structfield></entry>
<entry><type>Name</type></entry>
<entry></entry>
<entry>Name of the publication</entry>
</row>
<row>
<entry><structfield>pubowner</structfield></entry>
<entry><type>oid</type></entry>
<entry><literal><link linkend="catalog-pg-authid"><structname>pg_authid</structname></link>.oid</literal></entry>
<entry>Owner of the publication</entry>
</row>
<row>
<entry><structfield>puballtables</structfield></entry>
<entry><type>bool</type></entry>
<entry></entry>
<entry>If true, this publication automatically includes all tables
in the database, including any that will be created in the future.
</entry>
</row>
<row>
<entry><structfield>pubinsert</structfield></entry>
<entry><type>bool</type></entry>
<entry></entry>
<entry>If true, <command>INSERT</command> operations are replicated for
tables in the publication.</entry>
</row>
<row>
<entry><structfield>pubupdate</structfield></entry>
<entry><type>bool</type></entry>
<entry></entry>
<entry>If true, <command>UPDATE</command> operations are replicated for
tables in the publication.</entry>
</row>
<row>
<entry><structfield>pubdelete</structfield></entry>
<entry><type>bool</type></entry>
<entry></entry>
<entry>If true, <command>DELETE</command> operations are replicated for
tables in the publication.</entry>
</row>
</tbody>
</tgroup>
</table>
</sect1>
<sect1 id="catalog-pg-publication-rel">
<title><structname>pg_publication_rel</structname></title>
<indexterm zone="catalog-pg-publication-rel">
<primary>pg_publication_rel</primary>
</indexterm>
<para>
The catalog <structname>pg_publication_rel</structname> contains the
mapping between relations and publications in the database. This is a
many-to-many mapping. See also <xref linkend="view-pg-publication-tables">
for a more user-friendly view of this information.
</para>
<table>
<title><structname>pg_publication_rel</structname> Columns</title>
<tgroup cols="4">
<thead>
<row>
<entry>Name</entry>
<entry>Type</entry>
<entry>References</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry><structfield>prpubid</structfield></entry>
<entry><type>oid</type></entry>
<entry><literal><link linkend="catalog-pg-publication"><structname>pg_publication</structname></link>.oid</literal></entry>
<entry>Reference to publication</entry>
</row>
<row>
<entry><structfield>prrelid</structfield></entry>
<entry><type>oid</type></entry>
<entry><literal><link linkend="catalog-pg-class"><structname>pg_class</structname></link>.oid</literal></entry>
<entry>Reference to relation</entry>
</row>
</tbody>
</tgroup>
</table>
</sect1>
<sect1 id="catalog-pg-range">
<title><structname>pg_range</structname></title>
@@ -6150,6 +6296,109 @@
</sect1>
<sect1 id="catalog-pg-subscription">
<title><structname>pg_subscription</structname></title>
<indexterm zone="catalog-pg-subscription">
<primary>pg_subscription</primary>
</indexterm>
<para>
The catalog <structname>pg_subscription</structname> contains all existing
logical replication subscriptions. For more information about logical
replication see <xref linkend="logical-replication">.
</para>
<para>
Unlike most system catalogs, <structname>pg_subscription</structname> is
shared across all databases of a cluster: There is only one copy
of <structname>pg_subscription</structname> per cluster, not one per
database.
</para>
<para>
Access to this catalog is restricted from normal users. Normal users can
use the view <xref linkend="pg-stat-subscription"> to get some information
about subscriptions.
</para>
<table>
<title><structname>pg_subscription</structname> Columns</title>
<tgroup cols="4">
<thead>
<row>
<entry>Name</entry>
<entry>Type</entry>
<entry>References</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry><structfield>oid</structfield></entry>
<entry><type>oid</type></entry>
<entry></entry>
<entry>Row identifier (hidden attribute; must be explicitly selected)</entry>
</row>
<row>
<entry><structfield>subdbid</structfield></entry>
<entry><type>oid</type></entry>
<entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
<entry>OID of the database which the subscription resides in</entry>
</row>
<row>
<entry><structfield>subname</structfield></entry>
<entry><type>name</type></entry>
<entry></entry>
<entry>Name of the subscription</entry>
</row>
<row>
<entry><structfield>subowner</structfield></entry>
<entry><type>oid</type></entry>
<entry><literal><link linkend="catalog-pg-authid"><structname>pg_authid</structname></link>.oid</literal></entry>
<entry>Owner of the subscription</entry>
</row>
<row>
<entry><structfield>subenabled</structfield></entry>
<entry><type>bool</type></entry>
<entry></entry>
<entry>If true, the subscription is enabled and should be replicating.</entry>
</row>
<row>
<entry><structfield>subconninfo</structfield></entry>
<entry><type>text</type></entry>
<entry></entry>
<entry>Connection string to the upstream database</entry>
</row>
<row>
<entry><structfield>subslotname</structfield></entry>
<entry><type>name</type></entry>
<entry></entry>
<entry>Name of the replication slot in the upstream database. Also used
for local replication origin name.</entry>
</row>
<row>
<entry><structfield>subpublications</structfield></entry>
<entry><type>text[]</type></entry>
<entry></entry>
<entry>Array of subscribed publication names. These reference the
publications on the publisher server. For more on publications
see <xref linkend="logical-replication-publication">.
</entry>
</row>
</tbody>
</tgroup>
</table>
</sect1>
<sect1 id="catalog-pg-tablespace">
<title><structname>pg_tablespace</structname></title>
@@ -7589,6 +7838,11 @@
<entry>prepared transactions</entry>
</row>
<row>
<entry><link linkend="view-pg-publication-tables"><structname>pg_publication_tables</structname></link></entry>
<entry>publications and their associated tables</entry>
</row>
<row>
<entry><link linkend="view-pg-replication-origin-status"><structname>pg_replication_origin_status</structname></link></entry>
<entry>information about replication origins, including replication progress</entry>
@@ -8871,6 +9125,61 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
</sect1>
<sect1 id="view-pg-publication-tables">
<title><structname>pg_publication_tables</structname></title>
<indexterm zone="view-pg-publication-tables">
<primary>pg_publication_tables</primary>
</indexterm>
<para>
The view <structname>pg_publication_tables</structname> provides
information about the mapping between publications and the tables they
contain. Unlike the underlying
catalog <structname>pg_publication_rel</structname>, this view expands
publications defined as <literal>FOR ALL TABLES</literal>, so for such
publications there will be a row for each eligible table.
</para>
<table>
<title><structname>pg_publication_tables</structname> Columns</title>
<tgroup cols="4">
<thead>
<row>
<entry>Name</entry>
<entry>Type</entry>
<entry>References</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry><structfield>pubname</structfield></entry>
<entry><type>name</type></entry>
<entry><literal><link linkend="catalog-pg-publication"><structname>pg_publication</structname></link>.pubname</literal></entry>
<entry>Name of publication</entry>
</row>
<row>
<entry><structfield>schemaname</structfield></entry>
<entry><type>name</type></entry>
<entry><literal><link linkend="catalog-pg-namespace"><structname>pg_namespace</structname></link>.nspname</literal></entry>
<entry>Name of schema containing table</entry>
</row>
<row>
<entry><structfield>tablename</structfield></entry>
<entry><type>name</type></entry>
<entry><literal><link linkend="catalog-pg-class"><structname>pg_class</structname></link>.relname</literal></entry>
<entry>Name of table</entry>
</row>
</tbody>
</tgroup>
</table>
</sect1>
<sect1 id="view-pg-replication-origin-status">
<title><structname>pg_replication_origin_status</structname></title>
View
@@ -3411,6 +3411,47 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
</variablelist>
</sect2>
<sect2 id="runtime-config-replication-subscriber">
<title>Subscribers</title>
<para>
These settings control the behavior of a logical replication subscriber.
Their values on the publisher are irrelevant.
</para>
<para>
Note that <varname>wal_receiver_timeout</varname> and
<varname>wal_retrieve_retry_interval</varname> configuration parameters
affect the logical replication workers as well.
</para>
<variablelist>
<varlistentry id="guc-max-logical-replication-workers" xreflabel="max_logical_replication_workers">
<term><varname>max_logical_replication_workers</varname> (<type>int</type>)
<indexterm>
<primary><varname>max_logical_replication_workers</> configuration parameter</primary>
</indexterm>
</term>
<listitem>
<para>
Specifies maximum number of logical replication workers. This includes
both apply workers and table synchronization workers.
</para>
<para>
Logical replication workers are taken from the pool defined by
<varname>max_worker_processes</varname>.
</para>
<para>
The default value is 4.
</para>
</listitem>
</varlistentry>
</variablelist>
</sect2>
</sect1>
<sect1 id="runtime-config-query">
Oops, something went wrong.

0 comments on commit 665d1fa

Please sign in to comment.