Add backend support for injection points

Injection points are a new facility that makes possible for developers
to run custom code in pre-defined code paths.  Its goal is to provide
ways to design and run advanced tests, for cases like:
- Race conditions, where processes need to do actions in a controlled
ordered manner.
- Forcing a state, like an ERROR, FATAL or even PANIC for OOM, to force
recovery, etc.
- Arbitrary sleeps.

This implements some basics, and there are plans to extend it more in
the future depending on what's required.  Hence, this commit adds a set
of routines in the backend that allows developers to attach, detach and
run injection points:
- A code path calling an injection point can be declared with the macro
INJECTION_POINT(name).
- InjectionPointAttach() and InjectionPointDetach() to respectively
attach and detach a callback to/from an injection point.  An injection
point name is registered in a shmem hash table with a library name and a
function name, which will be used to load the callback attached to an
injection point when its code path is run.

Injection point names are just strings, so as an injection point can be
declared and run by out-of-core extensions and modules, with callbacks
defined in external libraries.

This facility is hidden behind a dedicated switch for ./configure and
meson, disabled by default.

Note that backends use a local cache to store callbacks already loaded,
cleaning up their cache if a callback has found to be removed on a
best-effort basis.  This could be refined further but any tests but what
we have here was fine with the tests I've written while implementing
these backend APIs.

Author: Michael Paquier, with doc suggestions from Ashutosh Bapat.
Reviewed-by: Ashutosh Bapat, Nathan Bossart, Álvaro Herrera, Dilip
Kumar, Amul Sul, Nazir Bilal Yavuz
Discussion: https://postgr.es/m/ZTiV8tn_MIb_H2rE@paquier.xyz

configure

@@ -759,6 +759,7 @@ CPPFLAGS
 LDFLAGS
 CFLAGS
 CC
+enable_injection_points
 enable_tap_tests
 enable_dtrace
 DTRACEFLAGS
@@ -839,6 +840,7 @@ enable_profiling
 enable_coverage
 enable_dtrace
 enable_tap_tests
+enable_injection_points
 with_blocksize
 with_segsize
 with_segsize_blocks
@@ -1532,6 +1534,8 @@ Optional Features:
   --enable-coverage       build with coverage testing instrumentation
   --enable-dtrace         build with DTrace support
   --enable-tap-tests      enable TAP tests (requires Perl and IPC::Run)
+  --enable-injection-points
+                          enable injection points (for testing)
   --enable-depend         turn on automatic dependency tracking
   --enable-cassert        enable assertion checks (for debugging)
   --disable-largefile     omit support for large files
@@ -3682,6 +3686,36 @@ fi
 
 
 
+#
+# Injection points
+#
+
+
+# Check whether --enable-injection-points was given.
+if test "${enable_injection_points+set}" = set; then :
+  enableval=$enable_injection_points;
+  case $enableval in
+    yes)
+
+$as_echo "#define USE_INJECTION_POINTS 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --enable-injection-points option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  enable_injection_points=no
+
+fi
+
+
+
+
 #
 # Block size
 #

configure.ac

@@ -250,6 +250,13 @@ PGAC_ARG_BOOL(enable, tap-tests, no,
               [enable TAP tests (requires Perl and IPC::Run)])
 AC_SUBST(enable_tap_tests)
 
+#
+# Injection points
+#
+PGAC_ARG_BOOL(enable, injection-points, no, [enable injection points (for testing)],
+              [AC_DEFINE([USE_INJECTION_POINTS], 1, [Define to 1 to build with injection points. (--enable-injection-points)])])
+AC_SUBST(enable_injection_points)
+
 #
 # Block size
 #

doc/src/sgml/installation.sgml

@@ -1656,6 +1656,21 @@ build-postgresql:
        </listitem>
       </varlistentry>
 
+      <varlistentry id="configure-option-enable-injection-points">
+       <term><option>--enable-injection-points</option></term>
+       <listitem>
+        <para>
+         Compiles <productname>PostgreSQL</productname> with support for
+         injection points in the server. Injection points allow to run
+         user-defined code from within the server in pre-defined code paths.
+         This helps in testing and in the investigation of concurrency scenarios
+         in a controlled fashion. This option is disabled by default.  See
+         <xref linkend="xfunc-addin-injection-points"/> for more details.  This
+         option is intended to be used only by developers for testing.
+        </para>
+       </listitem>
+      </varlistentry>
+
       <varlistentry id="configure-option-with-segsize-blocks">
        <term><option>--with-segsize-blocks=SEGSIZE_BLOCKS</option></term>
        <listitem>
@@ -3160,6 +3175,21 @@ ninja install
       </listitem>
      </varlistentry>
 
+     <varlistentry id="configure-injection-points-meson">
+      <term><option>-Dinjection_points={ true | false }</option></term>
+      <listitem>
+       <para>
+        Compiles <productname>PostgreSQL</productname> with support for
+        injection points in the server. Injection points allow to run
+        user-defined code from within the server in pre-defined code paths.
+        This helps in testing and in the investigation of concurrency scenarios
+        in a controlled fashion. This option is disabled by default.  See
+        <xref linkend="xfunc-addin-injection-points"/> for more details.  This
+        option is intended to be used only by developers for testing.
+       </para>
+      </listitem>
+     </varlistentry>
+
       <varlistentry id="configure-segsize-blocks-meson">
        <term><option>-Dsegsize_blocks=SEGSIZE_BLOCKS</option></term>
        <listitem>

doc/src/sgml/xfunc.sgml

@@ -3599,6 +3599,75 @@ uint32 WaitEventExtensionNew(const char *wait_event_name)
     </para>
    </sect2>
 
+   <sect2 id="xfunc-addin-injection-points">
+    <title>Injection Points</title>
+
+    <para>
+     An injection point with a given <literal>name</literal> is declared using
+     macro:
+<programlisting>
+INJECTION_POINT(name);
+</programlisting>
+
+     There are a few injection points already declared at strategic points
+     within the server code. After adding a new injection point the code needs
+     to be compiled in order for that injection point to be available in the
+     binary. Add-ins written in C-language can declare injection points in
+     their own code using the same macro.
+    </para>
+
+    <para>
+     Add-ins can attach callbacks to an already-declared injection point by
+     calling:
+<programlisting>
+extern void InjectionPointAttach(const char *name,
+                                 const char *library,
+                                 const char *function);
+</programlisting>
+
+     <literal>name</literal> is the name of the injection point, which when
+     reached during execution will execute the <literal>function</literal>
+     loaded from <literal>library</literal>.
+    </para>
+
+    <para>
+     Here is an example of callback for
+     <literal>InjectionPointCallback</literal>:
+<programlisting>
+static void
+custom_injection_callback(const char *name)
+{
+    elog(NOTICE, "%s: executed custom callback", name);
+}
+</programlisting>
+     This callback prints a message to server error log with severity
+     <literal>NOTICE</literal>, but callbacks may implement more complex
+     logic.
+    </para>
+
+    <para>
+     Optionally, it is possible to detach an injection point by calling:
+<programlisting>
+extern void InjectionPointDetach(const char *name);
+</programlisting>
+    </para>
+
+    <para>
+     A callback attached to an injection point is available across all the
+     backends including the backends started after
+     <literal>InjectionPointAttach</literal> is called. It remains attached
+     while the server is running or until the injection point is detached
+     using <literal>InjectionPointDetach</literal>.
+    </para>
+
+    <para>
+     Enabling injections points requires
+     <option>--enable-injection-points</option> with
+     <command>configure</command> or <option>-Dinjection_points=true</option>
+     with <application>Meson</application>.
+    </para>
+   </sect2>
+
    <sect2 id="extend-cpp">
     <title>Using C++ for Extensibility</title>
 

meson.build

@@ -431,6 +431,7 @@ meson_bin = find_program(meson_binpath, native: true)
 ###############################################################
 
 cdata.set('USE_ASSERT_CHECKING', get_option('cassert') ? 1 : false)
+cdata.set('USE_INJECTION_POINTS', get_option('injection_points') ? 1 : false)
 
 blocksize = get_option('blocksize').to_int() * 1024
 

meson_options.txt

@@ -43,6 +43,9 @@ option('cassert', type: 'boolean', value: false,
 option('tap_tests', type: 'feature', value: 'auto',
   description: 'Enable TAP tests')
 
+option('injection_points', type: 'boolean', value: false,
+  description: 'Enable injection points')
+
 option('PG_TEST_EXTRA', type: 'string', value: '',
   description: 'Enable selected extra tests')
 

src/Makefile.global.in

@@ -203,6 +203,7 @@ enable_nls	= @enable_nls@
 enable_debug	= @enable_debug@
 enable_dtrace	= @enable_dtrace@
 enable_coverage	= @enable_coverage@
+enable_injection_points = @enable_injection_points@
 enable_tap_tests	= @enable_tap_tests@
 
 python_includespec	= @python_includespec@

src/backend/storage/ipc/ipci.c

@@ -51,6 +51,7 @@
 #include "storage/sinvaladt.h"
 #include "storage/spin.h"
 #include "utils/guc.h"
+#include "utils/injection_point.h"
 #include "utils/snapmgr.h"
 #include "utils/wait_event.h"
 
@@ -151,6 +152,7 @@ CalculateShmemSize(int *num_semaphores)
 	size = add_size(size, AsyncShmemSize());
 	size = add_size(size, StatsShmemSize());
 	size = add_size(size, WaitEventExtensionShmemSize());
+	size = add_size(size, InjectionPointShmemSize());
 #ifdef EXEC_BACKEND
 	size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -354,6 +356,7 @@ CreateOrAttachShmemStructs(void)
 	AsyncShmemInit();
 	StatsShmemInit();
 	WaitEventExtensionShmemInit();
+	InjectionPointShmemInit();
 }
 
 /*

src/backend/storage/lmgr/lwlocknames.txt

@@ -56,3 +56,4 @@ NotifyQueueTailLock					47
 WaitEventExtensionLock				48
 WALSummarizerLock					49
 DSMRegistryLock						50
+InjectionPointLock				51

src/backend/utils/activity/wait_event_names.txt

@@ -330,6 +330,7 @@ NotifyQueueTail	"Waiting to update limit on <command>NOTIFY</command> message st
 WaitEventExtension	"Waiting to read or update custom wait events information for extensions."
 WALSummarizer	"Waiting to read or update WAL summarization state."
 DSMRegistry	"Waiting to read or update the dynamic shared memory registry."
+InjectionPoint	"Waiting to read or update information related to injection points."
 
 #
 # END OF PREDEFINED LWLOCKS (DO NOT CHANGE THIS LINE)

src/backend/utils/misc/Makefile

@@ -21,6 +21,7 @@ OBJS = \
 	guc_funcs.o \
 	guc_tables.o \
 	help_config.o \
+	injection_point.o \
 	pg_config.o \
 	pg_controldata.o \
 	pg_rusage.o \