Implement make compose, using docker-compose for local interactive te…

…sting. (#825)

* Implement make compose.

Similar to `make cluster`, this allows to start a local cluster to play with
multiple Postgres nodes registered in a pg_auto_failover monitor. This time
the command uses docker-compose to run all the nodes in the background, and
each node has its own pre-created external docker volume.

* Use --build-arg PGVERSION=... in make compose.

* Introduce another needed wait point in the tmux script.

* Clean-up: we don't need both --name and PG_AUTOCTL_NODE_NAME.

* Ignore some files for docker

Co-authored-by: Jelte Fennema <github-tech@jeltef.nl>

.dockerignore

@@ -2,3 +2,60 @@
 src/bin/pg_autoctl/pg_autoctl
 docs
 Dockerfile
+
+# Global excludes across all subdirectories
+*.o
+*.bc
+*.so
+*.so.[0-9]
+*.so.[0-9].[0-9]
+*.sl
+*.sl.[0-9]
+*.sl.[0-9].[0-9]
+*.dylib
+*.dll
+*.a
+*.mo
+*.pot
+objfiles.txt
+.deps/
+*.gcno
+*.gcda
+*.gcov
+*.gcov.out
+lcov.info
+coverage/
+*.vcproj
+*.vcxproj
+win32ver.rc
+*.exe
+lib*dll.def
+lib*.pc
+*.log
+
+# Local excludes in root directory
+/config.log
+/config.status
+/pgsql.sln
+/pgsql.sln.cache
+/Debug/
+/Release/
+/autom4te.cache
+/Makefile.global
+/src/Makefile.custom
+/tests/__pycache__/
+/env/
+
+# Exclude generated SQL files
+pgautofailover--?.?.sql
+!pgautofailover--1.0.sql
+
+# Exclude generated PDF and PNG files for the graphics
+docs/tikz/*.pdf
+docs/tikz/*.png
+
+# Exclude our demo/test tmux directory
+tmux/
+valgrind/*
+
+.ccls-cache/

.gitignore

@@ -26,6 +26,7 @@ win32ver.rc
 *.exe
 lib*dll.def
 lib*.pc
+*.log
 
 # Local excludes in root directory
 /config.log

Dockerfile

@@ -50,8 +50,6 @@ RUN apt-get update \
     postgresql-common \
 	&& rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install pyroute2>=0.5.17
-
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list
 
@@ -72,6 +70,7 @@ RUN apt-get update \
 	&& rm -rf /var/lib/apt/lists/*
 
 RUN pip3 install pyroute2>=0.5.17
+
 RUN adduser --disabled-password --gecos '' docker
 RUN adduser docker sudo
 RUN adduser docker postgres

Makefile

@@ -261,12 +261,27 @@ tmux-session: bin
          --binpath $(BINPATH)             \
          --layout $(TMUX_LAYOUT)
 
+tmux-compose-session:
+	$(PG_AUTOCTL) do tmux compose session \
+         --root $(TMUX_TOP_DIR)           \
+         --first-pgport $(FIRST_PGPORT)   \
+         --nodes $(NODES)                 \
+         --async-nodes $(NODES_ASYNC)     \
+         --node-priorities $(NODES_PRIOS) \
+         --sync-standbys $(NODES_SYNC_SB) \
+         $(CLUSTER_OPTS)                  \
+         --binpath $(BINPATH)             \
+         --layout $(TMUX_LAYOUT)
+
 cluster: install tmux-clean
 	# This is explicitly not a target, otherwise when make uses multiple jobs
 	# tmux-clean and tmux-session can have a race condidition where tmux-clean
 	# removes the files that are just created by tmux-session.
 	$(MAKE) tmux-session
 
+compose:
+	$(MAKE) tmux-compose-session
+
 valgrind-session: build-test
 	docker run                             \
 	    --name $(TEST_CONTAINER_NAME) 	   \
@@ -302,7 +317,7 @@ azdrop: all
 .PHONY: monitor clean-monitor check-monitor install-monitor
 .PHONY: bin clean-bin install-bin
 .PHONY: build-test run-test
-.PHONY: tmux-clean cluster
+.PHONY: tmux-clean cluster compose
 .PHONY: azcluster azdrop az
 .PHONY: build-image
 .PHONY: build-test-pg10 build-test-pg11 build-test-pg142

src/bin/pg_autoctl/cli_do_root.c

@@ -221,6 +221,66 @@ CommandLine do_pgsetup_commands =
 					 "Manage a local Postgres setup", NULL, NULL,
 					 NULL, do_pgsetup);
 
+CommandLine do_tmux_compose_config =
+	make_command("config",
+				 "Produce a docker-compose configuration file for a demo",
+				 "[option ...]",
+				 "  --root            path where to create a cluster\n"
+				 "  --first-pgport    first Postgres port to use (5500)\n"
+				 "  --nodes           number of Postgres nodes to create (2)\n"
+				 "  --async-nodes     number of async nodes within nodes (0)\n"
+				 "  --node-priorities list of nodes priorities (50)\n"
+				 "  --sync-standbys   number-sync-standbys to set (0 or 1)\n"
+				 "  --skip-pg-hba     use --skip-pg-hba when creating nodes\n"
+				 "  --layout          tmux layout to use (even-vertical)\n"
+				 "  --binpath         path to the pg_autoctl binary (current binary path)",
+				 cli_do_tmux_script_getopts,
+				 cli_do_tmux_compose_config);
+
+CommandLine do_tmux_compose_script =
+	make_command("script",
+				 "Produce a tmux script for a demo or a test case (debug only)",
+				 "[option ...]",
+				 "  --root            path where to create a cluster\n"
+				 "  --first-pgport    first Postgres port to use (5500)\n"
+				 "  --nodes           number of Postgres nodes to create (2)\n"
+				 "  --async-nodes     number of async nodes within nodes (0)\n"
+				 "  --node-priorities list of nodes priorities (50)\n"
+				 "  --sync-standbys   number-sync-standbys to set (0 or 1)\n"
+				 "  --skip-pg-hba     use --skip-pg-hba when creating nodes\n"
+				 "  --layout          tmux layout to use (even-vertical)\n"
+				 "  --binpath         path to the pg_autoctl binary (current binary path)",
+				 cli_do_tmux_script_getopts,
+				 cli_do_tmux_compose_script);
+
+CommandLine do_tmux_compose_session =
+	make_command("session",
+				 "Run a tmux session for a demo or a test case",
+				 "[option ...]",
+				 "  --root            path where to create a cluster\n"
+				 "  --first-pgport    first Postgres port to use (5500)\n"
+				 "  --nodes           number of Postgres nodes to create (2)\n"
+				 "  --async-nodes     number of async nodes within nodes (0)\n"
+				 "  --node-priorities list of nodes priorities (50)\n"
+				 "  --sync-standbys   number-sync-standbys to set (0 or 1)\n"
+				 "  --skip-pg-hba     use --skip-pg-hba when creating nodes\n"
+				 "  --layout          tmux layout to use (even-vertical)\n"
+				 "  --binpath         path to the pg_autoctl binary (current binary path)",
+				 cli_do_tmux_script_getopts,
+				 cli_do_tmux_compose_session);
+
+CommandLine *do_tmux_compose[] = {
+	&do_tmux_compose_config,
+	&do_tmux_compose_script,
+	&do_tmux_compose_session,
+	NULL
+};
+
+CommandLine do_tmux_compose_commands =
+	make_command_set("compose",
+					 "Set of facilities to handle docker-compose sessions",
+					 NULL, NULL, NULL, do_tmux_compose);
+
 CommandLine do_tmux_script =
 	make_command("script",
 				 "Produce a tmux script for a demo or a test case (debug only)",
@@ -290,6 +350,7 @@ CommandLine do_tmux_wait =
 				 cli_do_tmux_wait);
 
 CommandLine *do_tmux[] = {
+	&do_tmux_compose_commands,
 	&do_tmux_script,
 	&do_tmux_session,
 	&do_tmux_stop,

src/bin/pg_autoctl/cli_do_root.h

@@ -109,6 +109,11 @@ void cli_do_tmux_stop(int argc, char **argv);
 void cli_do_tmux_clean(int argc, char **argv);
 void cli_do_tmux_wait(int argc, char **argv);
 
+/* src/bin/pg_autoctl/cli_do_tmux_compose.c */
+void cli_do_tmux_compose_config(int argc, char **argv);
+void cli_do_tmux_compose_script(int argc, char **argv);
+void cli_do_tmux_compose_session(int argc, char **argv);
+
 /* src/bin/pg_autoctl/cli_do_azure.c */
 int cli_do_azure_getopts(int argc, char **argv);
 void cli_do_azure_create_environment(int argc, char **argv);

src/bin/pg_autoctl/cli_do_tmux.c

@@ -1,5 +1,5 @@
 /*
- * src/bin/pg_autoctl/cli_do_misc.c
+ * src/bin/pg_autoctl/cli_do_tmux.c
  *     Implementation of a CLI which lets you run operations on the local
  *     postgres server directly.
  *

src/bin/pg_autoctl/cli_do_tmux.h

@@ -56,6 +56,7 @@ typedef struct TmuxNodeArray
 	TmuxNode nodes[MAX_NODES];
 } TmuxNodeArray;
 
+extern char *tmux_banner[];
 extern TmuxOptions tmuxOptions;
 extern TmuxNodeArray tmuxNodeArray;