Add support for tablespaces (#870)

* Add support for tablespaces

There is a new folder of tests, using volumes to keep data persistant
against pod restarts and to allow for interaction within and without of
the running containers.

An additional Dockerfile has been added based off a local pg14 image.

This adds a newly _destructive_ action removes the data within a tablespace
directory. Perhaps this can be rebased off of the work that will do in-place
resotre before merging, depending on the caution of the core team.

Implementation note: this permits tablespace data directories to
created at the top level of mounted volumes.

Fixes #844

To run tests:
`TEST=tablespaces make test`

Allows for test-tablespaces to be run for all supported postgres versions
`PGVERSION=14 TEST=tablespaces make test`

* Run tablespace tests in travis for pg14

.travis.yml

@@ -7,6 +7,8 @@ cache:
   apt: true
   directories:
   - /home/travis/postgresql
+services:
+  - docker
 matrix:
   fast_finish: true
   include:
@@ -30,6 +32,7 @@ matrix:
     - env: PGVERSION=12 TEST=ssl
     - env: PGVERSION=13 TEST=ssl
     - env: PGVERSION=14 TEST=ssl
+    - env: PGVERSION=14 TEST=tablespaces DOCKERTEST=true
     - env: LINTING=true
 before_install:
   - git clone -b v0.7.18 --depth 1 https://github.com/citusdata/tools.git
@@ -60,6 +63,9 @@ script:
   - 'if [ -n "$LINTING" ]; then citus_indent --check; fi'
   - 'if [ -n "$LINTING" ]; then black --check .; fi'
   - 'if [ -n "$LINTING" ]; then ci/banned.h.sh; fi'
-  - 'if [ -z "$LINTING" ]; then make -j5 CFLAGS=-Werror; fi'
-  - 'if [ -z "$LINTING" ]; then sudo make install; fi'
-  - 'if [ -z "$LINTING" ]; then PATH=`pg_config --bindir`:$PATH make test; fi'
+  - 'if [ -z "$LINTING"] && [ -z "$DOCKERTEST" ]; then make -j5 CFLAGS=-Werror; fi'
+  - 'if [ -z "$LINTING"] && [ -z "$DOCKERTEST" ]; then sudo make install; fi'
+  - 'if [ -z "$LINTING"] && [ -z "$DOCKERTEST" ]; then PATH=`pg_config --bindir`:$PATH make test; fi'
+  - 'if [ -n "$LINTING"] && [ -n "$DOCKERTEST" ]; then make test; fi'
+after_script:
+  - 'if [ -n "$LINTING"] && [ -n "$DOCKERTEST" ]; then make -C tests/tablespaces teardown; fi'

CONTRIBUTING.md

@@ -80,6 +80,41 @@ make TEST=single run-test      # runs tests _not_ matching tests/test_multi*
 make TEST=test_auth run-test   # runs tests/test_auth.py
 ```
 
+#### Running tablespace tests
+
+The tablespace tests are similarly written using Python and the nose framework,
+with as much shared code as possible. They run using
+[docker compose](https://docs.docker.com/compose/), as postgres assumes that
+tablespaces live in the same location across replicas. This necessitates
+matching directory structures across the nodes, and thus, multiple,
+simultaneously running containers.
+
+Interaction with each node is done using `docker-compose` commands. Refer to
+the [Makefile](tests/tablespaces/Makefile) in the test directory for examples.
+
+To run the tests from the top-level directory:
+
+```bash
+TEST=tablespaces make test
+```
+
+Like the other tests, you can use `PGVERSION` to test against supported versions
+of postgres, for example:
+
+```bash
+PGVERSION=14 TEST=tablespaces make test
+```
+
+If the tests fail, the docker containers may be left around, and there is a
+companion teardown target:
+
+```bash
+make -C tests/tablespaces teardown
+```
+
+Refer to the [Makefile](tests/tablespaces/Makefile) for more potentially
+useful targets to use while developing with tablespaces.
+
 ### Producing the documentation diagrams
 
 The diagrams are TikZ sources, which means they're edited with your usual

Makefile

@@ -132,14 +132,19 @@ clean-bin:
 install-bin: bin
 	$(MAKE) -C src/bin/ install
 
+
 test:
+ifeq ($(TEST),tablespaces)
+	$(MAKE) -C tests/tablespaces run-test
+else
 	sudo -E env "PATH=${PATH}" USER=$(shell whoami) \
 		$(NOSETESTS)			\
 		--verbose				\
 		--nologcapture			\
 		--nocapture				\
 		--stop					\
 		${TEST_ARGUMENT}
+endif
 
 indent:
 	citus_indent

src/bin/pg_autoctl/pgctl.c

@@ -83,7 +83,7 @@ static bool pg_write_recovery_conf(const char *pgdata,
 								   ReplicationSource *replicationSource);
 static bool pg_write_standby_signal(const char *pgdata,
 									ReplicationSource *replicationSource);
-
+static bool ensure_empty_tablespace_dirs(const char *pgdata);
 
 /*
  * Get pg_ctl --version output in pgSetup->pg_version.
@@ -1172,6 +1172,83 @@ prepare_guc_settings_from_pgsetup(const char *configFilePath,
 }
 
 
+bool
+ensure_empty_tablespace_dirs(const char *pgdata)
+{
+	char *dirName = "pg_tblspc";
+	struct dirent *dirEntry = NULL;
+
+	char pgTblspcFullPath[MAXPGPATH] = { 0 };
+	sformat(pgTblspcFullPath, MAXPGPATH, "%s/%s", pgdata, dirName);
+
+	if (!directory_exists(pgTblspcFullPath))
+	{
+		log_debug("Postgres dir pg_tblspc does not exist at \"%s\"",
+				  pgTblspcFullPath);
+		return true;
+	}
+
+
+	/* open and scan through the Postgres tablespace directory */
+	DIR *tblspcDir = opendir(pgTblspcFullPath);
+
+	if (tblspcDir == NULL)
+	{
+		log_error("Failed to open Postgres tablespace directory \"%s\" at \"%s\"",
+				  dirName, pgdata);
+		return false;
+	}
+
+	while ((dirEntry = readdir(tblspcDir)) != NULL)
+	{
+		char tblspcDataDirPath[MAXPGPATH] = { 0 };
+
+		/* skip non-symlinks, as all tablespace dirs are symlinks */
+		if (dirEntry->d_type == DT_UNKNOWN)
+		{
+			struct stat structStat;
+			char dirEntryFullPath[MAXPGPATH] = { 0 };
+			sformat(dirEntryFullPath, MAXPGPATH, "%s/%s", pgTblspcFullPath,
+					dirEntry->d_name);
+			if (lstat(dirEntryFullPath, &structStat) != 0)
+			{
+				log_error("Failed to get file information for \"%s/%s\": %m",
+						  pgTblspcFullPath, dirEntry->d_name);
+				return false;
+			}
+			if (!S_ISLNK(structStat.st_mode))
+			{
+				log_debug("Non-symlink file found in tablespace directory: \"%s/%s\"",
+						  pgTblspcFullPath, dirEntry->d_name);
+				continue;
+			}
+		}
+		else if (dirEntry->d_type != DT_LNK)
+		{
+			log_debug("Non-symlink file found in tablespace directory: \"%s/%s\"",
+					  pgTblspcFullPath, dirEntry->d_name);
+			continue;
+		}
+
+		join_path_components(tblspcDataDirPath, pgTblspcFullPath, dirEntry->d_name);
+
+		log_debug("Removing contents of tablespace data directory \"%s\"",
+				  tblspcDataDirPath);
+
+		/* remove contents of tablespace data directory */
+		if (!rmtree(tblspcDataDirPath, false))
+		{
+			log_error(
+				"Failed to remove contents of existing tablespace data directory \"%s\": %m",
+				tblspcDataDirPath);
+			return false;
+		}
+	}
+
+	return true;
+}
+
+
 /*
  * Call pg_basebackup, using a temporary directory for the duration of the data
  * transfer.
@@ -1199,6 +1276,12 @@ pg_basebackup(const char *pgdata,
 		return false;
 	}
 
+	if (!ensure_empty_tablespace_dirs(pgdata))
+	{
+		/* errors have already been logged. */
+		return false;
+	}
+
 	/* call pg_basebackup */
 	path_in_same_directory(pg_ctl, "pg_basebackup", pg_basebackup);
 

tests/__init__.py

@@ -0,0 +1,2 @@
+# Initialize the tests package
+# https://docs.python.org/3/tutorial/modules.html#packages