Merge branch 'update-clone-image' into 'master'

feat: run clones with custom port and socket dir

See merge request postgres-ai/database-lab!780

Author: agneum

engine/.gitlab-ci.yml

@@ -462,7 +462,7 @@ bash-test-14:
 bash-test-15:
   <<: *bash_test
   variables:
-    POSTGRES_VERSION: 15rc1
+    POSTGRES_VERSION: 15
 
 integration-test:
   services:

engine/internal/provision/docker/docker.go

@@ -52,6 +52,8 @@ func RunContainer(r runners.Runner, c *resources.AppConfig) error {
 
 	unixSocketCloneDir, volumes := createDefaultVolumes(c)
 
+	log.Dbg(fmt.Sprintf("Host info: %#v", hostInfo))
+
 	if hostInfo.VirtualizationRole == "guest" {
 		// Build custom mounts rely on mounts of the Database Lab instance if it's running inside Docker container.
 		// We cannot use --volumes-from because it removes the ZFS mount point.
@@ -78,12 +80,12 @@ func RunContainer(r runners.Runner, c *resources.AppConfig) error {
 		"--detach",
 		"--publish", fmt.Sprintf("%[1]s:%[1]s", instancePort),
 		"--env", "PGDATA=" + c.DataDir(),
+		"--env", "PG_UNIX_SOCKET_DIR=" + unixSocketCloneDir,
+		"--env", "PG_SERVER_PORT=" + instancePort,
 		strings.Join(volumes, " "),
 		fmt.Sprintf("--label %s='%s'", LabelClone, c.Pool.Name),
 		strings.Join(containerFlags, " "),
 		c.DockerImage,
-		"-p", instancePort,
-		"-k", unixSocketCloneDir,
 	}, " ")
 
 	if _, err := r.Run(dockerRunCmd, true); err != nil {
@@ -104,7 +106,7 @@ func createDefaultVolumes(c *resources.AppConfig) (string, []string) {
 
 	// Directly mount PGDATA if Database Lab is running without any virtualization.
 	volumes := []string{
-		fmt.Sprintf("--volume %s:%s", c.DataDir(), c.DataDir()),
+		fmt.Sprintf("--volume %s:%s", c.CloneDir(), c.CloneDir()),
 		fmt.Sprintf("--volume %s:%s", unixSocketCloneDir, unixSocketCloneDir),
 	}
 
@@ -130,7 +132,7 @@ func getMountVolumes(r runners.Runner, c *resources.AppConfig, containerID strin
 
 func buildVolumesFromMountPoints(c *resources.AppConfig, mountPoints []types.MountPoint) []string {
 	unixSocketCloneDir := c.Pool.SocketCloneDir(c.CloneName)
-	mounts := tools.GetMountsFromMountPoints(c.DataDir(), mountPoints)
+	mounts := tools.GetMountsFromMountPoints(c.CloneDir(), mountPoints)
 	volumes := make([]string, 0, len(mounts))
 
 	for _, mountPoint := range mountPoints {

engine/internal/provision/docker/docker_test.go

@@ -61,7 +61,7 @@ func TestVolumesBuilding(t *testing.T) {
 			},
 			expectedVolumes: []string{
 				"--volume /var/lib/dblab/dblab_pool/sockets/dblab_clone_6000:/var/lib/dblab/dblab_pool/sockets/dblab_clone_6000:rshared",
-				"--volume /var/lib/dblab/dblab_pool/clones/dblab_clone_6000/data:/var/lib/dblab/dblab_pool/clones/dblab_clone_6000/data:rshared",
+				"--volume /var/lib/dblab/dblab_pool/clones/dblab_clone_6000:/var/lib/dblab/dblab_pool/clones/dblab_clone_6000:rshared",
 			},
 		},
 	}

engine/internal/provision/resources/appconfig.go

@@ -28,6 +28,12 @@ type DB struct {
 	DBName   string
 }
 
+// CloneDir returns the path of the clone directory.
+func (c *AppConfig) CloneDir() string {
+	// TODO(akartasov): Move to pool.
+	return path.Join(c.Pool.ClonesDir(), c.CloneName)
+}
+
 // DataDir returns the path of clone data.
 func (c *AppConfig) DataDir() string {
 	// TODO(akartasov): Move to pool.

engine/test/1.synthetic.sh

@@ -77,6 +77,7 @@ mkdir -p "${configDir}"
 curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG:-master}"/engine/configs/config.example.logical_generic.yml \
  --output "${configDir}/server.yml"
 
+# TODO: replace the dockerImage tag back to 'postgresai/extended-postgres' after releasing a new version with custom port and unix socket dir.
 # Edit the following options
 yq eval -i '
   .global.debug = true |
@@ -88,7 +89,7 @@ yq eval -i '
   .poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
   del(.retrieval.jobs[] | select(. == "logicalDump")) |
   del(.retrieval.jobs[] | select(. == "logicalRestore")) |
-  .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
+  .databaseContainer.dockerImage = "registry.gitlab.com/postgres-ai/custom-images/extended-postgres:" + strenv(POSTGRES_VERSION)
 ' "${configDir}/server.yml"
 
 # Edit the following options for PostgreSQL 9.6
@@ -99,8 +100,8 @@ if [ "${POSTGRES_VERSION}" = "9.6" ]; then
   ' "${configDir}/server.yml"
 fi
 
-# Edit the following options for PostgreSQL 15beta4
-if [ "${POSTGRES_VERSION}" = "15beta4" ]; then
+# Edit the following options for PostgreSQL 15
+if [ "${POSTGRES_VERSION}" = "15" ]; then
   yq eval -i '
   .databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain, logerrors, pg_stat_kcache" |
   .databaseConfigs.configs.log_directory = "log"

engine/test/2.logical_generic.sh

@@ -98,7 +98,7 @@ yq eval -i '
   .provision.portPool.to = env(DLE_PORT_POOL_TO) |
   .retrieval.spec.logicalDump.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump" |
   .retrieval.spec.logicalRestore.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump" |
-  .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
+  .databaseContainer.dockerImage = "registry.gitlab.com/postgres-ai/custom-images/extended-postgres:" + strenv(POSTGRES_VERSION)
 ' "${configDir}/server.yml"
 
 SHARED_PRELOAD_LIBRARIES="pg_stat_statements, auto_explain, pgaudit, logerrors, pg_stat_kcache"
@@ -112,8 +112,8 @@ if [ "${POSTGRES_VERSION}" = "9.6" ]; then
   SHARED_PRELOAD_LIBRARIES="pg_stat_statements, auto_explain"
 fi
 
-# Edit the following options for PostgreSQL 15beta4
-if [ "${POSTGRES_VERSION}" = "15beta4" ]; then
+# Edit the following options for PostgreSQL 15
+if [ "${POSTGRES_VERSION}" = "15" ]; then
   SHARED_PRELOAD_LIBRARIES="pg_stat_statements, auto_explain, logerrors, pg_stat_kcache"
 fi
 
@@ -175,7 +175,7 @@ PATCH_CONFIG_DATA=$(jq -n -c \
   --arg username "$SOURCE_USERNAME" \
   --arg password "$SOURCE_PASSWORD" \
   --arg spl "$SHARED_PRELOAD_LIBRARIES" \
-  --arg dockerImage "postgresai/extended-postgres:${POSTGRES_VERSION}" \
+  --arg dockerImage "registry.gitlab.com/postgres-ai/custom-images/extended-postgres:${POSTGRES_VERSION}" \
 '{
   "global": {
     "debug": true
@@ -245,7 +245,7 @@ if [[ $(yq eval '.retrieval.spec.logicalDump.options.source.connection.dbname' $
       $(yq eval '.retrieval.spec.logicalDump.options.source.connection.username' ${configDir}/server.yml) != "$SOURCE_USERNAME" ||
       $(yq eval '.retrieval.spec.logicalDump.options.source.connection.password' ${configDir}/server.yml) != "$SOURCE_PASSWORD" ||
       $(yq eval '.retrieval.refresh.timetable' ${configDir}/server.yml) != "5 0 * * 1" ||
-      $(yq eval '.databaseContainer.dockerImage' ${configDir}/server.yml) != "postgresai/extended-postgres:${POSTGRES_VERSION}" ||
+      $(yq eval '.databaseContainer.dockerImage' ${configDir}/server.yml) != "registry.gitlab.com/postgres-ai/custom-images/extended-postgres:${POSTGRES_VERSION}" ||
       $(yq eval '.databaseConfigs.configs.shared_buffers' ${configDir}/server.yml) != "256MB" ]] ; then
   echo "Configuration has not been updated properly"
   exit 1

engine/test/3.physical_walg.sh

@@ -52,7 +52,7 @@ yq eval -i '
   .poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
   .provision.portPool.from = env(DLE_PORT_POOL_FROM) |
   .provision.portPool.to = env(DLE_PORT_POOL_TO) |
-  .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) |
+  .databaseContainer.dockerImage = "registry.gitlab.com/postgres-ai/custom-images/extended-postgres:" + strenv(POSTGRES_VERSION) |
   .retrieval.spec.physicalRestore.options.walg.backupName = strenv(WALG_BACKUP_NAME) |
   .retrieval.spec.physicalRestore.options.sync.configs.shared_buffers = "512MB" |
   .retrieval.spec.physicalSnapshot.options.skipStartSnapshot = true

engine/test/4.physical_basebackup.sh

@@ -110,7 +110,7 @@ yq eval -i '
   .poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
   .provision.portPool.from = env(DLE_PORT_POOL_FROM) |
   .provision.portPool.to = env(DLE_PORT_POOL_TO) |
-  .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) |
+  .databaseContainer.dockerImage = "registry.gitlab.com/postgres-ai/custom-images/extended-postgres:" + strenv(POSTGRES_VERSION) |
   .retrieval.spec.physicalRestore.options.envs.PGUSER = strenv(SOURCE_USERNAME) |
   .retrieval.spec.physicalRestore.options.envs.PGPASSWORD = strenv(SOURCE_PASSWORD) |
   .retrieval.spec.physicalRestore.options.envs.PGHOST = strenv(SOURCE_HOST) |
@@ -128,8 +128,8 @@ if [ "${POSTGRES_VERSION}" = "9.6" ]; then
   ' "${configDir}/server.yml"
 fi
 
-# Edit the following options for PostgreSQL 15beta4
-if [ "${POSTGRES_VERSION}" = "15beta4" ]; then
+# Edit the following options for PostgreSQL 15
+if [ "${POSTGRES_VERSION}" = "15" ]; then
   yq eval -i '
   .databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain, logerrors, pg_stat_kcache"
   ' "${configDir}/server.yml"

engine/test/5.logical_rds.sh

@@ -48,7 +48,7 @@ yq eval -i '
   .poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
   .provision.portPool.from = env(DLE_PORT_POOL_FROM) |
   .provision.portPool.to = env(DLE_PORT_POOL_TO) |
-  .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) |
+  .databaseContainer.dockerImage = "registry.gitlab.com/postgres-ai/custom-images/extended-postgres:" + strenv(POSTGRES_VERSION) |
   .retrieval.spec.logicalDump.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump" |
   .retrieval.spec.logicalDump.options.source.connection.dbname = strenv(SOURCE_DBNAME) |
   .retrieval.spec.logicalDump.options.source.connection.username = strenv(SOURCE_USERNAME) |