Merge branch 'release_2_4' into issue_66

gsmolk · gsmolk · commit e9d950f7d2c5 · 2020-06-08T20:08:07.000+03:00
diff --git a/README.md b/README.md
@@ -54,15 +54,15 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
 
 ## Current release
 
-[2.3.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.4)
+[2.3.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.5)
 
 ## Documentation
 
 Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup)
 
 ## Installation and Setup
 ### Windows Installation
-Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.3).
+Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.5).
 
 ### Linux Installation
 #### pg_probackup for vanilla PostgreSQL
@@ -96,64 +96,64 @@ yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
 #SRPM Packages
 yumdownloader --source pg_probackup-{12,11,10,9.6,9.5}
 
-#RPM ALT Linux p7
+#RPM ALT Linux 7
 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list
 sudo apt-get update
 sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}
 sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
 
-#RPM ALT Linux p8
+#RPM ALT Linux 8
 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list
 sudo apt-get update
 sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}
 sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
 
-#RPM ALT Linux p9
+#RPM ALT Linux 9
 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list
 sudo apt-get update
 sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}
 sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo
 ```
 
-#### pg_probackup for PostgresPro Standart and Enterprise
+#### pg_probackup for PostgresPro Standard and Enterprise
 ```shell
 #DEB Ubuntu|Debian Packages
 sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list
 sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update
-sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}
-sudo apt-get install pg-probackup-{std,ent}-{11,10,9.6}-dbg
+sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}
+sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg
 
 #RPM Centos Packages
 rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm
-yum install pg_probackup-{std,ent}-{11,10,9.6}
-yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
+yum install pg_probackup-{std,ent}-{12,11,10,9.6}
+yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
 
 #RPM RHEL Packages
 rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm
-yum install pg_probackup-{std,ent}-{11,10,9.6}
-yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
+yum install pg_probackup-{std,ent}-{12,11,10,9.6}
+yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
 
 #RPM Oracle Linux Packages
 rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm
-yum install pg_probackup-{std,ent}-{11,10,9.6}
-yum install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
+yum install pg_probackup-{std,ent}-{12,11,10,9.6}
+yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
 
-#RPM ALT Linux p7
+#RPM ALT Linux 7
 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list
 sudo apt-get update
-sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}
-sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
+sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}
+sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
 
-#RPM ALT Linux p8
+#RPM ALT Linux 8
 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list
 sudo apt-get update
-sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}
-sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
+sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}
+sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
 
-#RPM ALT Linux p9
+#RPM ALT Linux 9
 sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update
-sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}
-sudo apt-get install pg_probackup-{std,ent}-{11,10,9.6}-debuginfo
+sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}
+sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo
 ```
 
 Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup).
diff --git a/src/backup.c b/src/backup.c
@@ -1855,7 +1855,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 		}
 
 		elog(LOG, "stop_lsn: %X/%X",
-			(uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn));
+			(uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));
 
 		/* Write backup_label and tablespace_map */
 		if (!exclusive_backup)
diff --git a/src/init.c b/src/init.c
@@ -67,28 +67,31 @@ do_add_instance(InstanceConfig *instance)
 
 	/* Ensure that all root directories already exist */
 	if (access(backup_path, F_OK) != 0)
-		elog(ERROR, "%s directory does not exist.", backup_path);
+		elog(ERROR, "Directory does not exist: '%s'", backup_path);
 
 	join_path_components(path, backup_path, BACKUPS_DIR);
 	if (access(path, F_OK) != 0)
-		elog(ERROR, "%s directory does not exist.", path);
+		elog(ERROR, "Directory does not exist: '%s'", path);
 
 	join_path_components(arclog_path_dir, backup_path, "wal");
 	if (access(arclog_path_dir, F_OK) != 0)
-		elog(ERROR, "%s directory does not exist.", arclog_path_dir);
+		elog(ERROR, "Directory does not exist: '%s'", arclog_path_dir);
 
-	/* Create directory for data files of this specific instance */
 	if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode))
-		elog(ERROR, "instance '%s' already exists", instance->backup_instance_path);
-	dir_create_dir(instance->backup_instance_path, DIR_PERMISSION);
+		elog(ERROR, "Instance '%s' backup directory already exists: '%s'",
+			instance->name, instance->backup_instance_path);
 
 	/*
 	 * Create directory for wal files of this specific instance.
 	 * Existence check is extra paranoid because if we don't have such a
 	 * directory in data dir, we shouldn't have it in wal as well.
 	 */
 	if (stat(instance->arclog_path, &st) == 0 && S_ISDIR(st.st_mode))
-		elog(ERROR, "arclog_path '%s' already exists", instance->arclog_path);
+		elog(ERROR, "Instance '%s' WAL archive directory already exists: '%s'",
+				instance->name, instance->arclog_path);
+
+	/* Create directory for data files of this specific instance */
+	dir_create_dir(instance->backup_instance_path, DIR_PERMISSION);
 	dir_create_dir(instance->arclog_path, DIR_PERMISSION);
 
 	/*
diff --git a/src/validate.c b/src/validate.c
@@ -58,13 +58,19 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
 	int			i;
 //	parray		*dbOid_exclude_list = NULL;
 
-	/* Check backup version */
+	/* Check backup program version */
 	if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION))
 		elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
 			"pg_probackup do not guarantee to be forward compatible. "
 			"Please upgrade pg_probackup binary.",
 				PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version);
 
+	/* Check backup server version */
+	if (strcmp(backup->server_version, PG_MAJORVERSION) != 0)
+        elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary "
+					"compiled with server version %s",
+                base36enc(backup->start_time), backup->server_version, PG_MAJORVERSION);
+
 	if (backup->status == BACKUP_STATUS_RUNNING)
 	{
 		elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation",
diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out
@@ -1 +1 @@
-pg_probackup 2.3.4
+pg_probackup 2.3.5
diff --git a/tests/init.py b/tests/init.py
@@ -1,6 +1,7 @@
 import os
 import unittest
 from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException
+import shutil
 
 
 module_name = 'init'
@@ -104,3 +105,53 @@ def test_abs_path(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    # @unittest.expectedFailure
+    def test_add_instance_idempotence(self):
+        """
+        https://github.com/postgrespro/pg_probackup/issues/219
+        """
+        fname = self.id().split(".")[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'))
+        self.init_pb(backup_dir)
+
+        self.add_instance(backup_dir, 'node', node)
+        shutil.rmtree(os.path.join(backup_dir, 'backups', 'node'))
+
+        dir_backups = os.path.join(backup_dir, 'backups', 'node')
+        dir_wal = os.path.join(backup_dir, 'wal', 'node')
+
+        try:
+            self.add_instance(backup_dir, 'node', node)
+            # we should die here because exception is what we expect to happen
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because page backup should not be possible "
+                "\n Output: {0} \n CMD: {1}".format(
+                    repr(self.output), self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Instance 'node' WAL archive directory already exists: ",
+                e.message,
+                "\n Unexpected Error Message: {0}\n CMD: {1}".format(
+                    repr(e.message), self.cmd))
+
+        try:
+            self.add_instance(backup_dir, 'node', node)
+            # we should die here because exception is what we expect to happen
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because page backup should not be possible "
+                "\n Output: {0} \n CMD: {1}".format(
+                    repr(self.output), self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Instance 'node' WAL archive directory already exists: ",
+                e.message,
+                "\n Unexpected Error Message: {0}\n CMD: {1}".format(
+                    repr(e.message), self.cmd))
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
diff --git a/tests/merge.py b/tests/merge.py
@@ -3,6 +3,7 @@
 import unittest
 import os
 from .helpers.ptrack_helpers import ProbackupTest, ProbackupException
+from testgres import QueryException
 import shutil
 from datetime import datetime, timedelta
 import time
@@ -175,18 +176,15 @@ def test_merge_compressed_backups_1(self):
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
             set_replication=True, initdb_params=["--data-checksums"],
-            pg_options={
-                'autovacuum': 'off'
-            }
-        )
+            pg_options={'autovacuum': 'off'})
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, "node", node)
         self.set_archiving(backup_dir, "node", node)
         node.slow_start()
 
         # Fill with data
-        node.pgbench_init(scale=5)
+        node.pgbench_init(scale=10)
 
         # Do compressed FULL backup
         self.backup_node(backup_dir, "node", node, options=['--compress', '--stream'])
@@ -196,7 +194,7 @@ def test_merge_compressed_backups_1(self):
         self.assertEqual(show_backup["backup-mode"], "FULL")
 
         # Change data
-        pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum'])
+        pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum'])
         pgbench.wait()
 
         # Do compressed DELTA backup
@@ -205,7 +203,7 @@ def test_merge_compressed_backups_1(self):
             backup_type="delta", options=['--compress', '--stream'])
 
         # Change data
-        pgbench = node.pgbench(options=['-T', '20', '-c', '2', '--no-vacuum'])
+        pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum'])
         pgbench.wait()
 
         # Do compressed PAGE backup
@@ -1478,10 +1476,12 @@ def test_crash_after_opening_backup_control_1(self):
 
         self.del_test_dir(module_name, fname)
 
+    @unittest.skip("skip")
     def test_crash_after_opening_backup_control_2(self):
         """
         check that crashing after opening backup_content.control
         for writing will not result in losing metadata about backup files
+        TODO: rewrite
         """
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -1531,7 +1531,7 @@ def test_crash_after_opening_backup_control_2(self):
         gdb.run_until_break()
 
         gdb.set_breakpoint('sprintf')
-        gdb.continue_execution_until_break(20)
+        gdb.continue_execution_until_break(1)
 
         gdb._execute('signal SIGKILL')
 
@@ -1568,10 +1568,12 @@ def test_crash_after_opening_backup_control_2(self):
 
         self.del_test_dir(module_name, fname)
 
+    @unittest.skip("skip")
     def test_losing_file_after_failed_merge(self):
         """
         check that crashing after opening backup_content.control
         for writing will not result in losing metadata about backup files
+        TODO: rewrite
         """
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
diff --git a/tests/validate.py b/tests/validate.py
@@ -3716,6 +3716,59 @@ def test_partial_validate_include(self):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
+    # @unittest.skip("skip")
+    def test_not_validate_diffenent_pg_version(self):
+        """Do not validate backup, if binary is compiled with different PG version"""
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            initdb_params=['--data-checksums'])
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        backup_id = self.backup_node(backup_dir, 'node', node)
+
+        control_file = os.path.join(
+            backup_dir, "backups", "node", backup_id,
+            "backup.control")
+
+        pg_version = node.major_version
+
+        if pg_version.is_integer():
+            pg_version = int(pg_version)
+
+        fake_new_pg_version = pg_version + 1
+
+        with open(control_file, 'r') as f:
+            data = f.read();
+
+        data = data.replace(str(pg_version), str(fake_new_pg_version))
+
+        with open(control_file, 'w') as f:
+            f.write(data);
+
+        try:
+            self.validate_pb(backup_dir)
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because validation is forbidden if server version of backup "
+                "is different from the server version of pg_probackup.\n Output: {0} \n CMD: {1}".format(
+                    repr(self.output), self.cmd))
+        except ProbackupException as e:
+            self.assertIn(
+                "ERROR: Backup {0} has server version".format(backup_id),
+                e.message,
+                "\n Unexpected Error Message: {0}\n CMD: {1}".format(
+                    repr(e.message), self.cmd))
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)
+
+
 # validate empty backup list
 # page from future during validate
 # page from future during backup
@@ -3727,4 +3780,4 @@ def test_partial_validate_include(self):
 # 715                             MAXALIGN(header.compressed_size), in);
 # 716                     if (read_len != MAXALIGN(header.compressed_size))
 # -> 717                             elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d",
-# 718                                     blknum, file->path, read_len, header.compressed_size);
+# 718                                     blknum, file->path, read_len, header.compressed_size);

Original file line number	Diff line number	Diff line change
`@@ -1855,7 +1855,7 @@ pg_stop_backup(pgBackup backup, PGconn pg_startbackup_conn,`
`1855`	`1855`	`}`
`1856`	`1856`
`1857`	`1857`	`elog(LOG, "stop_lsn: %X/%X",`
`1858`		`- (uint32) (stop_backup_lsn >> 32), (uint32) (stop_backup_lsn));`
	`1858`	`+ (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp));`
`1859`	`1859`
`1860`	`1860`	`/* Write backup_label and tablespace_map */`
`1861`	`1861`	`if (!exclusive_backup)`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-pg_probackup 2.3.4`
	`1`	`+pg_probackup 2.3.5`