Merge branch 'release_2_4' into issue_205

Author: gsmolk

README.md

@@ -1,4 +1,5 @@
 [![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup)
+[![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest)
 
 # pg_probackup
 

doc/pgprobackup.xml

@@ -1168,20 +1168,20 @@ CREATE EXTENSION ptrack;
       </listitem>
       <listitem>
       <para>
-       To enable tracking page updates, set <varname>ptrack_map_size</varname>
+       To enable tracking page updates, set <varname>ptrack.map_size</varname>
        parameter to a positive integer and restart the server.
       </para>
       <para>
        For optimal performance, it is recommended to set
-       <varname>ptrack_map_size</varname> to
+       <varname>ptrack.map_size</varname> to
        <literal><replaceable>N</replaceable> / 1024</literal>, where
        <replaceable>N</replaceable> is the size of the
        <productname>PostgreSQL</productname> cluster, in MB. If you set this
        parameter to a lower value, PTRACK is more likely to map several blocks
        together, which leads to false-positive results when tracking changed
        blocks and increases the incremental backup size as unchanged blocks
        can also be copied into the incremental backup.
-       Setting <varname>ptrack_map_size</varname> to a higher value
+       Setting <varname>ptrack.map_size</varname> to a higher value
        does not affect PTRACK operation. The maximum allowed value is 1024.
       </para>
      </listitem>
@@ -1201,11 +1201,11 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup;
 
    <note>
     <para>
-     If you change the <varname>ptrack_map_size</varname> parameter value,
+     If you change the <varname>ptrack.map_size</varname> parameter value,
      the previously created PTRACK map file is cleared,
      and tracking newly changed blocks starts from scratch. Thus, you have
      to retake a full backup before taking incremental PTRACK backups after
-     changing <varname>ptrack_map_size</varname>.
+     changing <varname>ptrack.map_size</varname>.
     </para>
    </note>
 

src/backup.c

@@ -435,10 +435,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 			/*
 			 * Build the page map from ptrack information.
 			 */
-			if (nodeInfo->ptrack_version_num == 20)
+			if (nodeInfo->ptrack_version_num >= 20)
 				make_pagemap_from_ptrack_2(backup_files_list, backup_conn,
-											nodeInfo->ptrack_schema,
-											prev_backup_start_lsn);
+										   nodeInfo->ptrack_schema,
+										   nodeInfo->ptrack_version_num,
+										   prev_backup_start_lsn);
 			else if (nodeInfo->ptrack_version_num == 15 ||
 					 nodeInfo->ptrack_version_num == 16 ||
 					 nodeInfo->ptrack_version_num == 17)
@@ -862,15 +863,10 @@ do_backup(time_t start_time, bool no_validate,
 #endif
 
 	get_ptrack_version(backup_conn, &nodeInfo);
-//	elog(WARNING, "ptrack_version_num %d", ptrack_version_num);
+	//	elog(WARNING, "ptrack_version_num %d", ptrack_version_num);
 
 	if (nodeInfo.ptrack_version_num > 0)
-	{
-		if (nodeInfo.ptrack_version_num >= 20)
-			nodeInfo.is_ptrack_enable = pg_ptrack_enable2(backup_conn);
-		else
-			nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn);
-	}
+		nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num);
 
 	if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
 	{

src/data.c

@@ -408,15 +408,12 @@ prepare_page(ConnectionArgs *conn_arg,
 		}
 	}
 
-	/* Get page via ptrack interface from PostgreSQL shared buffer.
-	 * We do this in following cases:
-	 * 1. PTRACK backup of 1.x versions
-	 * 2. During backup, regardless of backup mode, of PostgreSQL instance
-	 *    with ptrack support we encountered invalid page.
+	/*
+	 * Get page via ptrack interface from PostgreSQL shared buffer.
+	 * We do this only in the cases of PTRACK 1.x versions backup
 	 */
-	if ((backup_mode == BACKUP_MODE_DIFF_PTRACK
+	if (backup_mode == BACKUP_MODE_DIFF_PTRACK
 		&& (ptrack_version_num >= 15 && ptrack_version_num < 20))
-			|| !page_is_valid)
 	{
 		int rc = 0;
 		size_t page_size = 0;
@@ -440,7 +437,8 @@ prepare_page(ConnectionArgs *conn_arg,
 		memcpy(page, ptrack_page, BLCKSZ);
 		pg_free(ptrack_page);
 
-		/* UPD: It apprears that is possible to get zeroed page or page with invalid header
+		/*
+		 * UPD: It apprears that is possible to get zeroed page or page with invalid header
 		 * from shared buffer.
 		 * Note, that getting page with wrong checksumm from shared buffer is
 		 * acceptable.
@@ -462,7 +460,8 @@ prepare_page(ConnectionArgs *conn_arg,
 								from_fullpath, blknum, errormsg);
 		}
 
-		/* We must set checksum here, because it is outdated
+		/*
+		 * We must set checksum here, because it is outdated
 		 * in the block recieved from shared buffers.
 		 */
 		if (checksum_version)

src/pg_probackup.h

@@ -1009,19 +1009,21 @@ extern void parse_filelist_filenames(parray *files, const char *root);
 /* in ptrack.c */
 extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn);
 extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn,
-										const char *ptrack_schema, XLogRecPtr lsn);
+									   const char *ptrack_schema,
+									   int ptrack_version_num,
+									   XLogRecPtr lsn);
 extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num);
 extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo);
-extern bool pg_ptrack_enable(PGconn *backup_conn);
-extern bool pg_ptrack_enable2(PGconn *backup_conn);
+extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num);
 extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn);
 extern char *pg_ptrack_get_and_clear(Oid tablespace_oid,
 									 Oid db_oid,
 									 Oid rel_oid,
 									 size_t *result_size,
 									 PGconn *backup_conn);
 extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo);
-extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn);
+extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema,
+										 int ptrack_version_num, XLogRecPtr lsn);
 
 /* FIO */
 extern int fio_send_pages(FILE* out, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn,

src/ptrack.c

@@ -199,6 +199,8 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo)
 		nodeInfo->ptrack_version_num = 17;
 	else if (strcmp(ptrack_version_str, "2.0") == 0)
 		nodeInfo->ptrack_version_num = 20;
+	else if (strcmp(ptrack_version_str, "2.1") == 0)
+		nodeInfo->ptrack_version_num = 21;
 	else
 		elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s",
 			 ptrack_version_str);
@@ -210,19 +212,30 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo)
  * Check if ptrack is enabled in target instance
  */
 bool
-pg_ptrack_enable(PGconn *backup_conn)
+pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num)
 {
 	PGresult   *res_db;
+	bool		result = false;
 
-	res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL);
-
-	if (strcmp(PQgetvalue(res_db, 0, 0), "on") != 0)
+	if (ptrack_version_num < 20)
 	{
-		PQclear(res_db);
-		return false;
+		res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL);
+		result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0;
+	}
+	else if (ptrack_version_num == 20)
+	{
+		res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL);
+		result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0;
+	}
+	else
+	{
+		res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL);
+		result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 &&
+				 strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0;
 	}
+
 	PQclear(res_db);
-	return true;
+	return result;
 }
 
 
@@ -452,7 +465,11 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo)
 	{
 		char query[128];
 
-		sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema);
+		if (nodeInfo->ptrack_version_num == 20)
+			sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema);
+		else
+			sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema);
+
 		res = pgut_execute(backup_conn, query, 0, NULL);
 	}
 
@@ -504,7 +521,7 @@ pg_ptrack_get_block(ConnectionArgs *arguments,
 	if (arguments->cancel_conn == NULL)
 		arguments->cancel_conn = PQgetCancel(arguments->conn);
 
-	//elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum);
+	// elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum);
 
 	if (ptrack_version_num < 20)
 		res = pgut_execute_parallel(arguments->conn,
@@ -519,7 +536,11 @@ pg_ptrack_get_block(ConnectionArgs *arguments,
 		if (!ptrack_schema)
 			elog(ERROR, "Schema name of ptrack extension is missing");
 
-		sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema);
+		if (ptrack_version_num == 20)
+			sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema);
+		else
+			elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()");
+			// sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema);
 
 		res = pgut_execute_parallel(arguments->conn,
 									arguments->cancel_conn,
@@ -559,30 +580,12 @@ pg_ptrack_get_block(ConnectionArgs *arguments,
  * ----------------------------
  */
 
-/*
- * Check if ptrack is enabled in target instance
- */
-bool
-pg_ptrack_enable2(PGconn *backup_conn)
-{
-	PGresult   *res_db;
-
-	res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL);
-
-	if (strcmp(PQgetvalue(res_db, 0, 0), "0") == 0)
-	{
-		PQclear(res_db);
-		return false;
-	}
-	PQclear(res_db);
-	return true;
-}
-
 /*
  * Fetch a list of changed files with their ptrack maps.
  */
 parray *
-pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRecPtr lsn)
+pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema,
+						 int ptrack_version_num, XLogRecPtr lsn)
 {
 	PGresult   *res;
 	char		lsn_buf[17 + 1];
@@ -597,8 +600,12 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec
 	if (!ptrack_schema)
 		elog(ERROR, "Schema name of ptrack extension is missing");
 
-	sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1",
-			ptrack_schema);
+	if (ptrack_version_num == 20)
+		sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1",
+				ptrack_schema);
+	else
+		sprintf(query, "SELECT path, pagemap FROM %s.ptrack_get_pagemapset($1) ORDER BY 1",
+				ptrack_schema);
 
 	res = pgut_execute(backup_conn, query, 1, (const char **) params);
 	pfree(params[0]);
@@ -640,16 +647,18 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, XLogRec
  */
 void
 make_pagemap_from_ptrack_2(parray *files,
-							PGconn *backup_conn,
-							const char *ptrack_schema,
-							XLogRecPtr lsn)
+						   PGconn *backup_conn,
+						   const char *ptrack_schema,
+						   int ptrack_version_num,
+						   XLogRecPtr lsn)
 {
 	parray *filemaps;
 	int		file_i = 0;
 	page_map_entry *dummy_map = NULL;
 
 	/* Receive all available ptrack bitmaps at once */
-	filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, lsn);
+	filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema,
+										ptrack_version_num, lsn);
 
 	if (filemaps != NULL)
 		parray_qsort(filemaps, pgFileMapComparePath);

tests/helpers/ptrack_helpers.py

@@ -366,7 +366,8 @@ def make_simple_node(
 
         if ptrack_enable:
             if node.major_version > 11:
-                options['ptrack_map_size'] = '128MB'
+                options['ptrack.map_size'] = '128'
+                options['shared_preload_libraries'] = 'ptrack'
             else:
                 options['ptrack_enable'] = 'on'