[Issue #79]: remove partial validation

Author: gsmolk

src/catalog.c

@@ -415,6 +415,27 @@ catalog_get_backup_list(time_t requested_backup_id)
 	return NULL;
 }
 
+/*
+ * Create list of backup datafiles.
+ * If 'requested_backup_id' is INVALID_BACKUP_ID, exit with error.
+ * If valid backup id is passed only matching backup will be added to the list.
+ */
+parray *
+get_backup_filelist(pgBackup *backup)
+{
+	parray		*files = NULL;
+	char		backup_filelist_path[MAXPGPATH];
+
+	pgBackupGetPath(backup, backup_filelist_path, lengthof(backup_filelist_path), DATABASE_FILE_LIST);
+	files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST);
+
+	/* redundant sanity? */
+	if (!files)
+		elog(ERROR, "Failed to get filelist for backup %s", base36enc(backup->start_time));
+
+	return files;
+}
+
 /*
  * Lock list of backups. Function goes in backward direction.
  */

src/pg_probackup.h

@@ -521,8 +521,10 @@ extern pgRecoveryTarget *parseRecoveryTargetOptions(
 	const char *target_stop, const char *target_name,
 	const char *target_action);
 
-extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *files,
-							   parray *datname_list, PartialRestoreType partial_restore_type);
+extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
+										PartialRestoreType partial_restore_type);
+
+extern parray *get_backup_filelist(pgBackup *backup);
 
 /* in merge.c */
 extern void do_merge(time_t backup_id);

src/restore.c

@@ -373,7 +373,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
 				}
 			}
 
+			/* validate datafiles only */
 			pgBackupValidate(tmp_backup, params);
+
 			/* After pgBackupValidate() only following backup
 			 * states are possible: ERROR, RUNNING, CORRUPT and OK.
 			 * Validate WAL only for OK, because there is no point
@@ -390,6 +392,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
 		}
 
 		/* There is no point in wal validation of corrupted backups */
+		// TODO: there should be a way for a user to request only(!) WAL validation
 		if (!corrupted_backup)
 		{
 			/*
@@ -458,7 +461,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
 		 * throw an error.
 		 */
 		if (params->partial_db_list)
-			dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, dest_files, params->partial_db_list,
+			dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, params->partial_db_list,
 														  params->partial_restore_type);
 
 		/*
@@ -1185,19 +1188,22 @@ parseRecoveryTargetOptions(const char *target_time,
  * we always convert it into exclude_list.
  */
 parray *
-get_dbOid_exclude_list(pgBackup *backup, parray *files,
-					   parray *datname_list, PartialRestoreType partial_restore_type)
+get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
+										PartialRestoreType partial_restore_type)
 {
 	int i;
 	int j;
-	parray *database_map = NULL;
-	parray *dbOid_exclude_list = NULL;
-	pgFile *database_map_file = NULL;
-	pg_crc32	crc;
+//	pg_crc32	crc;
+	parray		*database_map = NULL;
+	parray		*dbOid_exclude_list = NULL;
+	pgFile		*database_map_file = NULL;
 	char		path[MAXPGPATH];
 	char		database_map_path[MAXPGPATH];
+	parray		*files = NULL;
+
+	files = get_backup_filelist(backup);
 
-	/* make sure that database_map is in backup_content.control */
+	/* look for 'database_map' file in backup_content.control */
 	for (i = 0; i < parray_num(files); i++)
 	{
 		pgFile	   *file = (pgFile *) parray_get(files, i);
@@ -1218,11 +1224,11 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files,
 	join_path_components(database_map_path, path, DATABASE_MAP);
 
 	/* check database_map CRC */
-	crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST);
-
-	if (crc != database_map_file->crc)
-		elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
-				database_map_file->path, crc, database_map_file->crc);
+//	crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST);
+//
+//	if (crc != database_map_file->crc)
+//		elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
+//				database_map_file->path, crc, database_map_file->crc);
 
 	/* get database_map from file */
 	database_map = read_database_map(backup);
@@ -1307,6 +1313,13 @@ get_dbOid_exclude_list(pgBackup *backup, parray *files,
 		elog(ERROR, "Failed to find a match in database_map of backup %s for partial restore",
 					base36enc(backup->start_time));
 
+	/* clean backup filelist */
+	if (files)
+	{
+		parray_walk(files, pgFileFree);
+		parray_free(files);
+	}
+
 	/* sort dbOid array in ASC order */
 	parray_qsort(dbOid_exclude_list, pgCompareOid);
 

src/validate.c

@@ -41,6 +41,7 @@ typedef struct
 
 /*
  * Validate backup files.
+ * TODO: partial validation.
  */
 void
 pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
@@ -55,7 +56,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
 	pthread_t  *threads;
 	validate_files_arg *threads_args;
 	int			i;
-	parray *dbOid_exclude_list = NULL;
+//	parray		*dbOid_exclude_list = NULL;
 
 	/* Check backup version */
 	if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION))
@@ -107,9 +108,9 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
 	pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST);
 	files = dir_read_file_list(base_path, external_prefix, path, FIO_BACKUP_HOST);
 
-	if (params && params->partial_db_list)
-		dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
-														  params->partial_restore_type);
+//	if (params && params->partial_db_list)
+//		dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
+//														params->partial_restore_type);
 
 	/* setup threads */
 	for (i = 0; i < parray_num(files); i++)
@@ -136,7 +137,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
 		arg->stop_lsn = backup->stop_lsn;
 		arg->checksum_version = backup->checksum_version;
 		arg->backup_version = parse_program_version(backup->program_version);
-		arg->dbOid_exclude_list = dbOid_exclude_list;
+//		arg->dbOid_exclude_list = dbOid_exclude_list;
 		/* By default there are some error */
 		threads_args[i].ret = 1;
 
@@ -204,14 +205,14 @@ pgBackupValidateFiles(void *arg)
 		 * If in partial validate, check if the file belongs to the database
 		 * we exclude. Only files from pgdata can be skipped.
 		 */
-		if (arguments->dbOid_exclude_list && file->external_dir_num == 0
-			&& parray_bsearch(arguments->dbOid_exclude_list,
-							   &file->dbOid, pgCompareOid))
-		{
-			elog(VERBOSE, "Skip file validation due to partial restore: \"%s\"",
-				 file->rel_path);
-			continue;
-		}
+		//if (arguments->dbOid_exclude_list && file->external_dir_num == 0
+		//	&& parray_bsearch(arguments->dbOid_exclude_list,
+		//					   &file->dbOid, pgCompareOid))
+		//{
+		//	elog(VERBOSE, "Skip file validation due to partial restore: \"%s\"",
+		//		 file->rel_path);
+		//	continue;
+		//}
 
 		/*
 		 * Currently we don't compute checksums for