[Issue #169] pgFileGetCRC refactoring, now it operates only locally

Author: gsmolk

src/archive.c

@@ -594,11 +594,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed)
 	else
 #endif
 	{
-		crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST);
+		crc2 = fio_get_crc32(path2, FIO_BACKUP_HOST);
 	}
 
 	/* Get checksum of original file */
-	crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST);
+	crc1 = fio_get_crc32(path1, FIO_DB_HOST);
 
 	return EQ_CRC32C(crc1, crc2);
 }

src/backup.c

@@ -575,6 +575,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	{
 		parray     *xlog_files_list;
 		char		pg_xlog_path[MAXPGPATH];
+		char		wal_full_path[MAXPGPATH];
 
 		/* Scan backup PG_XLOG_DIR */
 		xlog_files_list = parray_new();
@@ -586,11 +587,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		for (i = 0; i < parray_num(xlog_files_list); i++)
 		{
 			pgFile	   *file = (pgFile *) parray_get(xlog_files_list, i);
+
+			join_path_components(wal_full_path, pg_xlog_path, file->rel_path);
+
 			if (S_ISREG(file->mode))
 			{
-				file->crc = pgFileGetCRC(file->path, true, false,
-										 &file->read_size, FIO_BACKUP_HOST);
-				file->write_size = file->read_size;
+				file->crc = pgFileGetCRC(wal_full_path, true, false);
+				file->write_size = file->size;
 			}
 			/* Remove file path root prefix*/
 			if (strstr(file->path, database_path) == file->path)
@@ -1805,10 +1808,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 			{
 				file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0,
 								 FIO_BACKUP_HOST);
-				file->crc = pgFileGetCRC(file->path, true, false,
-										 &file->read_size, FIO_BACKUP_HOST);
-				file->write_size = file->read_size;
-				file->uncompressed_size = file->read_size;
+
+				file->crc = pgFileGetCRC(backup_label, true, false);
+
+				file->write_size = file->size;
+				file->uncompressed_size = file->size;
 				free(file->path);
 				file->path = strdup(PG_BACKUP_LABEL_FILE);
 				parray_append(backup_files_list, file);
@@ -1854,9 +1858,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 								 FIO_BACKUP_HOST);
 				if (S_ISREG(file->mode))
 				{
-					file->crc = pgFileGetCRC(file->path, true, false,
-											 &file->read_size, FIO_BACKUP_HOST);
-					file->write_size = file->read_size;
+					file->crc = pgFileGetCRC(tablespace_map, true, false);
+					file->write_size = file->size;
 				}
 				free(file->path);
 				file->path = strdup(PG_TABLESPACE_MAP_FILE);

src/dir.c

@@ -259,10 +259,9 @@ pgFileDelete(pgFile *file, const char *full_path)
  * We cannot make decision about file decompression because
  * user may ask to backup already compressed files and we should be
  * obvious about it.
- * TODO: add decompression option.
  */
 pg_crc32
-pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok)
+pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
 {
 	FILE	   *fp;
 	pg_crc32	crc = 0;
@@ -316,65 +315,6 @@ pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok)
 	return crc;
 }
 
-/*
- * Read the file to compute its CRC.
- * As a handy side effect, we return filesize via bytes_read parameter.
- */
-pg_crc32
-pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted,
-			 size_t *bytes_read, fio_location location)
-{
-	FILE	   *fp;
-	pg_crc32	crc = 0;
-	char		buf[STDIO_BUFSIZE];
-	size_t		len = 0;
-	size_t		total = 0;
-	int			errno_tmp;
-
-	INIT_FILE_CRC32(use_crc32c, crc);
-
-	/* open file in binary read mode */
-	fp = fio_fopen(file_path, PG_BINARY_R, location);
-	if (fp == NULL)
-	{
-		if (!raise_on_deleted && errno == ENOENT)
-		{
-			FIN_FILE_CRC32(use_crc32c, crc);
-			return crc;
-		}
-		else
-			elog(ERROR, "cannot open file \"%s\": %s",
-				file_path, strerror(errno));
-	}
-
-	/* calc CRC of file */
-	for (;;)
-	{
-		if (interrupted)
-			elog(ERROR, "interrupted during CRC calculation");
-
-		len = fio_fread(fp, buf, sizeof(buf));
-		if (len == 0)
-			break;
-		/* update CRC */
-		COMP_FILE_CRC32(use_crc32c, crc, buf, len);
-		total += len;
-	}
-
-	if (bytes_read)
-		*bytes_read = total;
-
-	errno_tmp = errno;
-	if (len < 0)
-		elog(WARNING, "cannot read \"%s\": %s", file_path,
-			strerror(errno_tmp));
-
-	FIN_FILE_CRC32(use_crc32c, crc);
-	fio_fclose(fp);
-
-	return crc;
-}
-
 void
 pgFileFree(void *file)
 {
@@ -1775,7 +1715,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
 								 FIO_BACKUP_HOST);
 	pfree(file->path);
 	file->path = pgut_strdup(DATABASE_MAP);
-	file->crc = pgFileGetCRCnew(database_map_path, true, false);
+	file->crc = pgFileGetCRC(database_map_path, true, false);
 
 	file->write_size = file->read_size;
 	file->uncompressed_size = file->read_size;

src/pg_probackup.h

@@ -826,10 +826,9 @@ extern pgFile *pgFileInit(const char *path, const char *rel_path);
 extern void pgFileDelete(pgFile *file, const char *full_path);
 
 extern void pgFileFree(void *file);
-extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c,
-							 bool raise_on_deleted, size_t *bytes_read, fio_location location);
-extern pg_crc32 pgFileGetCRCnew(const char *file_path, bool missing_ok, bool use_crc32c);
-//extern pg_crc32 pgFileGetCRC_compressed(const char *file_path, bool use_crc32c, bool missing_ok);
+
+extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c);
+
 extern int pgFileCompareName(const void *f1, const void *f2);
 extern int pgFileComparePath(const void *f1, const void *f2);
 extern int pgFileMapComparePath(const void *f1, const void *f2);

src/utils/file.c

@@ -926,7 +926,7 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location)
 		return crc;
 	}
 	else
-		return pgFileGetCRCnew(file_path, true, true);
+		return pgFileGetCRC(file_path, true, true);
 }
 
 /* Remove file */
@@ -1629,7 +1629,7 @@ void fio_communicate(int in, int out)
 			break;
 		  case FIO_GET_CRC32:
 			/* calculate crc32 for a file */
-			crc = pgFileGetCRCnew(buf, true, true);
+			crc = pgFileGetCRC(buf, true, true);
 			IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));
 			break;
 		  default:

src/validate.c

@@ -328,7 +328,7 @@ pgBackupValidateFiles(void *arg)
 				crc = pgFileGetCRC(file->path,
 								   arguments->backup_version <= 20021 ||
 								   arguments->backup_version >= 20025,
-								   true, NULL, FIO_LOCAL_HOST);
+								   false);
 			if (crc != file->crc)
 			{
 				elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",