[Issue #153] Optimize memory consumption: get rid of pgFile.path and pgFile.forkname, use pgFile.rel_path substring for pgFile.name

Author: gsmolk

src/archive.c

@@ -945,7 +945,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file,
 	/* get list of files from archive_status */
 	status_files = parray_new();
 	dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST);
-	parray_qsort(status_files, pgFileComparePath);
+	parray_qsort(status_files, pgFileCompareName);
 
 	for (i = 0; i < parray_num(status_files); i++)
 	{

src/backup.c

@@ -149,7 +149,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	parray	   *external_dirs = NULL;
 	parray	   *database_map = NULL;
 
-	pgFile	   *pg_control = NULL;
 	PGconn	   *master_conn = NULL;
 	PGconn	   *pg_startbackup_conn = NULL;
 
@@ -394,7 +393,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	 * Sorted array is used at least in parse_filelist_filenames(),
 	 * extractPageMap(), make_pagemap_from_ptrack().
 	 */
-	parray_qsort(backup_files_list, pgFileComparePath);
+	parray_qsort(backup_files_list, pgFileCompareRelPathWithExternal);
 
 	/* Extract information about files in backup_list parsing their names:*/
 	parse_filelist_filenames(backup_files_list, instance_config.pgdata);
@@ -468,26 +467,18 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		if (S_ISDIR(file->mode))
 		{
 			char		dirpath[MAXPGPATH];
-			char	   *dir_name;
-
-			if (file->external_dir_num)
-				dir_name = GetRelativePath(file->path,
-								parray_get(external_dirs,
-											file->external_dir_num - 1));
-			else
-				dir_name = GetRelativePath(file->path, instance_config.pgdata);
-
-			elog(VERBOSE, "Create directory \"%s\"", dir_name);
 
 			if (file->external_dir_num)
 			{
 				char		temp[MAXPGPATH];
 				snprintf(temp, MAXPGPATH, "%s%d", external_prefix,
 						 file->external_dir_num);
-				join_path_components(dirpath, temp, dir_name);
+				join_path_components(dirpath, temp, file->rel_path);
 			}
 			else
-				join_path_components(dirpath, database_path, dir_name);
+				join_path_components(dirpath, database_path, file->rel_path);
+
+			elog(VERBOSE, "Create directory '%s'", dirpath);
 			fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST);
 		}
 
@@ -590,16 +581,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	 */
 	if (current.from_replica && !exclusive_backup)
 	{
-		char		pg_control_path[MAXPGPATH];
-
-		snprintf(pg_control_path, sizeof(pg_control_path), "%s/%s",
-				 instance_config.pgdata, XLOG_CONTROL_FILE);
+		pgFile	   *pg_control = NULL;
 
 		for (i = 0; i < parray_num(backup_files_list); i++)
 		{
 			pgFile	   *tmp_file = (pgFile *) parray_get(backup_files_list, i);
 
-			if (strcmp(tmp_file->path, pg_control_path) == 0)
+			if (tmp_file->external_dir_num == 0 &&
+				(strcmp(tmp_file->rel_path, XLOG_CONTROL_FILE) == 0))
 			{
 				pg_control = tmp_file;
 				break;
@@ -608,7 +597,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 
 		if (!pg_control)
 			elog(ERROR, "Failed to find file \"%s\" in backup filelist.",
-							pg_control_path);
+							XLOG_CONTROL_FILE);
 
 		set_min_recovery_point(pg_control, database_path, current.stop_lsn);
 	}
@@ -636,21 +625,23 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 
 			join_path_components(wal_full_path, pg_xlog_path, file->rel_path);
 
-			if (S_ISREG(file->mode))
-			{
-				file->crc = pgFileGetCRC(wal_full_path, true, false);
-				file->write_size = file->size;
-			}
-			/* Remove file path root prefix*/
-			if (strstr(file->path, database_path) == file->path)
-			{
-				char	   *ptr = file->path;
+			if (!S_ISREG(file->mode))
+				continue;
 
-				file->path = pgut_strdup(GetRelativePath(ptr, database_path));
-				file->rel_path = pgut_strdup(file->path);
-				free(ptr);
-			}
+			file->crc = pgFileGetCRC(wal_full_path, true, false);
+			file->write_size = file->size;
+
+			/* overwrite rel_path, because now it is relative to
+			 * /backup_dir/backups/instance_name/backup_id/database/pg_xlog/
+			 */
+			pg_free(file->rel_path);
+
+			file->rel_path = pgut_strdup(GetRelativePath(wal_full_path, database_path));
+			file->name = last_dir_separator(file->rel_path);
+
+			/* Now it is relative to /backup_dir/backups/instance_name/backup_id/database/ */
 		}
+
 		/* Add xlog files into the list of backed up files */
 		parray_concat(backup_files_list, xlog_files_list);
 		parray_free(xlog_files_list);
@@ -1884,8 +1875,6 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 
 				file->write_size = file->size;
 				file->uncompressed_size = file->size;
-				free(file->path);
-				file->path = strdup(PG_BACKUP_LABEL_FILE);
 				parray_append(backup_files_list, file);
 			}
 		}
@@ -1932,8 +1921,7 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 					file->crc = pgFileGetCRC(tablespace_map, true, false);
 					file->write_size = file->size;
 				}
-				free(file->path);
-				file->path = strdup(PG_TABLESPACE_MAP_FILE);
+
 				parray_append(backup_files_list, file);
 			}
 		}
@@ -2204,13 +2192,13 @@ parse_filelist_filenames(parray *files, const char *root)
 	while (i < parray_num(files))
 	{
 		pgFile	   *file = (pgFile *) parray_get(files, i);
-		char	   *relative;
+//		char	   *relative;
 		int 		sscanf_result;
 
-		relative = GetRelativePath(file->path, root);
+//		relative = GetRelativePath(file->rel_path, root);
 
 		if (S_ISREG(file->mode) &&
-			path_is_prefix_of_path(PG_TBLSPC_DIR, relative))
+			path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
 		{
 			/*
 			 * Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
@@ -2225,21 +2213,21 @@ parse_filelist_filenames(parray *files, const char *root)
 				 * Check that the file is located under
 				 * TABLESPACE_VERSION_DIRECTORY
 				 */
-				sscanf_result = sscanf(relative, PG_TBLSPC_DIR "/%u/%s/%u",
+				sscanf_result = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s/%u",
 									   &tblspcOid, tmp_rel_path, &dbOid);
 
 				/* Yes, it is */
 				if (sscanf_result == 2 &&
 					strncmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY,
 							strlen(TABLESPACE_VERSION_DIRECTORY)) == 0)
-					set_cfs_datafiles(files, root, relative, i);
+					set_cfs_datafiles(files, root, file->rel_path, i);
 			}
 		}
 
 		if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
 			file->name && file->name[0])
 		{
-			if (strcmp(file->forkName, "init") == 0)
+			if (file->forkName == INIT)
 			{
 				/*
 				 * Do not backup files of unlogged relations.
@@ -2290,7 +2278,6 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
 	int			p;
 	pgFile	   *prev_file;
 	char	   *cfs_tblspc_path;
-	char	   *relative_prev_file;
 
 	cfs_tblspc_path = strdup(relative);
 	if(!cfs_tblspc_path)
@@ -2302,22 +2289,21 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
 	for (p = (int) i; p >= 0; p--)
 	{
 		prev_file = (pgFile *) parray_get(files, (size_t) p);
-		relative_prev_file = GetRelativePath(prev_file->path, root);
 
-		elog(VERBOSE, "Checking file in cfs tablespace %s", relative_prev_file);
+		elog(VERBOSE, "Checking file in cfs tablespace %s", prev_file->rel_path);
 
-		if (strstr(relative_prev_file, cfs_tblspc_path) != NULL)
+		if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL)
 		{
 			if (S_ISREG(prev_file->mode) && prev_file->is_datafile)
 			{
 				elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s",
-					relative_prev_file, prev_file->name);
+					prev_file->rel_path, prev_file->name);
 				prev_file->is_cfs = true;
 			}
 		}
 		else
 		{
-			elog(VERBOSE, "Breaking on %s", relative_prev_file);
+			elog(VERBOSE, "Breaking on %s", prev_file->rel_path);
 			break;
 		}
 	}
@@ -2331,7 +2317,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
 void
 process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
 {
-	char	   *path;
+//	char	   *path;
 	char	   *rel_path;
 	BlockNumber blkno_inseg;
 	int			segno;
@@ -2343,16 +2329,15 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
 
 	rel_path = relpathperm(rnode, forknum);
 	if (segno > 0)
-		path = psprintf("%s/%s.%u", instance_config.pgdata, rel_path, segno);
+		f.rel_path = psprintf("%s.%u", rel_path, segno);
 	else
-		path = psprintf("%s/%s", instance_config.pgdata, rel_path);
+		f.rel_path = rel_path;
 
-	pg_free(rel_path);
+	f.external_dir_num = 0;
 
-	f.path = path;
 	/* backup_files_list should be sorted before */
 	file_item = (pgFile **) parray_bsearch(backup_files_list, &f,
-										   pgFileComparePath);
+										   pgFileCompareRelPathWithExternal);
 
 	/*
 	 * If we don't have any record of this file in the file map, it means
@@ -2372,7 +2357,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno)
 			pthread_mutex_unlock(&backup_pagemap_mutex);
 	}
 
-	pg_free(path);
+	pg_free(rel_path);
 }
 
 /*

src/catalog.c

@@ -471,6 +471,9 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id)
 
 		backup->root_dir = pgut_strdup(data_path);
 
+		backup->database_dir = pgut_malloc(MAXPGPATH);
+		join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR);
+
 		/* TODO: save encoded backup id */
 		backup->backup_id = backup->start_time;
 		if (requested_backup_id != INVALID_BACKUP_ID
@@ -841,6 +844,9 @@ pgBackupCreateDir(pgBackup *backup)
 	fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST);
 	backup->root_dir = pgut_strdup(path);
 
+	backup->database_dir = pgut_malloc(MAXPGPATH);
+	join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR);
+
 	/* create directories for actual backup files */
 	for (i = 0; i < parray_num(subdirs); i++)
 	{
@@ -873,7 +879,7 @@ catalog_get_timelines(InstanceConfig *instance)
 	/* read all xlog files that belong to this archive */
 	sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
 	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
-	parray_qsort(xlog_files_list, pgFileComparePath);
+	parray_qsort(xlog_files_list, pgFileCompareName);
 
 	timelineinfos = parray_new();
 	tlinfo = NULL;
@@ -1849,7 +1855,6 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		int       len = 0;
 		char      line[BLCKSZ];
 		pgFile   *file = (pgFile *) parray_get(files, i);
-		char     *path = file->path; /* for streamed WAL files */
 
 		if (S_ISDIR(file->mode))
 		{
@@ -1873,20 +1878,12 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 			}
 		}
 
-		/* for files from PGDATA and external files use rel_path
-		 * streamed WAL files has rel_path relative not to "database/"
-		 * but to "database/pg_wal", so for them use path.
-		 */
-		if ((root && strstr(path, root) == path) ||
-			(file->external_dir_num && external_list))
-				path = file->rel_path;
-
 		len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
 					 "\"mode\":\"%u\", \"is_datafile\":\"%u\", "
 					 "\"is_cfs\":\"%u\", \"crc\":\"%u\", "
 					 "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
 					 "\"dbOid\":\"%u\"",
-					path, file->write_size, file->mode,
+					file->rel_path, file->write_size, file->mode,
 					file->is_datafile ? 1 : 0,
 					file->is_cfs ? 1 : 0,
 					file->crc,
@@ -2257,6 +2254,7 @@ pgBackupInit(pgBackup *backup)
 	backup->server_version[0] = '\0';
 	backup->external_dir_str = NULL;
 	backup->root_dir = NULL;
+	backup->database_dir = NULL;
 	backup->files = NULL;
 	backup->note = NULL;
 	backup->content_crc = 0;
@@ -2272,6 +2270,7 @@ pgBackupFree(void *backup)
 	pfree(b->primary_conninfo);
 	pfree(b->external_dir_str);
 	pfree(b->root_dir);
+	pfree(b->database_dir);
 	pfree(b->note);
 	pfree(backup);
 }

src/checkdb.c

@@ -217,7 +217,7 @@ do_block_validation(char *pgdata, uint32 checksum_version)
 	 * 1 - create 'base'
 	 * 2 - create 'base/1'
 	 */
-	parray_qsort(files_list, pgFileComparePath);
+	parray_qsort(files_list, pgFileCompareRelPathWithExternal);
 	/* Extract information about files in pgdata parsing their names:*/
 	parse_filelist_filenames(files_list, pgdata);