Skip to content

Commit bf5b8f6

Browse files
gsmolkgsmolk
committed
[Issue #169] fio_sync() added
1 parent 577e876 commit bf5b8f6

File tree

5 files changed

+122
-46
lines changed

5 files changed

+122
-46
lines changed

‎src/pg_probackup.c‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ static char *backup_id_string = NULL;
6868
int num_threads = 1;
6969
bool stream_wal = false;
7070
bool progress = false;
71+
bool no_sync = false;
7172
#if PG_VERSION_NUM >= 100000
7273
char *replication_slot = NULL;
7374
#endif
@@ -155,13 +156,14 @@ static void opt_datname_include_list(ConfigOption *opt, const char *arg);
155156
static ConfigOption cmd_options[] =
156157
{
157158
/* directory options */
158-
{ 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT },
159+
{ 'b', 120, "help", &help_opt, SOURCE_CMD_STRICT },
159160
{ 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT },
160161
/* common options */
161162
{ 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT },
162-
{ 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT },
163-
{ 'b', 132, "progress", &progress, SOURCE_CMD_STRICT },
163+
{ 'b', 121, "stream", &stream_wal, SOURCE_CMD_STRICT },
164+
{ 'b', 122, "progress", &progress, SOURCE_CMD_STRICT },
164165
{ 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT },
166+
{ 'b', 123, "no-sync", &no_sync, SOURCE_CMD_STRICT },
165167
/* backup options */
166168
{ 'b', 133, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT },
167169
{ 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT },
@@ -756,8 +758,8 @@ main(int argc, char *argv[])
756758
}
757759
case RESTORE_CMD:
758760
return do_restore_or_validate(current.backup_id,
759-
recovery_target_options,
760-
restore_params);
761+
recovery_target_options,
762+
restore_params, no_sync);
761763
case VALIDATE_CMD:
762764
if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn)
763765
{
@@ -771,7 +773,8 @@ main(int argc, char *argv[])
771773
/* PITR validation and, optionally, partial validation */
772774
return do_restore_or_validate(current.backup_id,
773775
recovery_target_options,
774-
restore_params);
776+
restore_params,
777+
no_sync);
775778
case SHOW_CMD:
776779
return do_show(instance_name, current.backup_id, show_archive);
777780
case DELETE_CMD:

‎src/pg_probackup.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -644,7 +644,8 @@ extern char *pg_ptrack_get_block(ConnectionArgs *arguments,
644644
/* in restore.c */
645645
extern int do_restore_or_validate(time_t target_backup_id,
646646
pgRecoveryTarget *rt,
647-
pgRestoreParams *params);
647+
pgRestoreParams *params,
648+
bool no_sync);
648649
extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup);
649650
extern bool satisfy_recovery_target(const pgBackup *backup,
650651
const pgRecoveryTarget *rt);

‎src/restore.c‎

Lines changed: 50 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ static void pg12_recovery_config(pgBackup *backup, bool add_include);
6565

6666
static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
6767
parray *dbOid_exclude_list, pgRestoreParams *params,
68-
const char *pgdata_path);
68+
const char *pgdata_path, bool no_sync);
6969

7070
static void *restore_files_new(void *arg);
7171

@@ -118,7 +118,7 @@ set_orphan_status(parray *backups, pgBackup *parent_backup)
118118
*/
119119
int
120120
do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
121-
pgRestoreParams *params)
121+
pgRestoreParams *params, bool no_sync)
122122
{
123123
int i = 0;
124124
int j = 0;
@@ -490,14 +490,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
490490
dest_backup->server_version);
491491

492492
restore_chain(dest_backup, parent_chain, dbOid_exclude_list,
493-
params, instance_config.pgdata);
493+
params, instance_config.pgdata, no_sync);
494494

495495
/* Create recovery.conf with given recovery target parameters */
496496
create_recovery_conf(target_backup_id, rt, dest_backup, params);
497497
}
498498

499499
/* cleanup */
500-
parray_walk(backups, pgBackupFree); /* free backup->files */
500+
parray_walk(backups, pgBackupFree); /* TODO: free backup->files */
501501
parray_free(backups);
502502
parray_free(parent_chain);
503503

@@ -512,7 +512,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
512512
void
513513
restore_chain(pgBackup *dest_backup, parray *parent_chain,
514514
parray *dbOid_exclude_list, pgRestoreParams *params,
515-
const char *pgdata_path)
515+
const char *pgdata_path, bool no_sync)
516516
{
517517
int i;
518518
char control_file[MAXPGPATH];
@@ -524,6 +524,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
524524
restore_files_arg_new *threads_args;
525525
bool restore_isok = true;
526526

527+
char pretty_time[20];
527528
time_t start_time, end_time;
528529

529530
/* Preparations for actual restoring */
@@ -637,6 +638,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
637638
pg_atomic_clear_flag(&file->lock);
638639
}
639640

641+
fio_disconnect();
642+
640643
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
641644
threads_args = (restore_files_arg_new *) palloc(sizeof(restore_files_arg_new) *
642645
num_threads);
@@ -674,46 +677,56 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
674677
}
675678

676679
time(&end_time);
680+
pretty_time_interval(difftime(end_time, start_time),
681+
pretty_time, lengthof(pretty_time));
677682
if (restore_isok)
678-
elog(INFO, "Backup files are restored, time elapsed: %.0f sec",
679-
difftime(end_time, start_time));
683+
elog(INFO, "Backup files are restored, time elapsed: %s", pretty_time);
680684
else
681-
elog(ERROR, "Backup files restoring failed, time elapsed: %.0f sec",
682-
difftime(end_time, start_time));
685+
elog(ERROR, "Backup files restoring failed, time elapsed: %s", pretty_time);
683686

687+
if (no_sync)
688+
elog(WARNING, "Restored files are not synced to disk");
689+
else
690+
{
691+
elog(INFO, "Syncing restored files to disk");
692+
time(&start_time);
684693

685-
elog(INFO, "Sync restored backup files to disk");
686-
time(&start_time);
694+
for (i = 0; i < parray_num(dest_files); i++)
695+
{
696+
char to_fullpath[MAXPGPATH];
697+
pgFile *dest_file = (pgFile *) parray_get(dest_files, i);
687698

688-
for (i = 0; i < parray_num(dest_files); i++)
689-
{
690-
int out;
691-
char to_fullpath[MAXPGPATH];
692-
pgFile *dest_file = (pgFile *) parray_get(dest_files, i);
693-
694-
if (S_ISDIR(dest_file->mode) ||
695-
dest_file->external_dir_num > 0 ||
696-
(strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) ||
697-
(strcmp(DATABASE_MAP, dest_file->rel_path) == 0))
698-
continue;
699+
if (S_ISDIR(dest_file->mode))
700+
continue;
699701

700-
join_path_components(to_fullpath, pgdata_path, dest_file->rel_path);
702+
if (params->skip_external_dirs && dest_file->external_dir_num > 0)
703+
continue;
701704

702-
/* open destination file */
703-
out = fio_open(to_fullpath, O_WRONLY | PG_BINARY, FIO_DB_HOST);
704-
if (out < 0)
705-
elog(ERROR, "Cannot open file \"%s\": %s",
706-
to_fullpath, strerror(errno));
707-
708-
/* sync file */
709-
if (fio_flush(out) != 0 || fio_close(out) != 0)
710-
elog(ERROR, "Cannot sync file \"%s\": %s",
711-
to_fullpath, strerror(errno));
712-
}
705+
/* construct fullpath */
706+
if (dest_file->external_dir_num == 0)
707+
{
708+
if (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0)
709+
continue;
710+
if (strcmp(DATABASE_MAP, dest_file->rel_path) == 0)
711+
continue;
712+
join_path_components(to_fullpath, pgdata_path, dest_file->rel_path);
713+
}
714+
else
715+
{
716+
char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1);
717+
join_path_components(to_fullpath, external_path, dest_file->rel_path);
718+
}
713719

714-
time(&end_time);
715-
elog(INFO, "Restored backup files are synced, time elapsed: %.0f sec",
716-
difftime(end_time, start_time));
720+
/* TODO: write test for case: file to be synced is missing */
721+
if (fio_sync(to_fullpath, FIO_DB_HOST) != 0)
722+
elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno));
723+
}
724+
725+
time(&end_time);
726+
pretty_time_interval(difftime(end_time, start_time),
727+
pretty_time, lengthof(pretty_time));
728+
elog(INFO, "Restored backup files are synced, time elapsed: %s", pretty_time);
729+
}
717730

718731
/* cleanup */
719732
pfree(threads);
@@ -724,8 +737,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
724737

725738
parray_walk(dest_files, pgFileFree);
726739
parray_free(dest_files);
727-
728-
// elog(LOG, "Restore of backup %s is completed", base36enc(backup->start_time));
729740
}
730741

731742
/*

‎src/utils/file.c‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -797,6 +797,48 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location
797797
}
798798
}
799799

800+
/* Sync file to disk */
801+
int fio_sync(char const* path, fio_location location)
802+
{
803+
if (fio_is_remote(location))
804+
{
805+
fio_header hdr;
806+
size_t path_len = strlen(path) + 1;
807+
hdr.cop = FIO_SYNC;
808+
hdr.handle = -1;
809+
hdr.size = path_len;
810+
811+
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
812+
IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len);
813+
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
814+
815+
if (hdr.arg != 0)
816+
{
817+
errno = hdr.arg;
818+
return -1;
819+
}
820+
821+
return 0;
822+
}
823+
else
824+
{
825+
int fd;
826+
827+
fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS);
828+
if (fd < 0)
829+
return -1;
830+
831+
if (fsync(fd) < 0)
832+
{
833+
close(fd);
834+
return -1;
835+
}
836+
close(fd);
837+
838+
return 0;
839+
}
840+
}
841+
800842
/* Remove file */
801843
int fio_unlink(char const* path, fio_location location)
802844
{
@@ -1348,6 +1390,7 @@ void fio_communicate(int in, int out)
13481390
fio_header hdr;
13491391
struct stat st;
13501392
int rc;
1393+
int tmp_fd;
13511394

13521395
#ifdef WIN32
13531396
SYS_CHECK(setmode(in, _O_BINARY));
@@ -1470,6 +1513,22 @@ void fio_communicate(int in, int out)
14701513
Assert(hdr.size == sizeof(fio_send_request));
14711514
fio_send_pages_impl(fd[hdr.handle], out, (fio_send_request*)buf);
14721515
break;
1516+
case FIO_SYNC:
1517+
/* open file and fsync it */
1518+
tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS);
1519+
if (tmp_fd < 0)
1520+
hdr.arg = errno;
1521+
else
1522+
{
1523+
if (fsync(tmp_fd) == 0)
1524+
hdr.arg = 0;
1525+
else
1526+
hdr.arg = errno;
1527+
}
1528+
close(tmp_fd);
1529+
1530+
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
1531+
break;
14731532
default:
14741533
Assert(false);
14751534
}

‎src/utils/file.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ typedef enum
1515
FIO_OPEN,
1616
FIO_CLOSE,
1717
FIO_WRITE,
18+
FIO_SYNC,
1819
FIO_RENAME,
1920
FIO_SYMLINK,
2021
FIO_UNLINK,
@@ -92,6 +93,7 @@ extern int fio_fstat(int fd, struct stat* st);
9293
extern int fio_truncate(int fd, off_t size);
9394
extern int fio_close(int fd);
9495
extern void fio_disconnect(void);
96+
extern int fio_sync(char const* path, fio_location location);
9597

9698
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
9799
extern int fio_symlink(char const* target, char const* link_path, fio_location location);

0 commit comments

Comments
 (0)