Skip to content

Commit 7bdc682

Browse files
gsmolkgsmolk
committed
[Issue #66] added "-I | --incremenal-mode" option
1 parent 1c773d1 commit 7bdc682

File tree

3 files changed

+83
-60
lines changed

3 files changed

+83
-60
lines changed

‎src/pg_probackup.c‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,7 @@ static pgRestoreParams *restore_params = NULL;
9898
time_t current_time = 0;
9999
bool restore_as_replica = false;
100100
bool no_validate = false;
101-
bool incremental = false;
102-
bool incremental_lsn = false;
101+
IncrRestoreMode incremental_mode = INCR_NONE;
103102

104103
bool skip_block_validation = false;
105104
bool skip_external_dirs = false;
@@ -152,6 +151,7 @@ static ProbackupSubcmd backup_subcmd = NO_CMD;
152151

153152
static bool help_opt = false;
154153

154+
static void opt_incr_restore_mode(ConfigOption *opt, const char *arg);
155155
static void opt_backup_mode(ConfigOption *opt, const char *arg);
156156
static void opt_show_format(ConfigOption *opt, const char *arg);
157157

@@ -205,8 +205,7 @@ static ConfigOption cmd_options[] =
205205
{ 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT },
206206
{ 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT },
207207
{ 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT },
208-
{ 'b', 161, "incremental", &incremental, SOURCE_CMD_STRICT },
209-
{ 'b', 167, "incremental-lsn", &incremental_lsn, SOURCE_CMD_STRICT },
208+
{ 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT },
210209
/* checkdb options */
211210
{ 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT },
212211
{ 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT },
@@ -704,9 +703,6 @@ main(int argc, char *argv[])
704703
if (replication_slot != NULL)
705704
restore_as_replica = true;
706705

707-
if (!incremental && incremental_lsn)
708-
incremental = true;
709-
710706
/* keep all params in one structure */
711707
restore_params = pgut_new(pgRestoreParams);
712708
restore_params->is_restore = (backup_subcmd == RESTORE_CMD);
@@ -719,8 +715,7 @@ main(int argc, char *argv[])
719715
restore_params->partial_db_list = NULL;
720716
restore_params->partial_restore_type = NONE;
721717
restore_params->primary_conninfo = primary_conninfo;
722-
restore_params->incremental = incremental;
723-
restore_params->incremental_lsn = incremental_lsn;
718+
restore_params->incremental_mode = incremental_mode;
724719

725720
/* handle partial restore parameters */
726721
if (datname_exclude_list && datname_include_list)
@@ -878,6 +873,29 @@ main(int argc, char *argv[])
878873
return 0;
879874
}
880875

876+
static void
877+
opt_incr_restore_mode(ConfigOption *opt, const char *arg)
878+
{
879+
if (pg_strcasecmp(arg, "none") == 0)
880+
{
881+
incremental_mode = INCR_NONE;
882+
return;
883+
}
884+
else if (pg_strcasecmp(arg, "checksum") == 0)
885+
{
886+
incremental_mode = INCR_CHECKSUM;
887+
return;
888+
}
889+
else if (pg_strcasecmp(arg, "lsn") == 0)
890+
{
891+
incremental_mode = INCR_LSN;
892+
return;
893+
}
894+
895+
/* Backup mode is invalid, so leave with an error */
896+
elog(ERROR, "Invalid value for '--incremental-mode' option: '%s'", arg);
897+
}
898+
881899
static void
882900
opt_backup_mode(ConfigOption *opt, const char *arg)
883901
{

‎src/pg_probackup.h‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,13 @@ typedef struct db_map_entry
119119
char *datname;
120120
} db_map_entry;
121121

122+
typedef enum IncrRestoreMode
123+
{
124+
INCR_NONE,
125+
INCR_CHECKSUM,
126+
INCR_LSN
127+
} IncrRestoreMode;
128+
122129
typedef enum PartialRestoreType
123130
{
124131
NONE,
@@ -453,16 +460,17 @@ typedef struct pgRestoreParams
453460
bool restore_as_replica;
454461
bool skip_external_dirs;
455462
bool skip_block_validation; //Start using it
456-
bool incremental;
457-
bool incremental_lsn;
458-
XLogRecPtr horizonLsn;
459463
const char *restore_command;
460464
const char *primary_slot_name;
465+
const char *primary_conninfo;
466+
467+
/* options for incremental restore */
468+
IncrRestoreMode incremental_mode;
469+
XLogRecPtr shift_lsn;
461470

462471
/* options for partial restore */
463472
PartialRestoreType partial_restore_type;
464473
parray *partial_db_list;
465-
const char *primary_conninfo;
466474
} pgRestoreParams;
467475

468476
/* Options needed for set-backup command */

‎src/restore.c‎

Lines changed: 44 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,8 @@ typedef struct
2929
const char *to_root;
3030
size_t restored_bytes;
3131
bool use_bitmap;
32-
bool incremental;
33-
bool incremental_lsn;
34-
XLogRecPtr horizonLsn;
32+
IncrRestoreMode incremental_mode;
33+
XLogRecPtr shift_lsn; /* used only in LSN incremental_mode */
3534

3635
/*
3736
* Return value from the thread.
@@ -52,7 +51,7 @@ static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
5251
parray *dbOid_exclude_list, pgRestoreParams *params,
5352
const char *pgdata_path, bool no_sync);
5453
static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
55-
bool incremental_lsn);
54+
IncrRestoreMode incremental_mode);
5655

5756
/*
5857
* Iterate over backup list to find all ancestors of the broken parent_backup
@@ -117,7 +116,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
117116
parray *dbOid_exclude_list = NULL;
118117
bool pgdata_is_empty = true;
119118
bool tblspaces_are_empty = true;
120-
XLogRecPtr horizonLsn = InvalidXLogRecPtr;
119+
XLogRecPtr shift_lsn = InvalidXLogRecPtr;
121120

122121
if (params->is_restore)
123122
{
@@ -128,14 +127,14 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
128127
if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST))
129128
{
130129
/* Check that remote system is NOT running and systemd id is the same as ours */
131-
if (params->incremental)
130+
if (params->incremental_mode != INCR_NONE)
132131
{
133132
elog(INFO, "Running incremental restore into nonempty directory: \"%s\"",
134133
instance_config.pgdata);
135134

136135
check_incremental_compatibility(instance_config.pgdata,
137136
instance_config.system_identifier,
138-
params->incremental_lsn);
137+
params->incremental_mode);
139138
}
140139
else
141140
elog(ERROR, "Restore destination is not empty: \"%s\"",
@@ -328,13 +327,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
328327
base36enc(dest_backup->start_time));
329328
}
330329

331-
/*
332-
* We have found full backup by link,
333-
* now we need to walk the list to find its index.
334-
*
335-
* TODO I think we should rewrite it someday to use double linked list
336-
* and avoid relying on sort order anymore.
337-
*/
330+
/* We have found full backup */
338331
base_full_backup = tmp_backup;
339332
}
340333

@@ -347,14 +340,18 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
347340
*/
348341
if (params->is_restore)
349342
{
350-
check_tablespace_mapping(dest_backup, params->incremental, &tblspaces_are_empty);
343+
check_tablespace_mapping(dest_backup, params->incremental_mode != INCR_NONE, &tblspaces_are_empty);
351344

352345
if (pgdata_is_empty && tblspaces_are_empty)
353-
params->incremental = false;
346+
{
347+
elog(INFO, "Destination directory and tablespace directories are empty, "
348+
"disabled incremental restore");
349+
params->incremental_mode = INCR_NONE;
350+
}
354351

355352
/* no point in checking external directories if their restore is not requested */
356353
if (!params->skip_external_dirs)
357-
check_external_dir_mapping(dest_backup, params->incremental);
354+
check_external_dir_mapping(dest_backup, params->incremental_mode != INCR_NONE);
358355
}
359356

360357
/* At this point we are sure that parent chain is whole
@@ -387,7 +384,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
387384
* F - destination backup
388385
* * - switch point
389386
*
390-
* When running incremental restore in shift mode, we get a bitmap of pages,
387+
* When running incremental restore in 'lsn' mode, we get a bitmap of pages,
391388
* whose LSN is less than shift-LSN (backup C stop_lsn).
392389
* So when restoring file, we can skip restore of pages coming from
393390
* A, B and C.
@@ -408,7 +405,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
408405
* We must be able to differentiate the scenario A and scenario B.
409406
*
410407
*/
411-
if (params->is_restore && params->incremental && params->incremental_lsn)
408+
if (params->is_restore && params->incremental_mode == INCR_LSN)
412409
{
413410
RedoParams redo;
414411
parray *timelines = NULL;
@@ -418,7 +415,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
418415

419416
if (!timelines)
420417
elog(WARNING, "Failed to get history for redo timeline %i, "
421-
"multi-timeline incremental restore in shift mode is impossible", redo.tli);
418+
"multi-timeline incremental restore in 'lsn' mode is impossible", redo.tli);
422419

423420
tmp_backup = dest_backup;
424421

@@ -432,17 +429,17 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
432429
*/
433430
if (redo.tli == tmp_backup->tli)
434431
{
435-
elog(INFO, "Backup %s is chosen as shiftpoint",
432+
elog(INFO, "Backup %s is chosen as shiftpoint, its Stop LSN will be used as shift LSN",
436433
base36enc(tmp_backup->start_time));
437434

438-
horizonLsn = tmp_backup->stop_lsn;
435+
shift_lsn = tmp_backup->stop_lsn;
439436
break;
440437
}
441438

442439
if (!timelines)
443440
{
444441
elog(WARNING, "Redo timeline %i differs from target timeline %i, "
445-
"in this case, to safely run incremental restore in shift mode, "
442+
"in this case, to safely run incremental restore in 'lsn' mode, "
446443
"the history file for timeline %i is mandatory",
447444
redo.tli, tmp_backup->tli, redo.tli);
448445
break;
@@ -451,7 +448,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
451448
/* check whether the candidate tli is a part of redo TLI history */
452449
if (tliIsPartOfHistory(timelines, tmp_backup->tli))
453450
{
454-
horizonLsn = tmp_backup->stop_lsn;
451+
shift_lsn = tmp_backup->stop_lsn;
455452
break;
456453
}
457454
else
@@ -463,23 +460,23 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
463460
tmp_backup = tmp_backup->parent_backup_link;
464461
}
465462

466-
if (XLogRecPtrIsInvalid(horizonLsn))
467-
elog(ERROR, "Cannot perform incremental restore of backup chain %s in shift mode, "
463+
if (XLogRecPtrIsInvalid(shift_lsn))
464+
elog(ERROR, "Cannot perform incremental restore of backup chain %s in 'lsn' mode, "
468465
"because destination directory redo point %X/%X on tli %i is out of reach",
469466
base36enc(dest_backup->start_time),
470467
(uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli);
471468
else
472469
elog(INFO, "Destination directory redo point %X/%X on tli %i is within reach of "
473-
"backup %s with STOP LSN %X/%X on tli %i, incremental restore in shift mode is possible",
470+
"backup %s with Stop LSN %X/%X on tli %i, incremental restore in 'lsn' mode is possible",
474471
(uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli,
475472
base36enc(tmp_backup->start_time),
476473
(uint32) (tmp_backup->stop_lsn >> 32), (uint32) tmp_backup->stop_lsn,
477474
tmp_backup->tli);
478475

479476
elog(INFO, "shift LSN: %X/%X",
480-
(uint32) (horizonLsn >> 32), (uint32) horizonLsn);
477+
(uint32) (shift_lsn >> 32), (uint32) shift_lsn);
481478

482-
params->horizonLsn = horizonLsn;
479+
params->shift_lsn = shift_lsn;
483480
}
484481

485482
/* for validation or restore with enabled validation */
@@ -706,7 +703,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
706703
{
707704
use_bitmap = false;
708705

709-
if (params->incremental)
706+
if (params->incremental_mode != INCR_NONE)
710707
elog(ERROR, "incremental restore is not possible for backups older than 2.3.0 version");
711708
}
712709

@@ -715,7 +712,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
715712
*/
716713
if (use_bitmap && parray_num(parent_chain) == 1)
717714
{
718-
if (params->incremental && params->incremental_lsn)
715+
if (params->incremental_mode == INCR_NONE)
719716
use_bitmap = true;
720717
else
721718
use_bitmap = false;
@@ -726,7 +723,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
726723
*/
727724
create_data_directories(dest_files, instance_config.pgdata,
728725
dest_backup->root_dir, true,
729-
params->incremental, FIO_DB_HOST);
726+
params->incremental_mode != INCR_NONE,
727+
FIO_DB_HOST);
730728

731729
/*
732730
* Restore dest_backup external directories.
@@ -777,7 +775,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
777775
}
778776

779777
/* Get list of files in destination directory and remove redundant files */
780-
if (params->incremental)
778+
if (params->incremental_mode != INCR_NONE)
781779
{
782780
pgdata_files = parray_new();
783781

@@ -883,9 +881,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
883881
arg->skip_external_dirs = params->skip_external_dirs;
884882
arg->to_root = pgdata_path;
885883
arg->use_bitmap = use_bitmap;
886-
arg->incremental = params->incremental;
887-
arg->incremental_lsn = params->incremental_lsn;
888-
arg->horizonLsn = params->horizonLsn;
884+
arg->incremental_mode = params->incremental_mode;
885+
arg->shift_lsn = params->shift_lsn;
889886
threads_args[i].restored_bytes = 0;
890887
/* By default there are some error */
891888
threads_args[i].ret = 1;
@@ -1080,7 +1077,7 @@ restore_files(void *arg)
10801077
join_path_components(to_fullpath, external_path, dest_file->rel_path);
10811078
}
10821079

1083-
if (arguments->incremental &&
1080+
if (arguments->incremental_mode != INCR_NONE &&
10841081
parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc))
10851082
{
10861083
already_exists = true;
@@ -1096,14 +1093,13 @@ restore_files(void *arg)
10961093
dest_file->is_datafile && !dest_file->is_cfs &&
10971094
dest_file->n_blocks > 0)
10981095
{
1099-
if (arguments->incremental_lsn)
1096+
if (arguments->incremental_mode == INCR_LSN)
11001097
{
1101-
/* TODO: return lsn_map */
11021098
lsn_map = fio_get_lsn_map(to_fullpath, arguments->dest_backup->checksum_version,
1103-
dest_file->n_blocks, arguments->horizonLsn,
1099+
dest_file->n_blocks, arguments->shift_lsn,
11041100
dest_file->segno * RELSEG_SIZE, FIO_DB_HOST);
11051101
}
1106-
else
1102+
else if (arguments->incremental_mode == INCR_CHECKSUM)
11071103
{
11081104
checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version,
11091105
dest_file->n_blocks, arguments->dest_backup->stop_lsn,
@@ -1144,7 +1140,7 @@ restore_files(void *arg)
11441140
arguments->restored_bytes += restore_data_file(arguments->parent_chain,
11451141
dest_file, out, to_fullpath,
11461142
arguments->use_bitmap, checksum_map,
1147-
arguments->horizonLsn, lsn_map);
1143+
arguments->shift_lsn, lsn_map);
11481144
}
11491145
else
11501146
{
@@ -1943,7 +1939,8 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
19431939
* Depending on type of incremental restore requirements are differs.
19441940
*/
19451941
void
1946-
check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bool incremental_lsn)
1942+
check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
1943+
IncrRestoreMode incremental_mode)
19471944
{
19481945
uint64 system_id_pgdata;
19491946
bool success = true;
@@ -1993,16 +1990,16 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bo
19931990
* TODO: maybe there should be some other signs, pointing to pg_control
19941991
* desynchronization with cluster state.
19951992
*/
1996-
if (incremental_lsn)
1993+
if (incremental_mode == INCR_LSN)
19971994
{
19981995
snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata);
19991996
if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0)
20001997
{
20011998
elog(WARNING, "Destination directory contains \"backup_control\" file. "
20021999
"This does NOT mean that you should delete this file and retry, only that "
2003-
"lsn-based incremental restore can corrupt data, when applied to instance "
2004-
"with pg_control not synchronized with cluster state."
2005-
"Consider to use checksum-based incremental restore.");
2000+
"incremental restore in 'lsn' mode can produce incorrect result, when applied "
2001+
"to cluster with pg_control not synchronized with cluster state."
2002+
"Consider to use incremental restore in 'checksum' mode");
20062003
success = false;
20072004
}
20082005
}

0 commit comments

Comments
 (0)