From 2479595602b002ac4bf4110e803423e93a0a88a1 Mon Sep 17 00:00:00 2001 From: dvst <4395369+triwada@users.noreply.github.com> Date: Tue, 6 Jul 2021 18:14:20 +1000 Subject: [PATCH] Add files via upload --- src/catalog.c | 4 ++ src/delete.c | 159 +++++++++++++++++++++++++++++++++++++++++++++ src/help.c | 22 +++++++ src/pg_probackup.c | 8 +++ src/pg_probackup.h | 6 +- src/util.c | 2 + 6 files changed, 200 insertions(+), 1 deletion(-) diff --git a/src/catalog.c b/src/catalog.c index 3ba17e9fd..0c4a0427f 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2623,6 +2623,10 @@ readBackupControlFile(const char *path) backup->status = BACKUP_STATUS_DELETING; else if (strcmp(status, "DELETED") == 0) backup->status = BACKUP_STATUS_DELETED; + else if (strcmp(status, "DETACHING") == 0) + backup->status = BACKUP_STATUS_DETACHING; + else if (strcmp(status, "DETACHED") == 0) + backup->status = BACKUP_STATUS_DETACHED; else if (strcmp(status, "DONE") == 0) backup->status = BACKUP_STATUS_DONE; else if (strcmp(status, "ORPHAN") == 0) diff --git a/src/delete.c b/src/delete.c index d1afa2874..dbfd47b33 100644 --- a/src/delete.c +++ b/src/delete.c @@ -113,6 +113,165 @@ do_delete(time_t backup_id) parray_free(backup_list); } +void +do_detach(time_t backup_id) +{ + int i; + parray *backup_list, + *delete_list; + pgBackup *target_backup = NULL; + size_t size_to_delete = 0; + char size_to_delete_pretty[20]; + + /* Get complete list of backups */ + backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + + delete_list = parray_new(); + + /* Find backup to be deleted and make increment backups array to be deleted */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + + if (backup->start_time == backup_id) + { + target_backup = backup; + break; + } + } + + /* sanity */ + if (!target_backup) + elog(ERROR, "Failed to find backup %s, cannot detach", base36enc(backup_id)); + + /* form delete list */ + for (i = 0; i < parray_num(backup_list); i++) + { + pgBackup *backup = (pgBackup *) parray_get(backup_list, i); + + /* check if backup is descendant of delete target */ + if (is_parent(target_backup->start_time, backup, true)) + { + parray_append(delete_list, backup); + + elog(LOG, "Backup %s %s be detached", + base36enc(backup->start_time), dry_run? "can":"will"); + + size_to_delete += backup->data_bytes; + if (backup->stream) + size_to_delete += backup->wal_bytes; + } + } + + /* Report the resident size to delete */ + if (size_to_delete >= 0) + { + pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); + elog(INFO, "Resident data size to free by detach of backup %s : %s", + base36enc(target_backup->start_time), size_to_delete_pretty); + } + + if (!dry_run) + { + /* Lock marked for delete backups */ + catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false, true); + + /* Delete backups from the end of list */ + for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) + { + pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); + + if (interrupted) + elog(ERROR, "interrupted during detach backup"); + + detach_backup_files(backup); + } + } + + + /* cleanup */ + parray_free(delete_list); + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); +} + +/* +* Detach backup files of the backup and update the status of the backup to +* BACKUP_STATUS_DETACHED. +* TODO: delete files on multiple threads +*/ +void +detach_backup_files(pgBackup *backup) +{ + size_t i; + char timestamp[100]; + parray *files; + size_t num_files; + char full_path[MAXPGPATH]; + + /* + * If the backup was detached already, there is nothing to do. + */ + if (backup->status == BACKUP_STATUS_DETACHED) + { + elog(WARNING, "Backup %s already detached", + base36enc(backup->start_time)); + return; + } + + if (backup->recovery_time) + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); + else + time2iso(timestamp, lengthof(timestamp), backup->start_time, false); + + elog(INFO, "Detach: %s %s", + base36enc(backup->start_time), timestamp); + elog(INFO, "Backup paths:\n root_dir = %s\n database_dir = %s\n", + backup->root_dir, backup->database_dir); + + + /* + * Update STATUS to BACKUP_STATUS_DETACHING in preparation for the case which + * the error occurs before deleting all backup files. + */ + write_backup_status(backup, BACKUP_STATUS_DETACHING, instance_name, false); + elog(INFO, "Set status in control file: DETACHING"); + + /* list files to be deleted */ + files = parray_new(); + dir_list_file(files, backup->database_dir, false, false, true, false, false, 0, FIO_BACKUP_HOST); + + /* delete leaf node first */ + parray_qsort(files, pgFileCompareRelPathWithExternalDesc); + num_files = parray_num(files); + for (i = 0; i < num_files; i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + join_path_components(full_path, backup->database_dir, file->rel_path); + + if (interrupted) + elog(ERROR, "interrupted during detach backup"); + + if (progress) + elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"", + i + 1, num_files, full_path); + + pgFileDelete(file->mode, full_path); + } + + parray_walk(files, pgFileFree); + parray_free(files); + backup->status = BACKUP_STATUS_DETACHED; + + /* Update STATUS to BACKUP_STATUS_DETACHED */ + write_backup_status(backup, BACKUP_STATUS_DETACHED, instance_name, true); + elog(INFO, "Set status in control file: DETACHED"); + + return; +} + + /* * Merge and purge backups by retention policy. Retention policy is configured by * retention_redundancy and retention_window variables. diff --git a/src/help.c b/src/help.c index f72dc90dc..d02a20108 100644 --- a/src/help.c +++ b/src/help.c @@ -15,6 +15,7 @@ static void help_restore(void); static void help_validate(void); static void help_show(void); static void help_delete(void); +static void help_detach(void); static void help_merge(void); static void help_set_backup(void); static void help_set_config(void); @@ -40,6 +41,8 @@ help_command(char *command) help_show(); else if (strcmp(command, "delete") == 0) help_delete(); + else if (strcmp(command, "detach") == 0) + help_detach(); else if (strcmp(command, "merge") == 0) help_merge(); else if (strcmp(command, "set-backup") == 0) @@ -200,6 +203,11 @@ help_pg_probackup(void) printf(_(" [--dry-run] [--no-validate] [--no-sync]\n")); printf(_(" [--help]\n")); + printf(_("\n %s detach -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-j num-threads] [--progress]\n")); + printf(_(" [-i backup-id]\n")); + printf(_(" [--help]\n")); + printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); printf(_(" -i backup-id [--progress] [-j num-threads]\n")); printf(_(" [--no-validate] [--no-sync]\n")); @@ -680,6 +688,20 @@ help_delete(void) printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); } +static void +help_detach(void) +{ + printf(_("\n%s detach -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_(" [-j num-threads] [--progress]\n")); + printf(_(" [-i backup-id ]\n\n")); + + printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -i, --backup-id=backup-id backup to detach\n")); + printf(_(" -j, --threads=NUM number of parallel threads\n")); + printf(_(" --progress show progress\n\n")); +} + static void help_merge(void) { diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 854493bdc..689cd9c52 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -39,6 +39,7 @@ typedef enum ProbackupSubcmd RESTORE_CMD, VALIDATE_CMD, DELETE_CMD, + DETACH_CMD, MERGE_CMD, SHOW_CMD, SET_CONFIG_CMD, @@ -334,6 +335,8 @@ main(int argc, char *argv[]) backup_subcmd = VALIDATE_CMD; else if (strcmp(argv[1], "delete") == 0) backup_subcmd = DELETE_CMD; + else if (strcmp(argv[1], "detach") == 0) + backup_subcmd = DETACH_CMD; else if (strcmp(argv[1], "merge") == 0) backup_subcmd = MERGE_CMD; else if (strcmp(argv[1], "show") == 0) @@ -405,6 +408,7 @@ main(int argc, char *argv[]) backup_subcmd == RESTORE_CMD || backup_subcmd == VALIDATE_CMD || backup_subcmd == DELETE_CMD || + backup_subcmd == DETACH_CMD || backup_subcmd == MERGE_CMD || backup_subcmd == SET_CONFIG_CMD || backup_subcmd == SET_BACKUP_CMD) @@ -663,6 +667,7 @@ main(int argc, char *argv[]) if (backup_subcmd != RESTORE_CMD && backup_subcmd != VALIDATE_CMD && backup_subcmd != DELETE_CMD && + backup_subcmd != DETACH_CMD && backup_subcmd != MERGE_CMD && backup_subcmd != SET_BACKUP_CMD && backup_subcmd != SHOW_CMD) @@ -850,6 +855,9 @@ main(int argc, char *argv[]) else do_delete(current.backup_id); break; + case DETACH_CMD: + do_detach(current.backup_id); + break; case MERGE_CMD: do_merge(current.backup_id, no_validate, no_sync); break; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index fca08bdac..6eb7d2997 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -284,7 +284,9 @@ typedef enum BackupStatus BACKUP_STATUS_DELETED, /* data files have been deleted */ BACKUP_STATUS_DONE, /* completed but not validated yet */ BACKUP_STATUS_ORPHAN, /* backup validity is unknown but at least one parent backup is corrupted */ - BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ + BACKUP_STATUS_DETACHING, /* data files are being detached */ + BACKUP_STATUS_DETACHED, /* data files have been detached */ + BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ } BackupStatus; typedef enum BackupMode @@ -869,6 +871,8 @@ extern int do_show(const char *instance_name, time_t requested_backup_id, bool s /* in delete.c */ extern void do_delete(time_t backup_id); extern void delete_backup_files(pgBackup *backup); +extern void do_detach(time_t backup_id); +extern void detach_backup_files(pgBackup *backup); extern void do_retention(bool no_validate, bool no_sync); extern int do_delete_instance(void); extern void do_delete_status(InstanceConfig *instance_config, const char *status); diff --git a/src/util.c b/src/util.c index 9fd0114bb..576d8c45f 100644 --- a/src/util.c +++ b/src/util.c @@ -30,6 +30,8 @@ static const char *statusName[] = "DELETED", "DONE", "ORPHAN", + "DETACHING", + "DETACHED", "CORRUPT" };