From c96a4a38cb4e923821a491548a5e148551430da0 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 30 Jun 2020 10:28:24 +0200 Subject: [PATCH] add incremental backup patches and fix typo: s/BPS/PBS/ --- ...ckup-proxmox-backup-patches-for-qemu.patch | 12 +- ...irty-bitmap-tracking-for-incremental.patch | 243 ++++++++++++++++++ debian/patches/series | 1 + 3 files changed, 250 insertions(+), 6 deletions(-) create mode 100644 debian/patches/pve/0051-PVE-Backup-Add-dirty-bitmap-tracking-for-incremental.patch diff --git a/debian/patches/pve/0030-PVE-Backup-proxmox-backup-patches-for-qemu.patch b/debian/patches/pve/0030-PVE-Backup-proxmox-backup-patches-for-qemu.patch index c92c3e7..3d5dc1a 100644 --- a/debian/patches/pve/0030-PVE-Backup-proxmox-backup-patches-for-qemu.patch +++ b/debian/patches/pve/0030-PVE-Backup-proxmox-backup-patches-for-qemu.patch @@ -99,12 +99,12 @@ index 4c8c375172..4f03881286 100644 + + qmp_backup( + backup_file, -+ false, NULL, // BPS password -+ false, NULL, // BPS keyfile -+ false, NULL, // BPS key_password -+ false, NULL, // BPS fingerprint -+ false, NULL, // BPS backup-id -+ false, 0, // BPS backup-time ++ false, NULL, // PBS password ++ false, NULL, // PBS keyfile ++ false, NULL, // PBS key_password ++ false, NULL, // PBS fingerprint ++ false, NULL, // PBS backup-id ++ false, 0, // PBS backup-time + true, dir ? BACKUP_FORMAT_DIR : BACKUP_FORMAT_VMA, + false, NULL, false, NULL, !!devlist, + devlist, qdict_haskey(qdict, "speed"), speed, &error); diff --git a/debian/patches/pve/0051-PVE-Backup-Add-dirty-bitmap-tracking-for-incremental.patch b/debian/patches/pve/0051-PVE-Backup-Add-dirty-bitmap-tracking-for-incremental.patch new file mode 100644 index 0000000..1184921 --- /dev/null +++ b/debian/patches/pve/0051-PVE-Backup-Add-dirty-bitmap-tracking-for-incremental.patch @@ -0,0 +1,243 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Stefan Reiter +Date: Mon, 29 Jun 2020 11:06:03 +0200 +Subject: [PATCH] PVE-Backup: Add dirty-bitmap tracking for incremental backups + +Uses QEMU's existing MIRROR_SYNC_MODE_BITMAP and a dirty-bitmap on top +of all backed-up drives. This will only execute the data-write callback +for any changed chunks, the PBS rust code will reuse chunks from the +previous index for everything it doesn't receive if reuse_index is true. + +On error or cancellation, remove all dirty bitmaps to ensure +consistency. + +Only supported for PBS backups. + +Signed-off-by: Stefan Reiter +Signed-off-by: Dietmar Maurer +--- + block/monitor/block-hmp-cmds.c | 1 + + proxmox-backup-client.c | 3 +- + proxmox-backup-client.h | 1 + + pve-backup.c | 63 +++++++++++++++++++++++++++++++--- + qapi/block-core.json | 3 ++ + 5 files changed, 65 insertions(+), 6 deletions(-) + +diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c +index 4f03881286..0bc855132a 100644 +--- a/block/monitor/block-hmp-cmds.c ++++ b/block/monitor/block-hmp-cmds.c +@@ -1038,6 +1038,7 @@ void hmp_backup(Monitor *mon, const QDict *qdict) + false, NULL, // PBS fingerprint + false, NULL, // PBS backup-id + false, 0, // PBS backup-time ++ false, false, // PBS incremental + true, dir ? BACKUP_FORMAT_DIR : BACKUP_FORMAT_VMA, + false, NULL, false, NULL, !!devlist, + devlist, qdict_haskey(qdict, "speed"), speed, &error); +diff --git a/proxmox-backup-client.c b/proxmox-backup-client.c +index b7bc7f2574..0e9c584701 100644 +--- a/proxmox-backup-client.c ++++ b/proxmox-backup-client.c +@@ -95,6 +95,7 @@ proxmox_backup_co_register_image( + ProxmoxBackupHandle *pbs, + const char *device_name, + uint64_t size, ++ bool incremental, + Error **errp) + { + Coroutine *co = qemu_coroutine_self(); +@@ -104,7 +105,7 @@ proxmox_backup_co_register_image( + int pbs_res = -1; + + proxmox_backup_register_image_async( +- pbs, device_name, size ,proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err); ++ pbs, device_name, size, incremental, proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err); + qemu_coroutine_yield(); + if (pbs_res < 0) { + if (errp) error_setg(errp, "backup register image failed: %s", pbs_err ? pbs_err : "unknown error"); +diff --git a/proxmox-backup-client.h b/proxmox-backup-client.h +index b311bf8de8..20fd6b1719 100644 +--- a/proxmox-backup-client.h ++++ b/proxmox-backup-client.h +@@ -25,6 +25,7 @@ proxmox_backup_co_register_image( + ProxmoxBackupHandle *pbs, + const char *device_name, + uint64_t size, ++ bool incremental, + Error **errp); + + +diff --git a/pve-backup.c b/pve-backup.c +index bb917ee972..61a8b4d2a4 100644 +--- a/pve-backup.c ++++ b/pve-backup.c +@@ -28,6 +28,8 @@ + * + */ + ++const char *PBS_BITMAP_NAME = "pbs-incremental-dirty-bitmap"; ++ + static struct PVEBackupState { + struct { + // Everithing accessed from qmp_backup_query command is protected using lock +@@ -66,6 +68,7 @@ typedef struct PVEBackupDevInfo { + uint8_t dev_id; + bool completed; + char targetfile[PATH_MAX]; ++ BdrvDirtyBitmap *bitmap; + BlockDriverState *target; + } PVEBackupDevInfo; + +@@ -248,6 +251,18 @@ static void coroutine_fn pvebackup_co_cleanup(void *unused) + if (local_err != NULL) { + pvebackup_propagate_error(local_err); + } ++ } else { ++ // on error or cancel we cannot ensure synchronization of dirty ++ // bitmaps with backup server, so remove all and do full backup next ++ GList *l = backup_state.di_list; ++ while (l) { ++ PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data; ++ l = g_list_next(l); ++ ++ if (di->bitmap) { ++ bdrv_release_dirty_bitmap(di->bitmap); ++ } ++ } + } + + proxmox_backup_disconnect(backup_state.pbs); +@@ -470,12 +485,18 @@ static bool create_backup_jobs(void) { + + assert(di->target != NULL); + ++ MirrorSyncMode sync_mode = MIRROR_SYNC_MODE_FULL; ++ BitmapSyncMode bitmap_mode = BITMAP_SYNC_MODE_NEVER; ++ if (di->bitmap) { ++ sync_mode = MIRROR_SYNC_MODE_BITMAP; ++ bitmap_mode = BITMAP_SYNC_MODE_ON_SUCCESS; ++ } + AioContext *aio_context = bdrv_get_aio_context(di->bs); + aio_context_acquire(aio_context); + + BlockJob *job = backup_job_create( +- NULL, di->bs, di->target, backup_state.speed, MIRROR_SYNC_MODE_FULL, NULL, +- BITMAP_SYNC_MODE_NEVER, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, ++ NULL, di->bs, di->target, backup_state.speed, sync_mode, di->bitmap, ++ bitmap_mode, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, + JOB_DEFAULT, pvebackup_complete_cb, di, 1, NULL, &local_err); + + aio_context_release(aio_context); +@@ -526,6 +547,8 @@ typedef struct QmpBackupTask { + const char *fingerprint; + bool has_fingerprint; + int64_t backup_time; ++ bool has_incremental; ++ bool incremental; + bool has_format; + BackupFormat format; + bool has_config_file; +@@ -658,6 +681,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) + int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M) + firewall_name = "fw.conf"; + ++ bool incremental = task->has_incremental && task->incremental; ++ + char *pbs_err = NULL; + pbs = proxmox_backup_new( + task->backup_file, +@@ -677,7 +702,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) + goto err; + } + +- if (proxmox_backup_co_connect(pbs, task->errp) < 0) ++ int connect_result = proxmox_backup_co_connect(pbs, task->errp); ++ if (connect_result < 0) + goto err; + + /* register all devices */ +@@ -688,9 +714,29 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) + + const char *devname = bdrv_get_device_name(di->bs); + +- int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, task->errp); +- if (dev_id < 0) ++ BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME); ++ ++ bool use_incremental = false; ++ if (incremental) { ++ if (bitmap == NULL) { ++ bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp); ++ if (!bitmap) { ++ goto err; ++ } ++ /* mark entire bitmap as dirty to make full backup first */ ++ bdrv_set_dirty_bitmap(bitmap, 0, di->size); ++ } else { ++ use_incremental = true; ++ } ++ di->bitmap = bitmap; ++ } else if (bitmap != NULL) { ++ bdrv_release_dirty_bitmap(bitmap); ++ } ++ ++ int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, use_incremental, task->errp); ++ if (dev_id < 0) { + goto err; ++ } + + if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) { + goto err; +@@ -823,6 +869,10 @@ err: + PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data; + l = g_list_next(l); + ++ if (di->bitmap) { ++ bdrv_release_dirty_bitmap(di->bitmap); ++ } ++ + if (di->target) { + bdrv_unref(di->target); + } +@@ -864,6 +914,7 @@ UuidInfo *qmp_backup( + bool has_fingerprint, const char *fingerprint, + bool has_backup_id, const char *backup_id, + bool has_backup_time, int64_t backup_time, ++ bool has_incremental, bool incremental, + bool has_format, BackupFormat format, + bool has_config_file, const char *config_file, + bool has_firewall_file, const char *firewall_file, +@@ -882,6 +933,8 @@ UuidInfo *qmp_backup( + .backup_id = backup_id, + .has_backup_time = has_backup_time, + .backup_time = backup_time, ++ .has_incremental = has_incremental, ++ .incremental = incremental, + .has_format = has_format, + .format = format, + .has_config_file = has_config_file, +diff --git a/qapi/block-core.json b/qapi/block-core.json +index 8bdbccb397..f693bebdb4 100644 +--- a/qapi/block-core.json ++++ b/qapi/block-core.json +@@ -815,6 +815,8 @@ + # + # @backup-time: backup timestamp (Unix epoch, required for format 'pbs') + # ++# @incremental: sync incremental changes since last job (optional for format 'pbs') ++# + # Returns: the uuid of the backup job + # + ## +@@ -825,6 +827,7 @@ + '*fingerprint': 'str', + '*backup-id': 'str', + '*backup-time': 'int', ++ '*incremental': 'bool', + '*format': 'BackupFormat', + '*config-file': 'str', + '*firewall-file': 'str', +-- +2.20.1 + diff --git a/debian/patches/series b/debian/patches/series index f508f93..5d6a5d6 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -49,3 +49,4 @@ pve/0047-savevm-async-flush-IOThread-drives-async-before-ente.patch pve/0048-savevm-async-add-debug-timing-prints.patch pve/0049-Add-some-qemu_vfree-statements-to-prevent-memory-lea.patch pve/0050-Fix-backup-for-not-64k-aligned-storages.patch +pve/0051-PVE-Backup-Add-dirty-bitmap-tracking-for-incremental.patch