c/pve-qemu-qoup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add incremental backup patches

Browse Source 
and fix typo: s/BPS/PBS/
This commit is contained in:
Dietmar Maurer 2020-06-30 10:28:24 +02:00
parent f0b53ef0b2
commit c96a4a38cb
3 changed files with 250 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
debian/patches/pve/0030-PVE-Backup-proxmox-backup-patches-for-qemu.patch vendored
View File

@ -99,12 +99,12 @@ index 4c8c375172..4f03881286 100644
+
+ qmp_backup(
+ backup_file,
+ false, NULL, // BPS password
+ false, NULL, // BPS keyfile
+ false, NULL, // BPS key_password
+ false, NULL, // BPS fingerprint
+ false, NULL, // BPS backup-id
+ false, 0, // BPS backup-time
+ false, NULL, // PBS password
+ false, NULL, // PBS keyfile
+ false, NULL, // PBS key_password
+ false, NULL, // PBS fingerprint
+ false, NULL, // PBS backup-id
+ false, 0, // PBS backup-time
+ true, dir ? BACKUP_FORMAT_DIR : BACKUP_FORMAT_VMA,
+ false, NULL, false, NULL, !!devlist,
+ devlist, qdict_haskey(qdict, "speed"), speed, &error);

243
debian/patches/pve/0051-PVE-Backup-Add-dirty-bitmap-tracking-for-incremental.patch vendored Normal file
View File

@ -0,0 +1,243 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Stefan Reiter <s.reiter@proxmox.com>
Date: Mon, 29 Jun 2020 11:06:03 +0200
Subject: [PATCH] PVE-Backup: Add dirty-bitmap tracking for incremental backups
Uses QEMU's existing MIRROR_SYNC_MODE_BITMAP and a dirty-bitmap on top
of all backed-up drives. This will only execute the data-write callback
for any changed chunks, the PBS rust code will reuse chunks from the
previous index for everything it doesn't receive if reuse_index is true.
On error or cancellation, remove all dirty bitmaps to ensure
consistency.
Only supported for PBS backups.
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
---
block/monitor/block-hmp-cmds.c | 1 +
proxmox-backup-client.c | 3 +-
proxmox-backup-client.h | 1 +
pve-backup.c | 63 +++++++++++++++++++++++++++++++---
qapi/block-core.json | 3 ++
5 files changed, 65 insertions(+), 6 deletions(-)
diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
index 4f03881286..0bc855132a 100644
--- a/block/monitor/block-hmp-cmds.c
+++ b/block/monitor/block-hmp-cmds.c
@@ -1038,6 +1038,7 @@ void hmp_backup(Monitor *mon, const QDict *qdict)
false, NULL, // PBS fingerprint
false, NULL, // PBS backup-id
false, 0, // PBS backup-time
+ false, false, // PBS incremental
true, dir ? BACKUP_FORMAT_DIR : BACKUP_FORMAT_VMA,
false, NULL, false, NULL, !!devlist,
devlist, qdict_haskey(qdict, "speed"), speed, &error);
diff --git a/proxmox-backup-client.c b/proxmox-backup-client.c
index b7bc7f2574..0e9c584701 100644
--- a/proxmox-backup-client.c
+++ b/proxmox-backup-client.c
@@ -95,6 +95,7 @@ proxmox_backup_co_register_image(
ProxmoxBackupHandle *pbs,
const char *device_name,
uint64_t size,
+ bool incremental,
Error **errp)
{
Coroutine *co = qemu_coroutine_self();
@@ -104,7 +105,7 @@ proxmox_backup_co_register_image(
int pbs_res = -1;
proxmox_backup_register_image_async(
- pbs, device_name, size ,proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err);
+ pbs, device_name, size, incremental, proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err);
qemu_coroutine_yield();
if (pbs_res < 0) {
if (errp) error_setg(errp, "backup register image failed: %s", pbs_err ? pbs_err : "unknown error");
diff --git a/proxmox-backup-client.h b/proxmox-backup-client.h
index b311bf8de8..20fd6b1719 100644
--- a/proxmox-backup-client.h
+++ b/proxmox-backup-client.h
@@ -25,6 +25,7 @@ proxmox_backup_co_register_image(
ProxmoxBackupHandle *pbs,
const char *device_name,
uint64_t size,
+ bool incremental,
Error **errp);
diff --git a/pve-backup.c b/pve-backup.c
index bb917ee972..61a8b4d2a4 100644
--- a/pve-backup.c
+++ b/pve-backup.c
@@ -28,6 +28,8 @@
*
*/
+const char *PBS_BITMAP_NAME = "pbs-incremental-dirty-bitmap";
+
static struct PVEBackupState {
struct {
// Everithing accessed from qmp_backup_query command is protected using lock
@@ -66,6 +68,7 @@ typedef struct PVEBackupDevInfo {
uint8_t dev_id;
bool completed;
char targetfile[PATH_MAX];
+ BdrvDirtyBitmap *bitmap;
BlockDriverState *target;
} PVEBackupDevInfo;
@@ -248,6 +251,18 @@ static void coroutine_fn pvebackup_co_cleanup(void *unused)
if (local_err != NULL) {
pvebackup_propagate_error(local_err);
}
+ } else {
+ // on error or cancel we cannot ensure synchronization of dirty
+ // bitmaps with backup server, so remove all and do full backup next
+ GList *l = backup_state.di_list;
+ while (l) {
+ PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data;
+ l = g_list_next(l);
+
+ if (di->bitmap) {
+ bdrv_release_dirty_bitmap(di->bitmap);
+ }
+ }
}
proxmox_backup_disconnect(backup_state.pbs);
@@ -470,12 +485,18 @@ static bool create_backup_jobs(void) {
assert(di->target != NULL);
+ MirrorSyncMode sync_mode = MIRROR_SYNC_MODE_FULL;
+ BitmapSyncMode bitmap_mode = BITMAP_SYNC_MODE_NEVER;
+ if (di->bitmap) {
+ sync_mode = MIRROR_SYNC_MODE_BITMAP;
+ bitmap_mode = BITMAP_SYNC_MODE_ON_SUCCESS;
+ }
AioContext *aio_context = bdrv_get_aio_context(di->bs);
aio_context_acquire(aio_context);
BlockJob *job = backup_job_create(
- NULL, di->bs, di->target, backup_state.speed, MIRROR_SYNC_MODE_FULL, NULL,
- BITMAP_SYNC_MODE_NEVER, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
+ NULL, di->bs, di->target, backup_state.speed, sync_mode, di->bitmap,
+ bitmap_mode, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
JOB_DEFAULT, pvebackup_complete_cb, di, 1, NULL, &local_err);
aio_context_release(aio_context);
@@ -526,6 +547,8 @@ typedef struct QmpBackupTask {
const char *fingerprint;
bool has_fingerprint;
int64_t backup_time;
+ bool has_incremental;
+ bool incremental;
bool has_format;
BackupFormat format;
bool has_config_file;
@@ -658,6 +681,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M)
firewall_name = "fw.conf";
+ bool incremental = task->has_incremental && task->incremental;
+
char *pbs_err = NULL;
pbs = proxmox_backup_new(
task->backup_file,
@@ -677,7 +702,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
goto err;
}
- if (proxmox_backup_co_connect(pbs, task->errp) < 0)
+ int connect_result = proxmox_backup_co_connect(pbs, task->errp);
+ if (connect_result < 0)
goto err;
/* register all devices */
@@ -688,9 +714,29 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
const char *devname = bdrv_get_device_name(di->bs);
- int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, task->errp);
- if (dev_id < 0)
+ BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME);
+
+ bool use_incremental = false;
+ if (incremental) {
+ if (bitmap == NULL) {
+ bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp);
+ if (!bitmap) {
+ goto err;
+ }
+ /* mark entire bitmap as dirty to make full backup first */
+ bdrv_set_dirty_bitmap(bitmap, 0, di->size);
+ } else {
+ use_incremental = true;
+ }
+ di->bitmap = bitmap;
+ } else if (bitmap != NULL) {
+ bdrv_release_dirty_bitmap(bitmap);
+ }
+
+ int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, use_incremental, task->errp);
+ if (dev_id < 0) {
goto err;
+ }
if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) {
goto err;
@@ -823,6 +869,10 @@ err:
PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data;
l = g_list_next(l);
+ if (di->bitmap) {
+ bdrv_release_dirty_bitmap(di->bitmap);
+ }
+
if (di->target) {
bdrv_unref(di->target);
}
@@ -864,6 +914,7 @@ UuidInfo *qmp_backup(
bool has_fingerprint, const char *fingerprint,
bool has_backup_id, const char *backup_id,
bool has_backup_time, int64_t backup_time,
+ bool has_incremental, bool incremental,
bool has_format, BackupFormat format,
bool has_config_file, const char *config_file,
bool has_firewall_file, const char *firewall_file,
@@ -882,6 +933,8 @@ UuidInfo *qmp_backup(
.backup_id = backup_id,
.has_backup_time = has_backup_time,
.backup_time = backup_time,
+ .has_incremental = has_incremental,
+ .incremental = incremental,
.has_format = has_format,
.format = format,
.has_config_file = has_config_file,
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 8bdbccb397..f693bebdb4 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -815,6 +815,8 @@
#
# @backup-time: backup timestamp (Unix epoch, required for format 'pbs')
#
+# @incremental: sync incremental changes since last job (optional for format 'pbs')
+#
# Returns: the uuid of the backup job
#
##
@@ -825,6 +827,7 @@
'*fingerprint': 'str',
'*backup-id': 'str',
'*backup-time': 'int',
+ '*incremental': 'bool',
'*format': 'BackupFormat',
'*config-file': 'str',
'*firewall-file': 'str',
--
2.20.1

1
debian/patches/series vendored
View File

@ -49,3 +49,4 @@ pve/0047-savevm-async-flush-IOThread-drives-async-before-ente.patch
pve/0048-savevm-async-add-debug-timing-prints.patch
pve/0049-Add-some-qemu_vfree-statements-to-prevent-memory-lea.patch
pve/0050-Fix-backup-for-not-64k-aligned-storages.patch
pve/0051-PVE-Backup-Add-dirty-bitmap-tracking-for-incremental.patch