453 lines
17 KiB
Diff
453 lines
17 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Stefan Reiter <s.reiter@proxmox.com>
|
|
Date: Mon, 29 Jun 2020 11:06:03 +0200
|
|
Subject: [PATCH] PVE-Backup: Add dirty-bitmap tracking for incremental backups
|
|
|
|
Uses QEMU's existing MIRROR_SYNC_MODE_BITMAP and a dirty-bitmap on top
|
|
of all backed-up drives. This will only execute the data-write callback
|
|
for any changed chunks, the PBS rust code will reuse chunks from the
|
|
previous index for everything it doesn't receive if reuse_index is true.
|
|
|
|
On error or cancellation, remove all dirty bitmaps to ensure
|
|
consistency.
|
|
|
|
Add PBS/incremental specific information to query backup info QMP and
|
|
HMP commands.
|
|
|
|
Only supported for PBS backups.
|
|
|
|
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
|
|
Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
|
|
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
|
|
---
|
|
block/monitor/block-hmp-cmds.c | 1 +
|
|
monitor/hmp-cmds.c | 45 ++++++++++----
|
|
proxmox-backup-client.c | 3 +-
|
|
proxmox-backup-client.h | 1 +
|
|
pve-backup.c | 103 ++++++++++++++++++++++++++++++---
|
|
qapi/block-core.json | 12 +++-
|
|
6 files changed, 142 insertions(+), 23 deletions(-)
|
|
|
|
diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c
|
|
index f6668ab01d..3c06734e6d 100644
|
|
--- a/block/monitor/block-hmp-cmds.c
|
|
+++ b/block/monitor/block-hmp-cmds.c
|
|
@@ -1042,6 +1042,7 @@ void hmp_backup(Monitor *mon, const QDict *qdict)
|
|
false, NULL, // PBS fingerprint
|
|
false, NULL, // PBS backup-id
|
|
false, 0, // PBS backup-time
|
|
+ false, false, // PBS incremental
|
|
true, dir ? BACKUP_FORMAT_DIR : BACKUP_FORMAT_VMA,
|
|
false, NULL, false, NULL, !!devlist,
|
|
devlist, qdict_haskey(qdict, "speed"), speed, &error);
|
|
diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
|
|
index b2687eae3a..cfd7a60f32 100644
|
|
--- a/monitor/hmp-cmds.c
|
|
+++ b/monitor/hmp-cmds.c
|
|
@@ -221,19 +221,42 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict)
|
|
monitor_printf(mon, "End time: %s", ctime(&info->end_time));
|
|
}
|
|
|
|
- int per = (info->has_total && info->total &&
|
|
- info->has_transferred && info->transferred) ?
|
|
- (info->transferred * 100)/info->total : 0;
|
|
- int zero_per = (info->has_total && info->total &&
|
|
- info->has_zero_bytes && info->zero_bytes) ?
|
|
- (info->zero_bytes * 100)/info->total : 0;
|
|
monitor_printf(mon, "Backup file: %s\n", info->backup_file);
|
|
monitor_printf(mon, "Backup uuid: %s\n", info->uuid);
|
|
- monitor_printf(mon, "Total size: %zd\n", info->total);
|
|
- monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n",
|
|
- info->transferred, per);
|
|
- monitor_printf(mon, "Zero bytes: %zd (%d%%)\n",
|
|
- info->zero_bytes, zero_per);
|
|
+
|
|
+ if (!(info->has_total && info->total)) {
|
|
+ // this should not happen normally
|
|
+ monitor_printf(mon, "Total size: %d\n", 0);
|
|
+ } else {
|
|
+ bool incremental = false;
|
|
+ size_t total_or_dirty = info->total;
|
|
+ if (info->has_transferred) {
|
|
+ if (info->has_dirty && info->dirty) {
|
|
+ if (info->dirty < info->total) {
|
|
+ total_or_dirty = info->dirty;
|
|
+ incremental = true;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ int per = (info->transferred * 100)/total_or_dirty;
|
|
+
|
|
+ monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full");
|
|
+
|
|
+ int zero_per = (info->has_zero_bytes && info->zero_bytes) ?
|
|
+ (info->zero_bytes * 100)/info->total : 0;
|
|
+ monitor_printf(mon, "Total size: %zd\n", info->total);
|
|
+ monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n",
|
|
+ info->transferred, per);
|
|
+ monitor_printf(mon, "Zero bytes: %zd (%d%%)\n",
|
|
+ info->zero_bytes, zero_per);
|
|
+
|
|
+ if (info->has_reused) {
|
|
+ int reused_per = (info->reused * 100)/total_or_dirty;
|
|
+ monitor_printf(mon, "Reused bytes: %zd (%d%%)\n",
|
|
+ info->reused, reused_per);
|
|
+ }
|
|
+ }
|
|
}
|
|
|
|
qapi_free_BackupStatus(info);
|
|
diff --git a/proxmox-backup-client.c b/proxmox-backup-client.c
|
|
index a8f6653a81..4ce7bc0b5e 100644
|
|
--- a/proxmox-backup-client.c
|
|
+++ b/proxmox-backup-client.c
|
|
@@ -89,6 +89,7 @@ proxmox_backup_co_register_image(
|
|
ProxmoxBackupHandle *pbs,
|
|
const char *device_name,
|
|
uint64_t size,
|
|
+ bool incremental,
|
|
Error **errp)
|
|
{
|
|
Coroutine *co = qemu_coroutine_self();
|
|
@@ -98,7 +99,7 @@ proxmox_backup_co_register_image(
|
|
int pbs_res = -1;
|
|
|
|
proxmox_backup_register_image_async(
|
|
- pbs, device_name, size ,proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err);
|
|
+ pbs, device_name, size, incremental, proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err);
|
|
qemu_coroutine_yield();
|
|
if (pbs_res < 0) {
|
|
if (errp) error_setg(errp, "backup register image failed: %s", pbs_err ? pbs_err : "unknown error");
|
|
diff --git a/proxmox-backup-client.h b/proxmox-backup-client.h
|
|
index 1dda8b7d8f..8cbf645b2c 100644
|
|
--- a/proxmox-backup-client.h
|
|
+++ b/proxmox-backup-client.h
|
|
@@ -32,6 +32,7 @@ proxmox_backup_co_register_image(
|
|
ProxmoxBackupHandle *pbs,
|
|
const char *device_name,
|
|
uint64_t size,
|
|
+ bool incremental,
|
|
Error **errp);
|
|
|
|
|
|
diff --git a/pve-backup.c b/pve-backup.c
|
|
index 88f5ee133f..1c49cd178d 100644
|
|
--- a/pve-backup.c
|
|
+++ b/pve-backup.c
|
|
@@ -28,6 +28,8 @@
|
|
*
|
|
*/
|
|
|
|
+const char *PBS_BITMAP_NAME = "pbs-incremental-dirty-bitmap";
|
|
+
|
|
static struct PVEBackupState {
|
|
struct {
|
|
// Everithing accessed from qmp_backup_query command is protected using lock
|
|
@@ -39,7 +41,9 @@ static struct PVEBackupState {
|
|
uuid_t uuid;
|
|
char uuid_str[37];
|
|
size_t total;
|
|
+ size_t dirty;
|
|
size_t transferred;
|
|
+ size_t reused;
|
|
size_t zero_bytes;
|
|
} stat;
|
|
int64_t speed;
|
|
@@ -66,6 +70,7 @@ typedef struct PVEBackupDevInfo {
|
|
uint8_t dev_id;
|
|
bool completed;
|
|
char targetfile[PATH_MAX];
|
|
+ BdrvDirtyBitmap *bitmap;
|
|
BlockDriverState *target;
|
|
} PVEBackupDevInfo;
|
|
|
|
@@ -105,11 +110,12 @@ static bool pvebackup_error_or_canceled(void)
|
|
return error_or_canceled;
|
|
}
|
|
|
|
-static void pvebackup_add_transfered_bytes(size_t transferred, size_t zero_bytes)
|
|
+static void pvebackup_add_transfered_bytes(size_t transferred, size_t zero_bytes, size_t reused)
|
|
{
|
|
qemu_mutex_lock(&backup_state.stat.lock);
|
|
backup_state.stat.zero_bytes += zero_bytes;
|
|
backup_state.stat.transferred += transferred;
|
|
+ backup_state.stat.reused += reused;
|
|
qemu_mutex_unlock(&backup_state.stat.lock);
|
|
}
|
|
|
|
@@ -148,7 +154,8 @@ pvebackup_co_dump_pbs_cb(
|
|
pvebackup_propagate_error(local_err);
|
|
return pbs_res;
|
|
} else {
|
|
- pvebackup_add_transfered_bytes(size, !buf ? size : 0);
|
|
+ size_t reused = (pbs_res == 0) ? size : 0;
|
|
+ pvebackup_add_transfered_bytes(size, !buf ? size : 0, reused);
|
|
}
|
|
|
|
return size;
|
|
@@ -208,11 +215,11 @@ pvebackup_co_dump_vma_cb(
|
|
} else {
|
|
if (remaining >= VMA_CLUSTER_SIZE) {
|
|
assert(ret == VMA_CLUSTER_SIZE);
|
|
- pvebackup_add_transfered_bytes(VMA_CLUSTER_SIZE, zero_bytes);
|
|
+ pvebackup_add_transfered_bytes(VMA_CLUSTER_SIZE, zero_bytes, 0);
|
|
remaining -= VMA_CLUSTER_SIZE;
|
|
} else {
|
|
assert(ret == remaining);
|
|
- pvebackup_add_transfered_bytes(remaining, zero_bytes);
|
|
+ pvebackup_add_transfered_bytes(remaining, zero_bytes, 0);
|
|
remaining = 0;
|
|
}
|
|
}
|
|
@@ -248,6 +255,18 @@ static void coroutine_fn pvebackup_co_cleanup(void *unused)
|
|
if (local_err != NULL) {
|
|
pvebackup_propagate_error(local_err);
|
|
}
|
|
+ } else {
|
|
+ // on error or cancel we cannot ensure synchronization of dirty
|
|
+ // bitmaps with backup server, so remove all and do full backup next
|
|
+ GList *l = backup_state.di_list;
|
|
+ while (l) {
|
|
+ PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data;
|
|
+ l = g_list_next(l);
|
|
+
|
|
+ if (di->bitmap) {
|
|
+ bdrv_release_dirty_bitmap(di->bitmap);
|
|
+ }
|
|
+ }
|
|
}
|
|
|
|
proxmox_backup_disconnect(backup_state.pbs);
|
|
@@ -303,6 +322,12 @@ static void pvebackup_complete_cb(void *opaque, int ret)
|
|
// remove self from job queue
|
|
backup_state.di_list = g_list_remove(backup_state.di_list, di);
|
|
|
|
+ if (di->bitmap && ret < 0) {
|
|
+ // on error or cancel we cannot ensure synchronization of dirty
|
|
+ // bitmaps with backup server, so remove all and do full backup next
|
|
+ bdrv_release_dirty_bitmap(di->bitmap);
|
|
+ }
|
|
+
|
|
g_free(di);
|
|
|
|
qemu_mutex_unlock(&backup_state.backup_mutex);
|
|
@@ -472,12 +497,18 @@ static bool create_backup_jobs(void) {
|
|
|
|
assert(di->target != NULL);
|
|
|
|
+ MirrorSyncMode sync_mode = MIRROR_SYNC_MODE_FULL;
|
|
+ BitmapSyncMode bitmap_mode = BITMAP_SYNC_MODE_NEVER;
|
|
+ if (di->bitmap) {
|
|
+ sync_mode = MIRROR_SYNC_MODE_BITMAP;
|
|
+ bitmap_mode = BITMAP_SYNC_MODE_ON_SUCCESS;
|
|
+ }
|
|
AioContext *aio_context = bdrv_get_aio_context(di->bs);
|
|
aio_context_acquire(aio_context);
|
|
|
|
BlockJob *job = backup_job_create(
|
|
- NULL, di->bs, di->target, backup_state.speed, MIRROR_SYNC_MODE_FULL, NULL,
|
|
- BITMAP_SYNC_MODE_NEVER, false, NULL, &perf, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
|
|
+ NULL, di->bs, di->target, backup_state.speed, sync_mode, di->bitmap,
|
|
+ bitmap_mode, false, NULL, &perf, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT,
|
|
JOB_DEFAULT, pvebackup_complete_cb, di, NULL, &local_err);
|
|
|
|
aio_context_release(aio_context);
|
|
@@ -528,6 +559,8 @@ typedef struct QmpBackupTask {
|
|
const char *fingerprint;
|
|
bool has_fingerprint;
|
|
int64_t backup_time;
|
|
+ bool has_use_dirty_bitmap;
|
|
+ bool use_dirty_bitmap;
|
|
bool has_format;
|
|
BackupFormat format;
|
|
bool has_config_file;
|
|
@@ -619,6 +652,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
|
|
}
|
|
|
|
size_t total = 0;
|
|
+ size_t dirty = 0;
|
|
|
|
l = di_list;
|
|
while (l) {
|
|
@@ -656,6 +690,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
|
|
int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M)
|
|
firewall_name = "fw.conf";
|
|
|
|
+ bool use_dirty_bitmap = task->has_use_dirty_bitmap && task->use_dirty_bitmap;
|
|
+
|
|
char *pbs_err = NULL;
|
|
pbs = proxmox_backup_new(
|
|
task->backup_file,
|
|
@@ -675,7 +711,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
|
|
goto err;
|
|
}
|
|
|
|
- if (proxmox_backup_co_connect(pbs, task->errp) < 0)
|
|
+ int connect_result = proxmox_backup_co_connect(pbs, task->errp);
|
|
+ if (connect_result < 0)
|
|
goto err;
|
|
|
|
/* register all devices */
|
|
@@ -686,9 +723,40 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
|
|
|
|
const char *devname = bdrv_get_device_name(di->bs);
|
|
|
|
- int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, task->errp);
|
|
- if (dev_id < 0)
|
|
+ BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME);
|
|
+ bool expect_only_dirty = false;
|
|
+
|
|
+ if (use_dirty_bitmap) {
|
|
+ if (bitmap == NULL) {
|
|
+ bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp);
|
|
+ if (!bitmap) {
|
|
+ goto err;
|
|
+ }
|
|
+ } else {
|
|
+ expect_only_dirty = proxmox_backup_check_incremental(pbs, devname, di->size) != 0;
|
|
+ }
|
|
+
|
|
+ if (expect_only_dirty) {
|
|
+ dirty += bdrv_get_dirty_count(bitmap);
|
|
+ } else {
|
|
+ /* mark entire bitmap as dirty to make full backup */
|
|
+ bdrv_set_dirty_bitmap(bitmap, 0, di->size);
|
|
+ dirty += di->size;
|
|
+ }
|
|
+ di->bitmap = bitmap;
|
|
+ } else {
|
|
+ dirty += di->size;
|
|
+
|
|
+ /* after a full backup the old dirty bitmap is invalid anyway */
|
|
+ if (bitmap != NULL) {
|
|
+ bdrv_release_dirty_bitmap(bitmap);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, expect_only_dirty, task->errp);
|
|
+ if (dev_id < 0) {
|
|
goto err;
|
|
+ }
|
|
|
|
if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) {
|
|
goto err;
|
|
@@ -697,6 +765,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
|
|
di->dev_id = dev_id;
|
|
}
|
|
} else if (format == BACKUP_FORMAT_VMA) {
|
|
+ dirty = total;
|
|
+
|
|
vmaw = vma_writer_create(task->backup_file, uuid, &local_err);
|
|
if (!vmaw) {
|
|
if (local_err) {
|
|
@@ -724,6 +794,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
|
|
}
|
|
}
|
|
} else if (format == BACKUP_FORMAT_DIR) {
|
|
+ dirty = total;
|
|
+
|
|
if (mkdir(task->backup_file, 0640) != 0) {
|
|
error_setg_errno(task->errp, errno, "can't create directory '%s'\n",
|
|
task->backup_file);
|
|
@@ -796,8 +868,10 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
|
|
char *uuid_str = g_strdup(backup_state.stat.uuid_str);
|
|
|
|
backup_state.stat.total = total;
|
|
+ backup_state.stat.dirty = dirty;
|
|
backup_state.stat.transferred = 0;
|
|
backup_state.stat.zero_bytes = 0;
|
|
+ backup_state.stat.reused = format == BACKUP_FORMAT_PBS && dirty >= total ? 0 : total - dirty;
|
|
|
|
qemu_mutex_unlock(&backup_state.stat.lock);
|
|
|
|
@@ -821,6 +895,10 @@ err:
|
|
PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data;
|
|
l = g_list_next(l);
|
|
|
|
+ if (di->bitmap) {
|
|
+ bdrv_release_dirty_bitmap(di->bitmap);
|
|
+ }
|
|
+
|
|
if (di->target) {
|
|
bdrv_unref(di->target);
|
|
}
|
|
@@ -862,6 +940,7 @@ UuidInfo *qmp_backup(
|
|
bool has_fingerprint, const char *fingerprint,
|
|
bool has_backup_id, const char *backup_id,
|
|
bool has_backup_time, int64_t backup_time,
|
|
+ bool has_use_dirty_bitmap, bool use_dirty_bitmap,
|
|
bool has_format, BackupFormat format,
|
|
bool has_config_file, const char *config_file,
|
|
bool has_firewall_file, const char *firewall_file,
|
|
@@ -880,6 +959,8 @@ UuidInfo *qmp_backup(
|
|
.backup_id = backup_id,
|
|
.has_backup_time = has_backup_time,
|
|
.backup_time = backup_time,
|
|
+ .has_use_dirty_bitmap = has_use_dirty_bitmap,
|
|
+ .use_dirty_bitmap = use_dirty_bitmap,
|
|
.has_format = has_format,
|
|
.format = format,
|
|
.has_config_file = has_config_file,
|
|
@@ -948,10 +1029,14 @@ BackupStatus *qmp_query_backup(Error **errp)
|
|
|
|
info->has_total = true;
|
|
info->total = backup_state.stat.total;
|
|
+ info->has_dirty = true;
|
|
+ info->dirty = backup_state.stat.dirty;
|
|
info->has_zero_bytes = true;
|
|
info->zero_bytes = backup_state.stat.zero_bytes;
|
|
info->has_transferred = true;
|
|
info->transferred = backup_state.stat.transferred;
|
|
+ info->has_reused = true;
|
|
+ info->reused = backup_state.stat.reused;
|
|
|
|
qemu_mutex_unlock(&backup_state.stat.lock);
|
|
|
|
diff --git a/qapi/block-core.json b/qapi/block-core.json
|
|
index 551ee28275..b9d6f52f0c 100644
|
|
--- a/qapi/block-core.json
|
|
+++ b/qapi/block-core.json
|
|
@@ -757,8 +757,13 @@
|
|
#
|
|
# @total: total amount of bytes involved in the backup process
|
|
#
|
|
+# @dirty: with incremental mode (PBS) this is the amount of bytes involved
|
|
+# in the backup process which are marked dirty.
|
|
+#
|
|
# @transferred: amount of bytes already backed up.
|
|
#
|
|
+# @reused: amount of bytes reused due to deduplication.
|
|
+#
|
|
# @zero-bytes: amount of 'zero' bytes detected.
|
|
#
|
|
# @start-time: time (epoch) when backup job started.
|
|
@@ -771,8 +776,8 @@
|
|
#
|
|
##
|
|
{ 'struct': 'BackupStatus',
|
|
- 'data': {'*status': 'str', '*errmsg': 'str', '*total': 'int',
|
|
- '*transferred': 'int', '*zero-bytes': 'int',
|
|
+ 'data': {'*status': 'str', '*errmsg': 'str', '*total': 'int', '*dirty': 'int',
|
|
+ '*transferred': 'int', '*zero-bytes': 'int', '*reused': 'int',
|
|
'*start-time': 'int', '*end-time': 'int',
|
|
'*backup-file': 'str', '*uuid': 'str' } }
|
|
|
|
@@ -815,6 +820,8 @@
|
|
#
|
|
# @backup-time: backup timestamp (Unix epoch, required for format 'pbs')
|
|
#
|
|
+# @use-dirty-bitmap: use dirty bitmap to detect incremental changes since last job (optional for format 'pbs')
|
|
+#
|
|
# Returns: the uuid of the backup job
|
|
#
|
|
##
|
|
@@ -825,6 +832,7 @@
|
|
'*fingerprint': 'str',
|
|
'*backup-id': 'str',
|
|
'*backup-time': 'int',
|
|
+ '*use-dirty-bitmap': 'bool',
|
|
'*format': 'BackupFormat',
|
|
'*config-file': 'str',
|
|
'*firewall-file': 'str',
|