c/pve-qemu-qoup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
db5d2a4b77
pve-qemu-qoup/debian/patches/pve/0028-PVE-Backup-add-vma-backup-format-code.patch
Fiona Ebner db5d2a4b77 squash related patches 
where there is no good reason to keep them separate. It's a pain
during rebase if there are multiple patches changing the same code
over and over again. This was especially bad for the backup-related
patches. If the history of patches really is needed, it can be
extracted via git. Additionally, compilation with partial application
of patches was broken since a long time, because one of the master key
changes became part of an earlier patch during a past rebase.

If only the same files were changed by a subsequent patch and the
changes felt to belong together (obvious for later bug fixes, but also
done for features e.g. adding master key support for PBS), the patches
were squashed together.

The PBS namespace support patch was split into the individual parts
it changes, i.e. PBS block driver, pbs-restore binary and QMP backup
infrastructure, and squashed into the respective patches.

No code change is intended, git diff in the submodule should not show
any difference between applying all patches before this commit and
applying all patches after this commit.

The query-proxmox-support QMP function has been left as part of the
"PVE-Backup: Proxmox backup patches for QEMU" patch, because it's
currently only used there. If it ever is used elsewhere too, it can
be split out from there.

The recent alloc-track and BQL-related savevm-async changes have been
left separate for now, because it's not 100% clear they are the best
approach yet. This depends on what upstream decides about the BQL
stuff and whether and what kind of issues with the changes pop up.

The qemu-img dd snapshot patch has been re-ordered to after the other
qemu-img dd patches.

Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
2023-05-22 15:09:14 +02:00

2776 lines
78 KiB
Diff
Raw Blame History

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Dietmar Maurer <dietmar@proxmox.com>
Date: Mon, 6 Apr 2020 12:16:57 +0200
Subject: [PATCH] PVE-Backup: add vma backup format code
Notes about partial restoring: skipping a certain drive is done via a
map line of the form skip=drive-scsi0. Since in PVE, most archives are
compressed and piped to vma for restore, it's not easily possible to
skip reads.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
[FE: improvements during create
allow partial restore]
Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
---
block/meson.build | 2 +
meson.build | 5 +
vma-reader.c | 867 +++++++++++++++++++++++++++++++++++++++++++++
vma-writer.c | 793 +++++++++++++++++++++++++++++++++++++++++
vma.c | 878 ++++++++++++++++++++++++++++++++++++++++++++++
vma.h | 150 ++++++++
6 files changed, 2695 insertions(+)
create mode 100644 vma-reader.c
create mode 100644 vma-writer.c
create mode 100644 vma.c
create mode 100644 vma.h
diff --git a/block/meson.build b/block/meson.build
index 253fe49fa2..744b698a82 100644
--- a/block/meson.build
+++ b/block/meson.build
@@ -47,6 +47,8 @@ block_ss.add(files(
'zeroinit.c',
), zstd, zlib, gnutls)
+block_ss.add(files('../vma-writer.c'), libuuid)
+
softmmu_ss.add(when: 'CONFIG_TCG', if_true: files('blkreplay.c'))
softmmu_ss.add(files('block-ram-registrar.c'))
diff --git a/meson.build b/meson.build
index d964e741e7..603cdb97bb 100644
--- a/meson.build
+++ b/meson.build
@@ -1527,6 +1527,8 @@ keyutils = dependency('libkeyutils', required: false,
has_gettid = cc.has_function('gettid')
+libuuid = cc.find_library('uuid', required: true)
+
# libselinux
selinux = dependency('libselinux',
required: get_option('selinux'),
@@ -3646,6 +3648,9 @@ if have_tools
dependencies: [blockdev, qemuutil, gnutls, selinux],
install: true)
+ vma = executable('vma', files('vma.c', 'vma-reader.c') + genh,
+ dependencies: [authz, block, crypto, io, qom], install: true)
+
subdir('storage-daemon')
subdir('contrib/rdmacm-mux')
subdir('contrib/elf2dmp')
diff --git a/vma-reader.c b/vma-reader.c
new file mode 100644
index 0000000000..81a891c6b1
--- /dev/null
+++ b/vma-reader.c
@@ -0,0 +1,867 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+ * Copyright (C) 2012 Proxmox Server Solutions
+ *
+ * Authors:
+ * Dietmar Maurer (dietmar@proxmox.com)
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include <uuid/uuid.h>
+
+#include "qemu/timer.h"
+#include "qemu/ratelimit.h"
+#include "vma.h"
+#include "block/block.h"
+#include "sysemu/block-backend.h"
+
+static unsigned char zero_vma_block[VMA_BLOCK_SIZE];
+
+typedef struct VmaRestoreState {
+ BlockBackend *target;
+ bool write_zeroes;
+ unsigned long *bitmap;
+ int bitmap_size;
+ bool skip;
+} VmaRestoreState;
+
+struct VmaReader {
+ int fd;
+ GChecksum *md5csum;
+ GHashTable *blob_hash;
+ unsigned char *head_data;
+ VmaDeviceInfo devinfo[256];
+ VmaRestoreState rstate[256];
+ GList *cdata_list;
+ guint8 vmstate_stream;
+ uint32_t vmstate_clusters;
+ /* to show restore percentage if run with -v */
+ time_t start_time;
+ int64_t cluster_count;
+ int64_t clusters_read;
+ int64_t zero_cluster_data;
+ int64_t partial_zero_cluster_data;
+ int clusters_read_per;
+};
+
+static guint
+g_int32_hash(gconstpointer v)
+{
+ return *(const uint32_t *)v;
+}
+
+static gboolean
+g_int32_equal(gconstpointer v1, gconstpointer v2)
+{
+ return *((const uint32_t *)v1) == *((const uint32_t *)v2);
+}
+
+static int vma_reader_get_bitmap(VmaRestoreState *rstate, int64_t cluster_num)
+{
+ assert(rstate);
+ assert(rstate->bitmap);
+
+ unsigned long val, idx, bit;
+
+ idx = cluster_num / BITS_PER_LONG;
+
+ assert(rstate->bitmap_size > idx);
+
+ bit = cluster_num % BITS_PER_LONG;
+ val = rstate->bitmap[idx];
+
+ return !!(val & (1UL << bit));
+}
+
+static void vma_reader_set_bitmap(VmaRestoreState *rstate, int64_t cluster_num,
+ int dirty)
+{
+ assert(rstate);
+ assert(rstate->bitmap);
+
+ unsigned long val, idx, bit;
+
+ idx = cluster_num / BITS_PER_LONG;
+
+ assert(rstate->bitmap_size > idx);
+
+ bit = cluster_num % BITS_PER_LONG;
+ val = rstate->bitmap[idx];
+ if (dirty) {
+ if (!(val & (1UL << bit))) {
+ val |= 1UL << bit;
+ }
+ } else {
+ if (val & (1UL << bit)) {
+ val &= ~(1UL << bit);
+ }
+ }
+ rstate->bitmap[idx] = val;
+}
+
+typedef struct VmaBlob {
+ uint32_t start;
+ uint32_t len;
+ void *data;
+} VmaBlob;
+
+static const VmaBlob *get_header_blob(VmaReader *vmar, uint32_t pos)
+{
+ assert(vmar);
+ assert(vmar->blob_hash);
+
+ return g_hash_table_lookup(vmar->blob_hash, &pos);
+}
+
+static const char *get_header_str(VmaReader *vmar, uint32_t pos)
+{
+ const VmaBlob *blob = get_header_blob(vmar, pos);
+ if (!blob) {
+ return NULL;
+ }
+ const char *res = (char *)blob->data;
+ if (res[blob->len-1] != '\0') {
+ return NULL;
+ }
+ return res;
+}
+
+static ssize_t
+safe_read(int fd, unsigned char *buf, size_t count)
+{
+ ssize_t n;
+
+ do {
+ n = read(fd, buf, count);
+ } while (n < 0 && errno == EINTR);
+
+ return n;
+}
+
+static ssize_t
+full_read(int fd, unsigned char *buf, size_t len)
+{
+ ssize_t n;
+ size_t total;
+
+ total = 0;
+
+ while (len > 0) {
+ n = safe_read(fd, buf, len);
+
+ if (n == 0) {
+ return total;
+ }
+
+ if (n <= 0) {
+ break;
+ }
+
+ buf += n;
+ total += n;
+ len -= n;
+ }
+
+ if (len) {
+ return -1;
+ }
+
+ return total;
+}
+
+void vma_reader_destroy(VmaReader *vmar)
+{
+ assert(vmar);
+
+ if (vmar->fd >= 0) {
+ close(vmar->fd);
+ }
+
+ if (vmar->cdata_list) {
+ g_list_free(vmar->cdata_list);
+ }
+
+ int i;
+ for (i = 1; i < 256; i++) {
+ if (vmar->rstate[i].bitmap) {
+ g_free(vmar->rstate[i].bitmap);
+ }
+ if (vmar->rstate[i].target) {
+ blk_unref(vmar->rstate[i].target);
+ }
+ }
+
+ if (vmar->md5csum) {
+ g_checksum_free(vmar->md5csum);
+ }
+
+ if (vmar->blob_hash) {
+ g_hash_table_destroy(vmar->blob_hash);
+ }
+
+ if (vmar->head_data) {
+ g_free(vmar->head_data);
+ }
+
+ g_free(vmar);
+
+};
+
+static int vma_reader_read_head(VmaReader *vmar, Error **errp)
+{
+ assert(vmar);
+ assert(errp);
+ assert(*errp == NULL);
+
+ unsigned char md5sum[16];
+ int i;
+ int ret = 0;
+
+ vmar->head_data = g_malloc(sizeof(VmaHeader));
+
+ if (full_read(vmar->fd, vmar->head_data, sizeof(VmaHeader)) !=
+ sizeof(VmaHeader)) {
+ error_setg(errp, "can't read vma header - %s",
+ errno ? g_strerror(errno) : "got EOF");
+ return -1;
+ }
+
+ VmaHeader *h = (VmaHeader *)vmar->head_data;
+
+ if (h->magic != VMA_MAGIC) {
+ error_setg(errp, "not a vma file - wrong magic number");
+ return -1;
+ }
+
+ uint32_t header_size = GUINT32_FROM_BE(h->header_size);
+ int need = header_size - sizeof(VmaHeader);
+ if (need <= 0) {
+ error_setg(errp, "wrong vma header size %d", header_size);
+ return -1;
+ }
+
+ vmar->head_data = g_realloc(vmar->head_data, header_size);
+ h = (VmaHeader *)vmar->head_data;
+
+ if (full_read(vmar->fd, vmar->head_data + sizeof(VmaHeader), need) !=
+ need) {
+ error_setg(errp, "can't read vma header data - %s",
+ errno ? g_strerror(errno) : "got EOF");
+ return -1;
+ }
+
+ memcpy(md5sum, h->md5sum, 16);
+ memset(h->md5sum, 0, 16);
+
+ g_checksum_reset(vmar->md5csum);
+ g_checksum_update(vmar->md5csum, vmar->head_data, header_size);
+ gsize csize = 16;
+ g_checksum_get_digest(vmar->md5csum, (guint8 *)(h->md5sum), &csize);
+
+ if (memcmp(md5sum, h->md5sum, 16) != 0) {
+ error_setg(errp, "wrong vma header chechsum");
+ return -1;
+ }
+
+ /* we can modify header data after checksum verify */
+ h->header_size = header_size;
+
+ h->version = GUINT32_FROM_BE(h->version);
+ if (h->version != 1) {
+ error_setg(errp, "wrong vma version %d", h->version);
+ return -1;
+ }
+
+ h->ctime = GUINT64_FROM_BE(h->ctime);
+ h->blob_buffer_offset = GUINT32_FROM_BE(h->blob_buffer_offset);
+ h->blob_buffer_size = GUINT32_FROM_BE(h->blob_buffer_size);
+
+ uint32_t bstart = h->blob_buffer_offset + 1;
+ uint32_t bend = h->blob_buffer_offset + h->blob_buffer_size;
+
+ if (bstart <= sizeof(VmaHeader)) {
+ error_setg(errp, "wrong vma blob buffer offset %d",
+ h->blob_buffer_offset);
+ return -1;
+ }
+
+ if (bend > header_size) {
+ error_setg(errp, "wrong vma blob buffer size %d/%d",
+ h->blob_buffer_offset, h->blob_buffer_size);
+ return -1;
+ }
+
+ while ((bstart + 2) <= bend) {
+ uint32_t size = vmar->head_data[bstart] +
+ (vmar->head_data[bstart+1] << 8);
+ if ((bstart + size + 2) <= bend) {
+ VmaBlob *blob = g_new0(VmaBlob, 1);
+ blob->start = bstart - h->blob_buffer_offset;
+ blob->len = size;
+ blob->data = vmar->head_data + bstart + 2;
+ g_hash_table_insert(vmar->blob_hash, &blob->start, blob);
+ }
+ bstart += size + 2;
+ }
+
+
+ int count = 0;
+ for (i = 1; i < 256; i++) {
+ VmaDeviceInfoHeader *dih = &h->dev_info[i];
+ uint32_t devname_ptr = GUINT32_FROM_BE(dih->devname_ptr);
+ uint64_t size = GUINT64_FROM_BE(dih->size);
+ const char *devname = get_header_str(vmar, devname_ptr);
+
+ if (size && devname) {
+ count++;
+ vmar->devinfo[i].size = size;
+ vmar->devinfo[i].devname = devname;
+
+ if (strcmp(devname, "vmstate") == 0) {
+ vmar->vmstate_stream = i;
+ }
+ }
+ }
+
+ for (i = 0; i < VMA_MAX_CONFIGS; i++) {
+ uint32_t name_ptr = GUINT32_FROM_BE(h->config_names[i]);
+ uint32_t data_ptr = GUINT32_FROM_BE(h->config_data[i]);
+
+ if (!(name_ptr && data_ptr)) {
+ continue;
+ }
+ const char *name = get_header_str(vmar, name_ptr);
+ const VmaBlob *blob = get_header_blob(vmar, data_ptr);
+
+ if (!(name && blob)) {
+ error_setg(errp, "vma contains invalid data pointers");
+ return -1;
+ }
+
+ VmaConfigData *cdata = g_new0(VmaConfigData, 1);
+ cdata->name = name;
+ cdata->data = blob->data;
+ cdata->len = blob->len;
+
+ vmar->cdata_list = g_list_append(vmar->cdata_list, cdata);
+ }
+
+ return ret;
+};
+
+VmaReader *vma_reader_create(const char *filename, Error **errp)
+{
+ assert(filename);
+ assert(errp);
+
+ VmaReader *vmar = g_new0(VmaReader, 1);
+
+ if (strcmp(filename, "-") == 0) {
+ vmar->fd = dup(0);
+ } else {
+ vmar->fd = open(filename, O_RDONLY);
+ }
+
+ if (vmar->fd < 0) {
+ error_setg(errp, "can't open file %s - %s\n", filename,
+ g_strerror(errno));
+ goto err;
+ }
+
+ vmar->md5csum = g_checksum_new(G_CHECKSUM_MD5);
+ if (!vmar->md5csum) {
+ error_setg(errp, "can't allocate cmsum\n");
+ goto err;
+ }
+
+ vmar->blob_hash = g_hash_table_new_full(g_int32_hash, g_int32_equal,
+ NULL, g_free);
+
+ if (vma_reader_read_head(vmar, errp) < 0) {
+ goto err;
+ }
+
+ return vmar;
+
+err:
+ if (vmar) {
+ vma_reader_destroy(vmar);
+ }
+
+ return NULL;
+}
+
+VmaHeader *vma_reader_get_header(VmaReader *vmar)
+{
+ assert(vmar);
+ assert(vmar->head_data);
+
+ return (VmaHeader *)(vmar->head_data);
+}
+
+GList *vma_reader_get_config_data(VmaReader *vmar)
+{
+ assert(vmar);
+ assert(vmar->head_data);
+
+ return vmar->cdata_list;
+}
+
+VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id)
+{
+ assert(vmar);
+ assert(dev_id);
+
+ if (vmar->devinfo[dev_id].size && vmar->devinfo[dev_id].devname) {
+ return &vmar->devinfo[dev_id];
+ }
+
+ return NULL;
+}
+
+static void allocate_rstate(VmaReader *vmar, guint8 dev_id,
+ BlockBackend *target, bool write_zeroes, bool skip)
+{
+ assert(vmar);
+ assert(dev_id);
+
+ vmar->rstate[dev_id].target = target;
+ vmar->rstate[dev_id].write_zeroes = write_zeroes;
+ vmar->rstate[dev_id].skip = skip;
+
+ int64_t size = vmar->devinfo[dev_id].size;
+
+ int64_t bitmap_size = (size/BDRV_SECTOR_SIZE) +
+ (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG - 1;
+ bitmap_size /= (VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE) * BITS_PER_LONG;
+
+ vmar->rstate[dev_id].bitmap_size = bitmap_size;
+ vmar->rstate[dev_id].bitmap = g_new0(unsigned long, bitmap_size);
+
+ vmar->cluster_count += size/VMA_CLUSTER_SIZE;
+}
+
+int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id, BlockBackend *target,
+ bool write_zeroes, bool skip, Error **errp)
+{
+ assert(vmar);
+ assert(target != NULL || skip);
+ assert(dev_id);
+ assert(vmar->rstate[dev_id].target == NULL && !vmar->rstate[dev_id].skip);
+
+ if (target != NULL) {
+ int64_t size = blk_getlength(target);
+ int64_t size_diff = size - vmar->devinfo[dev_id].size;
+
+ /* storage types can have different size restrictions, so it
+ * is not always possible to create an image with exact size.
+ * So we tolerate a size difference up to 4MB.
+ */
+ if ((size_diff < 0) || (size_diff > 4*1024*1024)) {
+ error_setg(errp, "vma_reader_register_bs for stream %s failed - "
+ "unexpected size %zd != %zd", vmar->devinfo[dev_id].devname,
+ size, vmar->devinfo[dev_id].size);
+ return -1;
+ }
+ }
+
+ allocate_rstate(vmar, dev_id, target, write_zeroes, skip);
+
+ return 0;
+}
+
+static ssize_t safe_write(int fd, void *buf, size_t count)
+{
+ ssize_t n;
+
+ do {
+ n = write(fd, buf, count);
+ } while (n < 0 && errno == EINTR);
+
+ return n;
+}
+
+static size_t full_write(int fd, void *buf, size_t len)
+{
+ ssize_t n;
+ size_t total;
+
+ total = 0;
+
+ while (len > 0) {
+ n = safe_write(fd, buf, len);
+ if (n < 0) {
+ return n;
+ }
+ buf += n;
+ total += n;
+ len -= n;
+ }
+
+ if (len) {
+ /* incomplete write ? */
+ return -1;
+ }
+
+ return total;
+}
+
+static int restore_write_data(VmaReader *vmar, guint8 dev_id,
+ BlockBackend *target, int vmstate_fd,
+ unsigned char *buf, int64_t sector_num,
+ int nb_sectors, Error **errp)
+{
+ assert(vmar);
+
+ if (dev_id == vmar->vmstate_stream) {
+ if (vmstate_fd >= 0) {
+ int len = nb_sectors * BDRV_SECTOR_SIZE;
+ int res = full_write(vmstate_fd, buf, len);
+ if (res < 0) {
+ error_setg(errp, "write vmstate failed %d", res);
+ return -1;
+ }
+ }
+ } else {
+ int res = blk_pwrite(target, sector_num * BDRV_SECTOR_SIZE, nb_sectors * BDRV_SECTOR_SIZE, buf, 0);
+ if (res < 0) {
+ error_setg(errp, "blk_pwrite to %s failed (%d)",
+ bdrv_get_device_name(blk_bs(target)), res);
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int restore_extent(VmaReader *vmar, unsigned char *buf,
+ int extent_size, int vmstate_fd,
+ bool verbose, bool verify, Error **errp)
+{
+ assert(vmar);
+ assert(buf);
+
+ VmaExtentHeader *ehead = (VmaExtentHeader *)buf;
+ int start = VMA_EXTENT_HEADER_SIZE;
+ int i;
+
+ for (i = 0; i < VMA_BLOCKS_PER_EXTENT; i++) {
+ uint64_t block_info = GUINT64_FROM_BE(ehead->blockinfo[i]);
+ uint64_t cluster_num = block_info & 0xffffffff;
+ uint8_t dev_id = (block_info >> 32) & 0xff;
+ uint16_t mask = block_info >> (32+16);
+ int64_t max_sector;
+
+ if (!dev_id) {
+ continue;
+ }
+
+ VmaRestoreState *rstate = &vmar->rstate[dev_id];
+ BlockBackend *target = NULL;
+
+ bool skip = rstate->skip;
+
+ if (dev_id != vmar->vmstate_stream) {
+ target = rstate->target;
+ if (!verify && !target && !skip) {
+ error_setg(errp, "got wrong dev id %d", dev_id);
+ return -1;
+ }
+
+ if (!skip) {
+ if (vma_reader_get_bitmap(rstate, cluster_num)) {
+ error_setg(errp, "found duplicated cluster %zd for stream %s",
+ cluster_num, vmar->devinfo[dev_id].devname);
+ return -1;
+ }
+ vma_reader_set_bitmap(rstate, cluster_num, 1);
+ }
+
+ max_sector = vmar->devinfo[dev_id].size/BDRV_SECTOR_SIZE;
+ } else {
+ max_sector = G_MAXINT64;
+ if (cluster_num != vmar->vmstate_clusters) {
+ error_setg(errp, "found out of order vmstate data");
+ return -1;
+ }
+ vmar->vmstate_clusters++;
+ }
+
+ vmar->clusters_read++;
+
+ if (verbose) {
+ time_t duration = time(NULL) - vmar->start_time;
+ int percent = (vmar->clusters_read*100)/vmar->cluster_count;
+ if (percent != vmar->clusters_read_per) {
+ printf("progress %d%% (read %zd bytes, duration %zd sec)\n",
+ percent, vmar->clusters_read*VMA_CLUSTER_SIZE,
+ duration);
+ fflush(stdout);
+ vmar->clusters_read_per = percent;
+ }
+ }
+
+ /* try to write whole clusters to speedup restore */
+ if (mask == 0xffff) {
+ if ((start + VMA_CLUSTER_SIZE) > extent_size) {
+ error_setg(errp, "short vma extent - too many blocks");
+ return -1;
+ }
+ int64_t sector_num = (cluster_num * VMA_CLUSTER_SIZE) /
+ BDRV_SECTOR_SIZE;
+ int64_t end_sector = sector_num +
+ VMA_CLUSTER_SIZE/BDRV_SECTOR_SIZE;
+
+ if (end_sector > max_sector) {
+ end_sector = max_sector;
+ }
+
+ if (end_sector <= sector_num) {
+ error_setg(errp, "got wrong block address - write beyond end");
+ return -1;
+ }
+
+ if (!verify && !skip) {
+ int nb_sectors = end_sector - sector_num;
+ if (restore_write_data(vmar, dev_id, target, vmstate_fd,
+ buf + start, sector_num, nb_sectors,
+ errp) < 0) {
+ return -1;
+ }
+ }
+
+ start += VMA_CLUSTER_SIZE;
+ } else {
+ int j;
+ int bit = 1;
+
+ for (j = 0; j < 16; j++) {
+ int64_t sector_num = (cluster_num*VMA_CLUSTER_SIZE +
+ j*VMA_BLOCK_SIZE)/BDRV_SECTOR_SIZE;
+
+ int64_t end_sector = sector_num +
+ VMA_BLOCK_SIZE/BDRV_SECTOR_SIZE;
+ if (end_sector > max_sector) {
+ end_sector = max_sector;
+ }
+
+ if (mask & bit) {
+ if ((start + VMA_BLOCK_SIZE) > extent_size) {
+ error_setg(errp, "short vma extent - too many blocks");
+ return -1;
+ }
+
+ if (end_sector <= sector_num) {
+ error_setg(errp, "got wrong block address - "
+ "write beyond end");
+ return -1;
+ }
+
+ if (!verify && !skip) {
+ int nb_sectors = end_sector - sector_num;
+ if (restore_write_data(vmar, dev_id, target, vmstate_fd,
+ buf + start, sector_num,
+ nb_sectors, errp) < 0) {
+ return -1;
+ }
+ }
+
+ start += VMA_BLOCK_SIZE;
+
+ } else {
+
+
+ if (end_sector > sector_num) {
+ /* Todo: use bdrv_co_write_zeroes (but that need to
+ * be run inside coroutine?)
+ */
+ int nb_sectors = end_sector - sector_num;
+ int zero_size = BDRV_SECTOR_SIZE*nb_sectors;
+ vmar->zero_cluster_data += zero_size;
+ if (mask != 0) {
+ vmar->partial_zero_cluster_data += zero_size;
+ }
+
+ if (rstate->write_zeroes && !verify && !skip) {
+ if (restore_write_data(vmar, dev_id, target, vmstate_fd,
+ zero_vma_block, sector_num,
+ nb_sectors, errp) < 0) {
+ return -1;
+ }
+ }
+ }
+ }
+
+ bit = bit << 1;
+ }
+ }
+ }
+
+ if (start != extent_size) {
+ error_setg(errp, "vma extent error - missing blocks");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int vma_reader_restore_full(VmaReader *vmar, int vmstate_fd,
+ bool verbose, bool verify,
+ Error **errp)
+{
+ assert(vmar);
+ assert(vmar->head_data);
+
+ int ret = 0;
+ unsigned char buf[VMA_MAX_EXTENT_SIZE];
+ int buf_pos = 0;
+ unsigned char md5sum[16];
+ VmaHeader *h = (VmaHeader *)vmar->head_data;
+
+ vmar->start_time = time(NULL);
+
+ while (1) {
+ int bytes = full_read(vmar->fd, buf + buf_pos, sizeof(buf) - buf_pos);
+ if (bytes < 0) {
+ error_setg(errp, "read failed - %s", g_strerror(errno));
+ return -1;
+ }
+
+ buf_pos += bytes;
+
+ if (!buf_pos) {
+ break; /* EOF */
+ }
+
+ if (buf_pos < VMA_EXTENT_HEADER_SIZE) {
+ error_setg(errp, "read short extent (%d bytes)", buf_pos);
+ return -1;
+ }
+
+ VmaExtentHeader *ehead = (VmaExtentHeader *)buf;
+
+ /* extract md5sum */
+ memcpy(md5sum, ehead->md5sum, sizeof(ehead->md5sum));
+ memset(ehead->md5sum, 0, sizeof(ehead->md5sum));
+
+ g_checksum_reset(vmar->md5csum);
+ g_checksum_update(vmar->md5csum, buf, VMA_EXTENT_HEADER_SIZE);
+ gsize csize = 16;
+ g_checksum_get_digest(vmar->md5csum, ehead->md5sum, &csize);
+
+ if (memcmp(md5sum, ehead->md5sum, 16) != 0) {
+ error_setg(errp, "wrong vma extent header chechsum");
+ return -1;
+ }
+
+ if (memcmp(h->uuid, ehead->uuid, sizeof(ehead->uuid)) != 0) {
+ error_setg(errp, "wrong vma extent uuid");
+ return -1;
+ }
+
+ if (ehead->magic != VMA_EXTENT_MAGIC || ehead->reserved1 != 0) {
+ error_setg(errp, "wrong vma extent header magic");
+ return -1;
+ }
+
+ int block_count = GUINT16_FROM_BE(ehead->block_count);
+ int extent_size = VMA_EXTENT_HEADER_SIZE + block_count*VMA_BLOCK_SIZE;
+
+ if (buf_pos < extent_size) {
+ error_setg(errp, "short vma extent (%d < %d)", buf_pos,
+ extent_size);
+ return -1;
+ }
+
+ if (restore_extent(vmar, buf, extent_size, vmstate_fd, verbose,
+ verify, errp) < 0) {
+ return -1;
+ }
+
+ if (buf_pos > extent_size) {
+ memmove(buf, buf + extent_size, buf_pos - extent_size);
+ buf_pos = buf_pos - extent_size;
+ } else {
+ buf_pos = 0;
+ }
+ }
+
+ bdrv_drain_all();
+
+ int i;
+ for (i = 1; i < 256; i++) {
+ VmaRestoreState *rstate = &vmar->rstate[i];
+ if (!rstate->target) {
+ continue;
+ }
+
+ if (blk_flush(rstate->target) < 0) {
+ error_setg(errp, "vma blk_flush %s failed",
+ vmar->devinfo[i].devname);
+ return -1;
+ }
+
+ if (vmar->devinfo[i].size &&
+ (strcmp(vmar->devinfo[i].devname, "vmstate") != 0)) {
+ assert(rstate->bitmap);
+
+ int64_t cluster_num, end;
+
+ end = (vmar->devinfo[i].size + VMA_CLUSTER_SIZE - 1) /
+ VMA_CLUSTER_SIZE;
+
+ for (cluster_num = 0; cluster_num < end; cluster_num++) {
+ if (!vma_reader_get_bitmap(rstate, cluster_num)) {
+ error_setg(errp, "detected missing cluster %zd "
+ "for stream %s", cluster_num,
+ vmar->devinfo[i].devname);
+ return -1;
+ }
+ }
+ }
+ }
+
+ if (verbose) {
+ if (vmar->clusters_read) {
+ printf("total bytes read %zd, sparse bytes %zd (%.3g%%)\n",
+ vmar->clusters_read*VMA_CLUSTER_SIZE,
+ vmar->zero_cluster_data,
+ (double)(100.0*vmar->zero_cluster_data)/
+ (vmar->clusters_read*VMA_CLUSTER_SIZE));
+
+ int64_t datasize = vmar->clusters_read*VMA_CLUSTER_SIZE-vmar->zero_cluster_data;
+ if (datasize) { // this does not make sense for empty files
+ printf("space reduction due to 4K zero blocks %.3g%%\n",
+ (double)(100.0*vmar->partial_zero_cluster_data) / datasize);
+ }
+ } else {
+ printf("vma archive contains no image data\n");
+ }
+ }
+ return ret;
+}
+
+int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
+ Error **errp)
+{
+ return vma_reader_restore_full(vmar, vmstate_fd, verbose, false, errp);
+}
+
+int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp)
+{
+ guint8 dev_id;
+
+ for (dev_id = 1; dev_id < 255; dev_id++) {
+ if (vma_reader_get_device_info(vmar, dev_id)) {
+ allocate_rstate(vmar, dev_id, NULL, false, false);
+ }
+ }
+
+ return vma_reader_restore_full(vmar, -1, verbose, true, errp);
+}
+
diff --git a/vma-writer.c b/vma-writer.c
new file mode 100644
index 0000000000..ac7da237d0
--- /dev/null
+++ b/vma-writer.c
@@ -0,0 +1,793 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+ * Copyright (C) 2012 Proxmox Server Solutions
+ *
+ * Authors:
+ * Dietmar Maurer (dietmar@proxmox.com)
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include <uuid/uuid.h>
+
+#include "vma.h"
+#include "block/block.h"
+#include "monitor/monitor.h"
+#include "qemu/main-loop.h"
+#include "qemu/coroutine.h"
+#include "qemu/cutils.h"
+#include "qemu/memalign.h"
+
+#define DEBUG_VMA 0
+
+#define DPRINTF(fmt, ...)\
+ do { if (DEBUG_VMA) { printf("vma: " fmt, ## __VA_ARGS__); } } while (0)
+
+#define WRITE_BUFFERS 5
+#define HEADER_CLUSTERS 8
+#define HEADERBUF_SIZE (VMA_CLUSTER_SIZE*HEADER_CLUSTERS)
+
+struct VmaWriter {
+ int fd;
+ FILE *cmd;
+ int status;
+ char errmsg[8192];
+ uuid_t uuid;
+ bool header_written;
+ bool closed;
+
+ /* we always write extents */
+ unsigned char *outbuf;
+ int outbuf_pos; /* in bytes */
+ int outbuf_count; /* in VMA_BLOCKS */
+ uint64_t outbuf_block_info[VMA_BLOCKS_PER_EXTENT];
+
+ unsigned char *headerbuf;
+
+ GChecksum *md5csum;
+ CoMutex flush_lock;
+ Coroutine *co_writer;
+
+ /* drive informations */
+ VmaStreamInfo stream_info[256];
+ guint stream_count;
+
+ guint8 vmstate_stream;
+ uint32_t vmstate_clusters;
+
+ /* header blob table */
+ char *header_blob_table;
+ uint32_t header_blob_table_size;
+ uint32_t header_blob_table_pos;
+
+ /* store for config blobs */
+ uint32_t config_names[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
+ uint32_t config_data[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
+ uint32_t config_count;
+};
+
+void vma_writer_set_error(VmaWriter *vmaw, const char *fmt, ...)
+{
+ va_list ap;
+
+ if (vmaw->status < 0) {
+ return;
+ }
+
+ vmaw->status = -1;
+
+ va_start(ap, fmt);
+ g_vsnprintf(vmaw->errmsg, sizeof(vmaw->errmsg), fmt, ap);
+ va_end(ap);
+
+ DPRINTF("vma_writer_set_error: %s\n", vmaw->errmsg);
+}
+
+static uint32_t allocate_header_blob(VmaWriter *vmaw, const char *data,
+ size_t len)
+{
+ if (len > 65535) {
+ return 0;
+ }
+
+ if (!vmaw->header_blob_table ||
+ (vmaw->header_blob_table_size <
+ (vmaw->header_blob_table_pos + len + 2))) {
+ int newsize = vmaw->header_blob_table_size + ((len + 2 + 511)/512)*512;
+
+ vmaw->header_blob_table = g_realloc(vmaw->header_blob_table, newsize);
+ memset(vmaw->header_blob_table + vmaw->header_blob_table_size,
+ 0, newsize - vmaw->header_blob_table_size);
+ vmaw->header_blob_table_size = newsize;
+ }
+
+ uint32_t cpos = vmaw->header_blob_table_pos;
+ vmaw->header_blob_table[cpos] = len & 255;
+ vmaw->header_blob_table[cpos+1] = (len >> 8) & 255;
+ memcpy(vmaw->header_blob_table + cpos + 2, data, len);
+ vmaw->header_blob_table_pos += len + 2;
+ return cpos;
+}
+
+static uint32_t allocate_header_string(VmaWriter *vmaw, const char *str)
+{
+ assert(vmaw);
+
+ size_t len = strlen(str) + 1;
+
+ return allocate_header_blob(vmaw, str, len);
+}
+
+int vma_writer_add_config(VmaWriter *vmaw, const char *name, gpointer data,
+ gsize len)
+{
+ assert(vmaw);
+ assert(!vmaw->header_written);
+ assert(vmaw->config_count < VMA_MAX_CONFIGS);
+ assert(name);
+ assert(data);
+
+ gchar *basename = g_path_get_basename(name);
+ uint32_t name_ptr = allocate_header_string(vmaw, basename);
+ g_free(basename);
+
+ if (!name_ptr) {
+ return -1;
+ }
+
+ uint32_t data_ptr = allocate_header_blob(vmaw, data, len);
+ if (!data_ptr) {
+ return -1;
+ }
+
+ vmaw->config_names[vmaw->config_count] = name_ptr;
+ vmaw->config_data[vmaw->config_count] = data_ptr;
+
+ vmaw->config_count++;
+
+ return 0;
+}
+
+int vma_writer_register_stream(VmaWriter *vmaw, const char *devname,
+ size_t size)
+{
+ assert(vmaw);
+ assert(devname);
+ assert(!vmaw->status);
+
+ if (vmaw->header_written) {
+ vma_writer_set_error(vmaw, "vma_writer_register_stream: header "
+ "already written");
+ return -1;
+ }
+
+ guint n = vmaw->stream_count + 1;
+
+ /* we can have dev_ids form 1 to 255 (0 reserved)
+ * 255(-1) reseverd for safety
+ */
+ if (n > 254) {
+ vma_writer_set_error(vmaw, "vma_writer_register_stream: "
+ "too many drives");
+ return -1;
+ }
+
+ if (size <= 0) {
+ vma_writer_set_error(vmaw, "vma_writer_register_stream: "
+ "got strange size %zd", size);
+ return -1;
+ }
+
+ DPRINTF("vma_writer_register_stream %s %zu %d\n", devname, size, n);
+
+ vmaw->stream_info[n].devname = g_strdup(devname);
+ vmaw->stream_info[n].size = size;
+
+ vmaw->stream_info[n].cluster_count = (size + VMA_CLUSTER_SIZE - 1) /
+ VMA_CLUSTER_SIZE;
+
+ vmaw->stream_count = n;
+
+ if (strcmp(devname, "vmstate") == 0) {
+ vmaw->vmstate_stream = n;
+ }
+
+ return n;
+}
+
+static void coroutine_fn yield_until_fd_writable(int fd)
+{
+ assert(qemu_in_coroutine());
+ AioContext *ctx = qemu_get_current_aio_context();
+ aio_set_fd_handler(ctx, fd, false, NULL, (IOHandler *)qemu_coroutine_enter,
+ NULL, NULL, qemu_coroutine_self());
+ qemu_coroutine_yield();
+ aio_set_fd_handler(ctx, fd, false, NULL, NULL, NULL, NULL, NULL);
+}
+
+static ssize_t coroutine_fn
+vma_queue_write(VmaWriter *vmaw, const void *buf, size_t bytes)
+{
+ DPRINTF("vma_queue_write enter %zd\n", bytes);
+
+ assert(vmaw);
+ assert(buf);
+ assert(bytes <= VMA_MAX_EXTENT_SIZE);
+
+ size_t done = 0;
+ ssize_t ret;
+
+ assert(vmaw->co_writer == NULL);
+
+ vmaw->co_writer = qemu_coroutine_self();
+
+ while (done < bytes) {
+ if (vmaw->status < 0) {
+ DPRINTF("vma_queue_write detected canceled backup\n");
+ done = -1;
+ break;
+ }
+ yield_until_fd_writable(vmaw->fd);
+ ret = write(vmaw->fd, buf + done, bytes - done);
+ if (ret > 0) {
+ done += ret;
+ DPRINTF("vma_queue_write written %zd %zd\n", done, ret);
+ } else if (ret < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ /* try again */
+ } else {
+ vma_writer_set_error(vmaw, "vma_queue_write: write error - %s",
+ g_strerror(errno));
+ done = -1; /* always return failure for partial writes */
+ break;
+ }
+ } else if (ret == 0) {
+ /* should not happen - simply try again */
+ }
+ }
+
+ vmaw->co_writer = NULL;
+
+ return (done == bytes) ? bytes : -1;
+}
+
+VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, Error **errp)
+{
+ const char *p;
+
+ assert(sizeof(VmaHeader) == (4096 + 8192));
+ assert(G_STRUCT_OFFSET(VmaHeader, config_names) == 2044);
+ assert(G_STRUCT_OFFSET(VmaHeader, config_data) == 3068);
+ assert(G_STRUCT_OFFSET(VmaHeader, dev_info) == 4096);
+ assert(sizeof(VmaExtentHeader) == 512);
+
+ VmaWriter *vmaw = g_new0(VmaWriter, 1);
+ vmaw->fd = -1;
+
+ vmaw->md5csum = g_checksum_new(G_CHECKSUM_MD5);
+ if (!vmaw->md5csum) {
+ error_setg(errp, "can't allocate cmsum\n");
+ goto err;
+ }
+
+ if (strstart(filename, "exec:", &p)) {
+ vmaw->cmd = popen(p, "w");
+ if (vmaw->cmd == NULL) {
+ error_setg(errp, "can't popen command '%s' - %s\n", p,
+ g_strerror(errno));
+ goto err;
+ }
+ vmaw->fd = fileno(vmaw->cmd);
+
+ /* try to use O_NONBLOCK */
+ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_NONBLOCK);
+
+ } else {
+ struct stat st;
+ int oflags;
+ const char *tmp_id_str;
+
+ if ((stat(filename, &st) == 0) && S_ISFIFO(st.st_mode)) {
+ oflags = O_NONBLOCK|O_WRONLY;
+ vmaw->fd = qemu_open(filename, oflags, errp);
+ } else if (strstart(filename, "/dev/fdset/", &tmp_id_str)) {
+ oflags = O_NONBLOCK|O_WRONLY;
+ vmaw->fd = qemu_open(filename, oflags, errp);
+ } else if (strstart(filename, "/dev/fdname/", &tmp_id_str)) {
+ vmaw->fd = monitor_get_fd(monitor_cur(), tmp_id_str, errp);
+ if (vmaw->fd < 0) {
+ goto err;
+ }
+ /* try to use O_NONBLOCK */
+ fcntl(vmaw->fd, F_SETFL, fcntl(vmaw->fd, F_GETFL)|O_NONBLOCK);
+ } else {
+ oflags = O_NONBLOCK|O_DIRECT|O_WRONLY|O_EXCL;
+ vmaw->fd = qemu_create(filename, oflags, 0644, errp);
+ }
+
+ if (vmaw->fd < 0) {
+ error_free(*errp);
+ *errp = NULL;
+ error_setg(errp, "can't open file %s - %s\n", filename,
+ g_strerror(errno));
+ goto err;
+ }
+ }
+
+ /* we use O_DIRECT, so we need to align IO buffers */
+
+ vmaw->outbuf = qemu_memalign(512, VMA_MAX_EXTENT_SIZE);
+ vmaw->headerbuf = qemu_memalign(512, HEADERBUF_SIZE);
+
+ vmaw->outbuf_count = 0;
+ vmaw->outbuf_pos = VMA_EXTENT_HEADER_SIZE;
+
+ vmaw->header_blob_table_pos = 1; /* start at pos 1 */
+
+ qemu_co_mutex_init(&vmaw->flush_lock);
+
+ uuid_copy(vmaw->uuid, uuid);
+
+ return vmaw;
+
+err:
+ if (vmaw) {
+ if (vmaw->cmd) {
+ pclose(vmaw->cmd);
+ } else if (vmaw->fd >= 0) {
+ close(vmaw->fd);
+ }
+
+ if (vmaw->md5csum) {
+ g_checksum_free(vmaw->md5csum);
+ }
+
+ g_free(vmaw);
+ }
+
+ return NULL;
+}
+
+static int coroutine_fn vma_write_header(VmaWriter *vmaw)
+{
+ assert(vmaw);
+ unsigned char *buf = vmaw->headerbuf;
+ VmaHeader *head = (VmaHeader *)buf;
+
+ int i;
+
+ DPRINTF("VMA WRITE HEADER\n");
+
+ if (vmaw->status < 0) {
+ return vmaw->status;
+ }
+
+ memset(buf, 0, HEADERBUF_SIZE);
+
+ head->magic = VMA_MAGIC;
+ head->version = GUINT32_TO_BE(1); /* v1 */
+ memcpy(head->uuid, vmaw->uuid, 16);
+
+ time_t ctime = time(NULL);
+ head->ctime = GUINT64_TO_BE(ctime);
+
+ for (i = 0; i < VMA_MAX_CONFIGS; i++) {
+ head->config_names[i] = GUINT32_TO_BE(vmaw->config_names[i]);
+ head->config_data[i] = GUINT32_TO_BE(vmaw->config_data[i]);
+ }
+
+ /* 32 bytes per device (12 used currently) = 8192 bytes max */
+ for (i = 1; i <= 254; i++) {
+ VmaStreamInfo *si = &vmaw->stream_info[i];
+ if (si->size) {
+ assert(si->devname);
+ uint32_t devname_ptr = allocate_header_string(vmaw, si->devname);
+ if (!devname_ptr) {
+ return -1;
+ }
+ head->dev_info[i].devname_ptr = GUINT32_TO_BE(devname_ptr);
+ head->dev_info[i].size = GUINT64_TO_BE(si->size);
+ }
+ }
+
+ uint32_t header_size = sizeof(VmaHeader) + vmaw->header_blob_table_size;
+ head->header_size = GUINT32_TO_BE(header_size);
+
+ if (header_size > HEADERBUF_SIZE) {
+ return -1; /* just to be sure */
+ }
+
+ uint32_t blob_buffer_offset = sizeof(VmaHeader);
+ memcpy(buf + blob_buffer_offset, vmaw->header_blob_table,
+ vmaw->header_blob_table_size);
+ head->blob_buffer_offset = GUINT32_TO_BE(blob_buffer_offset);
+ head->blob_buffer_size = GUINT32_TO_BE(vmaw->header_blob_table_pos);
+
+ g_checksum_reset(vmaw->md5csum);
+ g_checksum_update(vmaw->md5csum, (const guchar *)buf, header_size);
+ gsize csize = 16;
+ g_checksum_get_digest(vmaw->md5csum, (guint8 *)(head->md5sum), &csize);
+
+ return vma_queue_write(vmaw, buf, header_size);
+}
+
+static int coroutine_fn vma_writer_flush(VmaWriter *vmaw)
+{
+ assert(vmaw);
+
+ int ret;
+ int i;
+
+ if (vmaw->status < 0) {
+ return vmaw->status;
+ }
+
+ if (!vmaw->header_written) {
+ vmaw->header_written = true;
+ ret = vma_write_header(vmaw);
+ if (ret < 0) {
+ vma_writer_set_error(vmaw, "vma_writer_flush: write header failed");
+ return ret;
+ }
+ }
+
+ DPRINTF("VMA WRITE FLUSH %d %d\n", vmaw->outbuf_count, vmaw->outbuf_pos);
+
+
+ VmaExtentHeader *ehead = (VmaExtentHeader *)vmaw->outbuf;
+
+ ehead->magic = VMA_EXTENT_MAGIC;
+ ehead->reserved1 = 0;
+
+ for (i = 0; i < VMA_BLOCKS_PER_EXTENT; i++) {
+ ehead->blockinfo[i] = GUINT64_TO_BE(vmaw->outbuf_block_info[i]);
+ }
+
+ guint16 block_count = (vmaw->outbuf_pos - VMA_EXTENT_HEADER_SIZE) /
+ VMA_BLOCK_SIZE;
+
+ ehead->block_count = GUINT16_TO_BE(block_count);
+
+ memcpy(ehead->uuid, vmaw->uuid, sizeof(ehead->uuid));
+ memset(ehead->md5sum, 0, sizeof(ehead->md5sum));
+
+ g_checksum_reset(vmaw->md5csum);
+ g_checksum_update(vmaw->md5csum, vmaw->outbuf, VMA_EXTENT_HEADER_SIZE);
+ gsize csize = 16;
+ g_checksum_get_digest(vmaw->md5csum, ehead->md5sum, &csize);
+
+ int bytes = vmaw->outbuf_pos;
+ ret = vma_queue_write(vmaw, vmaw->outbuf, bytes);
+ if (ret != bytes) {
+ vma_writer_set_error(vmaw, "vma_writer_flush: failed write");
+ }
+
+ vmaw->outbuf_count = 0;
+ vmaw->outbuf_pos = VMA_EXTENT_HEADER_SIZE;
+
+ for (i = 0; i < VMA_BLOCKS_PER_EXTENT; i++) {
+ vmaw->outbuf_block_info[i] = 0;
+ }
+
+ return vmaw->status;
+}
+
+static int vma_count_open_streams(VmaWriter *vmaw)
+{
+ g_assert(vmaw != NULL);
+
+ int i;
+ int open_drives = 0;
+ for (i = 0; i <= 255; i++) {
+ if (vmaw->stream_info[i].size && !vmaw->stream_info[i].finished) {
+ open_drives++;
+ }
+ }
+
+ return open_drives;
+}
+
+
+/**
+ * You need to call this if the vma archive does not contain
+ * any data stream.
+ */
+int coroutine_fn
+vma_writer_flush_output(VmaWriter *vmaw)
+{
+ qemu_co_mutex_lock(&vmaw->flush_lock);
+ int ret = vma_writer_flush(vmaw);
+ qemu_co_mutex_unlock(&vmaw->flush_lock);
+ if (ret < 0) {
+ vma_writer_set_error(vmaw, "vma_writer_flush_header failed");
+ }
+ return ret;
+}
+
+/**
+ * all jobs should call this when there is no more data
+ * Returns: number of remaining stream (0 ==> finished)
+ */
+int coroutine_fn
+vma_writer_close_stream(VmaWriter *vmaw, uint8_t dev_id)
+{
+ g_assert(vmaw != NULL);
+
+ DPRINTF("vma_writer_set_status %d\n", dev_id);
+ if (!vmaw->stream_info[dev_id].size) {
+ vma_writer_set_error(vmaw, "vma_writer_close_stream: "
+ "no such stream %d", dev_id);
+ return -1;
+ }
+ if (vmaw->stream_info[dev_id].finished) {
+ vma_writer_set_error(vmaw, "vma_writer_close_stream: "
+ "stream already closed %d", dev_id);
+ return -1;
+ }
+
+ vmaw->stream_info[dev_id].finished = true;
+
+ int open_drives = vma_count_open_streams(vmaw);
+
+ if (open_drives <= 0) {
+ DPRINTF("vma_writer_set_status all drives completed\n");
+ vma_writer_flush_output(vmaw);
+ }
+
+ return open_drives;
+}
+
+int vma_writer_get_status(VmaWriter *vmaw, VmaStatus *status)
+{
+ int i;
+
+ g_assert(vmaw != NULL);
+
+ if (status) {
+ status->status = vmaw->status;
+ g_strlcpy(status->errmsg, vmaw->errmsg, sizeof(status->errmsg));
+ for (i = 0; i <= 255; i++) {
+ status->stream_info[i] = vmaw->stream_info[i];
+ }
+
+ uuid_unparse_lower(vmaw->uuid, status->uuid_str);
+ }
+
+ status->closed = vmaw->closed;
+
+ return vmaw->status;
+}
+
+static int vma_writer_get_buffer(VmaWriter *vmaw)
+{
+ int ret = 0;
+
+ qemu_co_mutex_lock(&vmaw->flush_lock);
+
+ /* wait until buffer is available */
+ while (vmaw->outbuf_count >= (VMA_BLOCKS_PER_EXTENT - 1)) {
+ ret = vma_writer_flush(vmaw);
+ if (ret < 0) {
+ vma_writer_set_error(vmaw, "vma_writer_get_buffer: flush failed");
+ break;
+ }
+ }
+
+ qemu_co_mutex_unlock(&vmaw->flush_lock);
+
+ return ret;
+}
+
+
+int64_t coroutine_fn
+vma_writer_write(VmaWriter *vmaw, uint8_t dev_id, int64_t cluster_num,
+ const unsigned char *buf, size_t *zero_bytes)
+{
+ g_assert(vmaw != NULL);
+ g_assert(zero_bytes != NULL);
+
+ *zero_bytes = 0;
+
+ if (vmaw->status < 0) {
+ return vmaw->status;
+ }
+
+ if (!dev_id || !vmaw->stream_info[dev_id].size) {
+ vma_writer_set_error(vmaw, "vma_writer_write: "
+ "no such stream %d", dev_id);
+ return -1;
+ }
+
+ if (vmaw->stream_info[dev_id].finished) {
+ vma_writer_set_error(vmaw, "vma_writer_write: "
+ "stream already closed %d", dev_id);
+ return -1;
+ }
+
+
+ if (cluster_num >= (((uint64_t)1)<<32)) {
+ vma_writer_set_error(vmaw, "vma_writer_write: "
+ "cluster number out of range");
+ return -1;
+ }
+
+ if (dev_id == vmaw->vmstate_stream) {
+ if (cluster_num != vmaw->vmstate_clusters) {
+ vma_writer_set_error(vmaw, "vma_writer_write: "
+ "non sequential vmstate write");
+ }
+ vmaw->vmstate_clusters++;
+ } else if (cluster_num >= vmaw->stream_info[dev_id].cluster_count) {
+ vma_writer_set_error(vmaw, "vma_writer_write: cluster number too big");
+ return -1;
+ }
+
+ /* wait until buffer is available */
+ if (vma_writer_get_buffer(vmaw) < 0) {
+ vma_writer_set_error(vmaw, "vma_writer_write: "
+ "vma_writer_get_buffer failed");
+ return -1;
+ }
+
+ DPRINTF("VMA WRITE %d %zd\n", dev_id, cluster_num);
+
+ uint64_t dev_size = vmaw->stream_info[dev_id].size;
+ uint16_t mask = 0;
+
+ if (buf) {
+ int i;
+ int bit = 1;
+ uint64_t byte_offset = cluster_num * VMA_CLUSTER_SIZE;
+ for (i = 0; i < 16; i++) {
+ const unsigned char *vmablock = buf + (i*VMA_BLOCK_SIZE);
+
+ // Note: If the source is not 64k-aligned, we might reach 4k blocks
+ // after the end of the device. Always mark these as zero in the
+ // mask, so the restore handles them correctly.
+ if (byte_offset < dev_size &&
+ !buffer_is_zero(vmablock, VMA_BLOCK_SIZE))
+ {
+ mask |= bit;
+ memcpy(vmaw->outbuf + vmaw->outbuf_pos, vmablock,
+ VMA_BLOCK_SIZE);
+
+ // prevent memory leakage on unaligned last block
+ if (byte_offset + VMA_BLOCK_SIZE > dev_size) {
+ uint64_t real_data_in_block = dev_size - byte_offset;
+ memset(vmaw->outbuf + vmaw->outbuf_pos + real_data_in_block,
+ 0, VMA_BLOCK_SIZE - real_data_in_block);
+ }
+
+ vmaw->outbuf_pos += VMA_BLOCK_SIZE;
+ } else {
+ DPRINTF("VMA WRITE %zd ZERO BLOCK %d\n", cluster_num, i);
+ vmaw->stream_info[dev_id].zero_bytes += VMA_BLOCK_SIZE;
+ *zero_bytes += VMA_BLOCK_SIZE;
+ }
+
+ byte_offset += VMA_BLOCK_SIZE;
+ bit = bit << 1;
+ }
+ } else {
+ DPRINTF("VMA WRITE %zd ZERO CLUSTER\n", cluster_num);
+ vmaw->stream_info[dev_id].zero_bytes += VMA_CLUSTER_SIZE;
+ *zero_bytes += VMA_CLUSTER_SIZE;
+ }
+
+ uint64_t block_info = ((uint64_t)mask) << (32+16);
+ block_info |= ((uint64_t)dev_id) << 32;
+ block_info |= (cluster_num & 0xffffffff);
+ vmaw->outbuf_block_info[vmaw->outbuf_count] = block_info;
+
+ DPRINTF("VMA WRITE MASK %zd %zx\n", cluster_num, block_info);
+
+ vmaw->outbuf_count++;
+
+ /** NOTE: We allways write whole clusters, but we correctly set
+ * transferred bytes. So transferred == size when when everything
+ * went OK.
+ */
+ size_t transferred = VMA_CLUSTER_SIZE;
+
+ if (dev_id != vmaw->vmstate_stream) {
+ uint64_t last = (cluster_num + 1) * VMA_CLUSTER_SIZE;
+ if (last > dev_size) {
+ uint64_t diff = last - dev_size;
+ if (diff >= VMA_CLUSTER_SIZE) {
+ vma_writer_set_error(vmaw, "vma_writer_write: "
+ "read after last cluster");
+ return -1;
+ }
+ transferred -= diff;
+ }
+ }
+
+ vmaw->stream_info[dev_id].transferred += transferred;
+
+ return transferred;
+}
+
+void vma_writer_error_propagate(VmaWriter *vmaw, Error **errp)
+{
+ if (vmaw->status < 0 && *errp == NULL) {
+ error_setg(errp, "%s", vmaw->errmsg);
+ }
+}
+
+int vma_writer_close(VmaWriter *vmaw, Error **errp)
+{
+ g_assert(vmaw != NULL);
+
+ int i;
+
+ qemu_co_mutex_lock(&vmaw->flush_lock); // wait for pending writes
+
+ assert(vmaw->co_writer == NULL);
+
+ if (vmaw->cmd) {
+ if (pclose(vmaw->cmd) < 0) {
+ vma_writer_set_error(vmaw, "vma_writer_close: "
+ "pclose failed - %s", g_strerror(errno));
+ }
+ } else {
+ if (close(vmaw->fd) < 0) {
+ vma_writer_set_error(vmaw, "vma_writer_close: "
+ "close failed - %s", g_strerror(errno));
+ }
+ }
+
+ for (i = 0; i <= 255; i++) {
+ VmaStreamInfo *si = &vmaw->stream_info[i];
+ if (si->size) {
+ if (!si->finished) {
+ vma_writer_set_error(vmaw, "vma_writer_close: "
+ "detected open stream '%s'", si->devname);
+ } else if ((si->transferred != si->size) &&
+ (i != vmaw->vmstate_stream)) {
+ vma_writer_set_error(vmaw, "vma_writer_close: "
+ "incomplete stream '%s' (%zd != %zd)",
+ si->devname, si->transferred, si->size);
+ }
+ }
+ }
+
+ for (i = 0; i <= 255; i++) {
+ vmaw->stream_info[i].finished = 1; /* mark as closed */
+ }
+
+ vmaw->closed = 1;
+
+ if (vmaw->status < 0 && *errp == NULL) {
+ error_setg(errp, "%s", vmaw->errmsg);
+ }
+
+ qemu_co_mutex_unlock(&vmaw->flush_lock);
+
+ return vmaw->status;
+}
+
+void vma_writer_destroy(VmaWriter *vmaw)
+{
+ assert(vmaw);
+
+ int i;
+
+ for (i = 0; i <= 255; i++) {
+ if (vmaw->stream_info[i].devname) {
+ g_free(vmaw->stream_info[i].devname);
+ }
+ }
+
+ if (vmaw->md5csum) {
+ g_checksum_free(vmaw->md5csum);
+ }
+
+ qemu_vfree(vmaw->headerbuf);
+ qemu_vfree(vmaw->outbuf);
+ g_free(vmaw);
+}
diff --git a/vma.c b/vma.c
new file mode 100644
index 0000000000..304f02bc84
--- /dev/null
+++ b/vma.c
@@ -0,0 +1,878 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+ * Copyright (C) 2012-2013 Proxmox Server Solutions
+ *
+ * Authors:
+ * Dietmar Maurer (dietmar@proxmox.com)
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "vma.h"
+#include "qemu/module.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+#include "qemu/cutils.h"
+#include "qemu/memalign.h"
+#include "qapi/qmp/qdict.h"
+#include "sysemu/block-backend.h"
+
+static void help(void)
+{
+ const char *help_msg =
+ "usage: vma command [command options]\n"
+ "\n"
+ "vma list <filename>\n"
+ "vma config <filename> [-c config]\n"
+ "vma create <filename> [-c config] pathname ...\n"
+ "vma extract <filename> [-r <fifo>] <targetdir>\n"
+ "vma verify <filename> [-v]\n"
+ ;
+
+ printf("%s", help_msg);
+ exit(1);
+}
+
+static const char *extract_devname(const char *path, char **devname, int index)
+{
+ assert(path);
+
+ const char *sep = strchr(path, '=');
+
+ if (sep) {
+ *devname = g_strndup(path, sep - path);
+ path = sep + 1;
+ } else {
+ if (index >= 0) {
+ *devname = g_strdup_printf("disk%d", index);
+ } else {
+ *devname = NULL;
+ }
+ }
+
+ return path;
+}
+
+static void print_content(VmaReader *vmar)
+{
+ assert(vmar);
+
+ VmaHeader *head = vma_reader_get_header(vmar);
+
+ GList *l = vma_reader_get_config_data(vmar);
+ while (l && l->data) {
+ VmaConfigData *cdata = (VmaConfigData *)l->data;
+ l = g_list_next(l);
+ printf("CFG: size: %d name: %s\n", cdata->len, cdata->name);
+ }
+
+ int i;
+ VmaDeviceInfo *di;
+ for (i = 1; i < 255; i++) {
+ di = vma_reader_get_device_info(vmar, i);
+ if (di) {
+ if (strcmp(di->devname, "vmstate") == 0) {
+ printf("VMSTATE: dev_id=%d memory: %zd\n", i, di->size);
+ } else {
+ printf("DEV: dev_id=%d size: %zd devname: %s\n",
+ i, di->size, di->devname);
+ }
+ }
+ }
+ /* ctime is the last entry we print */
+ printf("CTIME: %s", ctime(&head->ctime));
+ fflush(stdout);
+}
+
+static int list_content(int argc, char **argv)
+{
+ int c, ret = 0;
+ const char *filename;
+
+ for (;;) {
+ c = getopt(argc, argv, "h");
+ if (c == -1) {
+ break;
+ }
+ switch (c) {
+ case '?':
+ case 'h':
+ help();
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ }
+
+ /* Get the filename */
+ if ((optind + 1) != argc) {
+ help();
+ }
+ filename = argv[optind++];
+
+ Error *errp = NULL;
+ VmaReader *vmar = vma_reader_create(filename, &errp);
+
+ if (!vmar) {
+ g_error("%s", error_get_pretty(errp));
+ }
+
+ print_content(vmar);
+
+ vma_reader_destroy(vmar);
+
+ return ret;
+}
+
+typedef struct RestoreMap {
+ char *devname;
+ char *path;
+ char *format;
+ uint64_t throttling_bps;
+ char *throttling_group;
+ char *cache;
+ bool write_zero;
+ bool skip;
+} RestoreMap;
+
+static bool try_parse_option(char **line, const char *optname, char **out, const char *inbuf) {
+ size_t optlen = strlen(optname);
+ if (strncmp(*line, optname, optlen) != 0 || (*line)[optlen] != '=') {
+ return false;
+ }
+ if (*out) {
+ g_error("read map failed - duplicate value for option '%s'", optname);
+ }
+ char *value = (*line) + optlen + 1; /* including a '=' */
+ char *colon = strchr(value, ':');
+ if (!colon) {
+ g_error("read map failed - option '%s' not terminated ('%s')",
+ optname, inbuf);
+ }
+ *line = colon+1;
+ *out = g_strndup(value, colon - value);
+ return true;
+}
+
+static uint64_t verify_u64(const char *text) {
+ uint64_t value;
+ const char *endptr = NULL;
+ if (qemu_strtou64(text, &endptr, 0, &value) != 0 || !endptr || *endptr) {
+ g_error("read map failed - not a number: %s", text);
+ }
+ return value;
+}
+
+static int extract_content(int argc, char **argv)
+{
+ int c, ret = 0;
+ int verbose = 0;
+ const char *filename;
+ const char *dirname;
+ const char *readmap = NULL;
+
+ for (;;) {
+ c = getopt(argc, argv, "hvr:");
+ if (c == -1) {
+ break;
+ }
+ switch (c) {
+ case '?':
+ case 'h':
+ help();
+ break;
+ case 'r':
+ readmap = optarg;
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ help();
+ }
+ }
+
+ /* Get the filename */
+ if ((optind + 2) != argc) {
+ help();
+ }
+ filename = argv[optind++];
+ dirname = argv[optind++];
+
+ Error *errp = NULL;
+ VmaReader *vmar = vma_reader_create(filename, &errp);
+
+ if (!vmar) {
+ g_error("%s", error_get_pretty(errp));
+ }
+
+ if (mkdir(dirname, 0777) < 0) {
+ g_error("unable to create target directory %s - %s",
+ dirname, g_strerror(errno));
+ }
+
+ GList *l = vma_reader_get_config_data(vmar);
+ while (l && l->data) {
+ VmaConfigData *cdata = (VmaConfigData *)l->data;
+ l = g_list_next(l);
+ char *cfgfn = g_strdup_printf("%s/%s", dirname, cdata->name);
+ GError *err = NULL;
+ if (!g_file_set_contents(cfgfn, (gchar *)cdata->data, cdata->len,
+ &err)) {
+ g_error("unable to write file: %s", err->message);
+ }
+ }
+
+ GHashTable *devmap = g_hash_table_new(g_str_hash, g_str_equal);
+
+ if (readmap) {
+ print_content(vmar);
+
+ FILE *map = fopen(readmap, "r");
+ if (!map) {
+ g_error("unable to open fifo %s - %s", readmap, g_strerror(errno));
+ }
+
+ while (1) {
+ char inbuf[8192];
+ char *line = fgets(inbuf, sizeof(inbuf), map);
+ char *format = NULL;
+ char *bps = NULL;
+ char *group = NULL;
+ char *cache = NULL;
+ char *devname = NULL;
+ bool skip = false;
+ uint64_t bps_value = 0;
+ const char *path = NULL;
+ bool write_zero = true;
+
+ if (!line || line[0] == '\0' || !strcmp(line, "done\n")) {
+ break;
+ }
+ int len = strlen(line);
+ if (line[len - 1] == '\n') {
+ line[len - 1] = '\0';
+ len = len - 1;
+ if (len == 0) {
+ break;
+ }
+ }
+
+ if (strncmp(line, "skip", 4) == 0) {
+ if (len < 6 || line[4] != '=') {
+ g_error("read map failed - option 'skip' has no value ('%s')",
+ inbuf);
+ } else {
+ devname = line + 5;
+ skip = true;
+ }
+ } else {
+ while (1) {
+ if (!try_parse_option(&line, "format", &format, inbuf) &&
+ !try_parse_option(&line, "throttling.bps", &bps, inbuf) &&
+ !try_parse_option(&line, "throttling.group", &group, inbuf) &&
+ !try_parse_option(&line, "cache", &cache, inbuf))
+ {
+ break;
+ }
+ }
+
+ if (bps) {
+ bps_value = verify_u64(bps);
+ g_free(bps);
+ }
+
+ if (line[0] == '0' && line[1] == ':') {
+ path = line + 2;
+ write_zero = false;
+ } else if (line[0] == '1' && line[1] == ':') {
+ path = line + 2;
+ write_zero = true;
+ } else {
+ g_error("read map failed - parse error ('%s')", inbuf);
+ }
+
+ path = extract_devname(path, &devname, -1);
+ }
+
+ if (!devname) {
+ g_error("read map failed - no dev name specified ('%s')",
+ inbuf);
+ }
+
+ RestoreMap *map = g_new0(RestoreMap, 1);
+ map->devname = g_strdup(devname);
+ map->path = g_strdup(path);
+ map->format = format;
+ map->throttling_bps = bps_value;
+ map->throttling_group = group;
+ map->cache = cache;
+ map->write_zero = write_zero;
+ map->skip = skip;
+
+ g_hash_table_insert(devmap, map->devname, map);
+
+ };
+ }
+
+ int i;
+ int vmstate_fd = -1;
+ guint8 vmstate_stream = 0;
+
+ for (i = 1; i < 255; i++) {
+ VmaDeviceInfo *di = vma_reader_get_device_info(vmar, i);
+ if (di && (strcmp(di->devname, "vmstate") == 0)) {
+ vmstate_stream = i;
+ char *statefn = g_strdup_printf("%s/vmstate.bin", dirname);
+ vmstate_fd = open(statefn, O_WRONLY|O_CREAT|O_EXCL, 0644);
+ if (vmstate_fd < 0) {
+ g_error("create vmstate file '%s' failed - %s", statefn,
+ g_strerror(errno));
+ }
+ g_free(statefn);
+ } else if (di) {
+ char *devfn = NULL;
+ const char *format = NULL;
+ uint64_t throttling_bps = 0;
+ const char *throttling_group = NULL;
+ const char *cache = NULL;
+ int flags = BDRV_O_RDWR;
+ bool write_zero = true;
+ bool skip = false;
+
+ BlockBackend *blk = NULL;
+
+ if (readmap) {
+ RestoreMap *map;
+ map = (RestoreMap *)g_hash_table_lookup(devmap, di->devname);
+ if (map == NULL) {
+ g_error("no device name mapping for %s", di->devname);
+ }
+ devfn = map->path;
+ format = map->format;
+ throttling_bps = map->throttling_bps;
+ throttling_group = map->throttling_group;
+ cache = map->cache;
+ write_zero = map->write_zero;
+ skip = map->skip;
+ } else {
+ devfn = g_strdup_printf("%s/tmp-disk-%s.raw",
+ dirname, di->devname);
+ printf("DEVINFO %s %zd\n", devfn, di->size);
+
+ bdrv_img_create(devfn, "raw", NULL, NULL, NULL, di->size,
+ flags, true, &errp);
+ if (errp) {
+ g_error("can't create file %s: %s", devfn,
+ error_get_pretty(errp));
+ }
+
+ /* Note: we created an empty file above, so there is no
+ * need to write zeroes (so we generate a sparse file)
+ */
+ write_zero = false;
+ }
+
+ if (!skip) {
+ size_t devlen = strlen(devfn);
+ QDict *options = NULL;
+ bool writethrough;
+ if (format) {
+ /* explicit format from commandline */
+ options = qdict_new();
+ qdict_put_str(options, "driver", format);
+ } else if ((devlen > 4 && strcmp(devfn+devlen-4, ".raw") == 0) ||
+ strncmp(devfn, "/dev/", 5) == 0)
+ {
+ /* This part is now deprecated for PVE as well (just as qemu
+ * deprecated not specifying an explicit raw format, too.
+ */
+ /* explicit raw format */
+ options = qdict_new();
+ qdict_put_str(options, "driver", "raw");
+ }
+
+ if (cache && bdrv_parse_cache_mode(cache, &flags, &writethrough)) {
+ g_error("invalid cache option: %s\n", cache);
+ }
+
+ if (errp || !(blk = blk_new_open(devfn, NULL, options, flags, &errp))) {
+ g_error("can't open file %s - %s", devfn,
+ error_get_pretty(errp));
+ }
+
+ if (cache) {
+ blk_set_enable_write_cache(blk, !writethrough);
+ }
+
+ if (throttling_group) {
+ blk_io_limits_enable(blk, throttling_group);
+ }
+
+ if (throttling_bps) {
+ if (!throttling_group) {
+ blk_io_limits_enable(blk, devfn);
+ }
+
+ ThrottleConfig cfg;
+ throttle_config_init(&cfg);
+ cfg.buckets[THROTTLE_BPS_WRITE].avg = throttling_bps;
+ Error *err = NULL;
+ if (!throttle_is_valid(&cfg, &err)) {
+ error_report_err(err);
+ g_error("failed to apply throttling");
+ }
+ blk_set_io_limits(blk, &cfg);
+ }
+ }
+
+ if (vma_reader_register_bs(vmar, i, blk, write_zero, skip, &errp) < 0) {
+ g_error("%s", error_get_pretty(errp));
+ }
+
+ if (!readmap) {
+ g_free(devfn);
+ }
+ }
+ }
+
+ if (vma_reader_restore(vmar, vmstate_fd, verbose, &errp) < 0) {
+ g_error("restore failed - %s", error_get_pretty(errp));
+ }
+
+ if (!readmap) {
+ for (i = 1; i < 255; i++) {
+ VmaDeviceInfo *di = vma_reader_get_device_info(vmar, i);
+ if (di && (i != vmstate_stream)) {
+ char *tmpfn = g_strdup_printf("%s/tmp-disk-%s.raw",
+ dirname, di->devname);
+ char *fn = g_strdup_printf("%s/disk-%s.raw",
+ dirname, di->devname);
+ if (rename(tmpfn, fn) != 0) {
+ g_error("rename %s to %s failed - %s",
+ tmpfn, fn, g_strerror(errno));
+ }
+ }
+ }
+ }
+
+ vma_reader_destroy(vmar);
+
+ bdrv_close_all();
+
+ return ret;
+}
+
+static int verify_content(int argc, char **argv)
+{
+ int c, ret = 0;
+ int verbose = 0;
+ const char *filename;
+
+ for (;;) {
+ c = getopt(argc, argv, "hv");
+ if (c == -1) {
+ break;
+ }
+ switch (c) {
+ case '?':
+ case 'h':
+ help();
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ help();
+ }
+ }
+
+ /* Get the filename */
+ if ((optind + 1) != argc) {
+ help();
+ }
+ filename = argv[optind++];
+
+ Error *errp = NULL;
+ VmaReader *vmar = vma_reader_create(filename, &errp);
+
+ if (!vmar) {
+ g_error("%s", error_get_pretty(errp));
+ }
+
+ if (verbose) {
+ print_content(vmar);
+ }
+
+ if (vma_reader_verify(vmar, verbose, &errp) < 0) {
+ g_error("verify failed - %s", error_get_pretty(errp));
+ }
+
+ vma_reader_destroy(vmar);
+
+ bdrv_close_all();
+
+ return ret;
+}
+
+typedef struct BackupJob {
+ BlockBackend *target;
+ int64_t len;
+ VmaWriter *vmaw;
+ uint8_t dev_id;
+} BackupJob;
+
+#define BACKUP_SECTORS_PER_CLUSTER (VMA_CLUSTER_SIZE / BDRV_SECTOR_SIZE)
+
+static void coroutine_fn backup_run_empty(void *opaque)
+{
+ VmaWriter *vmaw = (VmaWriter *)opaque;
+
+ vma_writer_flush_output(vmaw);
+
+ Error *err = NULL;
+ if (vma_writer_close(vmaw, &err) != 0) {
+ g_warning("vma_writer_close failed %s", error_get_pretty(err));
+ }
+}
+
+static void coroutine_fn backup_run(void *opaque)
+{
+ BackupJob *job = (BackupJob *)opaque;
+ struct iovec iov;
+ QEMUIOVector qiov;
+
+ int64_t start, end, readlen;
+ int ret = 0;
+
+ unsigned char *buf = blk_blockalign(job->target, VMA_CLUSTER_SIZE);
+
+ start = 0;
+ end = DIV_ROUND_UP(job->len / BDRV_SECTOR_SIZE,
+ BACKUP_SECTORS_PER_CLUSTER);
+
+ for (; start < end; start++) {
+ iov.iov_base = buf;
+ iov.iov_len = VMA_CLUSTER_SIZE;
+ qemu_iovec_init_external(&qiov, &iov, 1);
+
+ if (start + 1 == end) {
+ memset(buf, 0, VMA_CLUSTER_SIZE);
+ readlen = job->len - start * VMA_CLUSTER_SIZE;
+ assert(readlen > 0 && readlen <= VMA_CLUSTER_SIZE);
+ } else {
+ readlen = VMA_CLUSTER_SIZE;
+ }
+
+ ret = blk_co_preadv(job->target, start * VMA_CLUSTER_SIZE,
+ readlen, &qiov, 0);
+ if (ret < 0) {
+ vma_writer_set_error(job->vmaw, "read error", -1);
+ goto out;
+ }
+
+ size_t zb = 0;
+ if (vma_writer_write(job->vmaw, job->dev_id, start, buf, &zb) < 0) {
+ vma_writer_set_error(job->vmaw, "backup_dump_cb vma_writer_write failed", -1);
+ goto out;
+ }
+ }
+
+
+out:
+ if (vma_writer_close_stream(job->vmaw, job->dev_id) <= 0) {
+ Error *err = NULL;
+ if (vma_writer_close(job->vmaw, &err) != 0) {
+ g_warning("vma_writer_close failed %s", error_get_pretty(err));
+ }
+ }
+ qemu_vfree(buf);
+}
+
+static int create_archive(int argc, char **argv)
+{
+ int i, c;
+ int verbose = 0;
+ const char *archivename;
+ GList *backup_coroutines = NULL;
+ GList *config_files = NULL;
+
+ for (;;) {
+ c = getopt(argc, argv, "hvc:");
+ if (c == -1) {
+ break;
+ }
+ switch (c) {
+ case '?':
+ case 'h':
+ help();
+ break;
+ case 'c':
+ config_files = g_list_append(config_files, optarg);
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ }
+
+
+ /* make sure we an archive name */
+ if ((optind + 1) > argc) {
+ help();
+ }
+
+ archivename = argv[optind++];
+
+ uuid_t uuid;
+ uuid_generate(uuid);
+
+ Error *local_err = NULL;
+ VmaWriter *vmaw = vma_writer_create(archivename, uuid, &local_err);
+
+ if (vmaw == NULL) {
+ g_error("%s", error_get_pretty(local_err));
+ }
+
+ GList *l = config_files;
+ while (l && l->data) {
+ char *name = l->data;
+ char *cdata = NULL;
+ gsize clen = 0;
+ GError *err = NULL;
+ if (!g_file_get_contents(name, &cdata, &clen, &err)) {
+ unlink(archivename);
+ g_error("Unable to read file: %s", err->message);
+ }
+
+ if (vma_writer_add_config(vmaw, name, cdata, clen) != 0) {
+ unlink(archivename);
+ g_error("Unable to append config data %s (len = %zd)",
+ name, clen);
+ }
+ l = g_list_next(l);
+ }
+
+ int devcount = 0;
+ while (optind < argc) {
+ const char *path = argv[optind++];
+ char *devname = NULL;
+ path = extract_devname(path, &devname, devcount++);
+
+ Error *errp = NULL;
+ BlockBackend *target;
+
+ target = blk_new_open(path, NULL, NULL, 0, &errp);
+ if (!target) {
+ unlink(archivename);
+ g_error("bdrv_open '%s' failed - %s", path, error_get_pretty(errp));
+ }
+ int64_t size = blk_getlength(target);
+ int dev_id = vma_writer_register_stream(vmaw, devname, size);
+ if (dev_id <= 0) {
+ unlink(archivename);
+ g_error("vma_writer_register_stream '%s' failed", devname);
+ }
+
+ BackupJob *job = g_new0(BackupJob, 1);
+ job->len = size;
+ job->target = target;
+ job->vmaw = vmaw;
+ job->dev_id = dev_id;
+
+ Coroutine *co = qemu_coroutine_create(backup_run, job);
+ // Don't enter coroutine yet, because it might write the header before
+ // all streams can be registered.
+ backup_coroutines = g_list_append(backup_coroutines, co);
+ }
+
+ VmaStatus vmastat;
+ int percent = 0;
+ int last_percent = -1;
+
+ if (devcount) {
+ GList *entry = backup_coroutines;
+ while (entry && entry->data) {
+ Coroutine *co = entry->data;
+ qemu_coroutine_enter(co);
+ entry = g_list_next(entry);
+ }
+
+ while (1) {
+ main_loop_wait(false);
+ vma_writer_get_status(vmaw, &vmastat);
+
+ if (verbose) {
+
+ uint64_t total = 0;
+ uint64_t transferred = 0;
+ uint64_t zero_bytes = 0;
+
+ int i;
+ for (i = 0; i < 256; i++) {
+ if (vmastat.stream_info[i].size) {
+ total += vmastat.stream_info[i].size;
+ transferred += vmastat.stream_info[i].transferred;
+ zero_bytes += vmastat.stream_info[i].zero_bytes;
+ }
+ }
+ percent = (transferred*100)/total;
+ if (percent != last_percent) {
+ fprintf(stderr, "progress %d%% %zd/%zd %zd\n", percent,
+ transferred, total, zero_bytes);
+ fflush(stderr);
+
+ last_percent = percent;
+ }
+ }
+
+ if (vmastat.closed) {
+ break;
+ }
+ }
+ } else {
+ Coroutine *co = qemu_coroutine_create(backup_run_empty, vmaw);
+ qemu_coroutine_enter(co);
+ while (1) {
+ main_loop_wait(false);
+ vma_writer_get_status(vmaw, &vmastat);
+ if (vmastat.closed) {
+ break;
+ }
+ }
+ }
+
+ bdrv_drain_all();
+
+ vma_writer_get_status(vmaw, &vmastat);
+
+ if (verbose) {
+ for (i = 0; i < 256; i++) {
+ VmaStreamInfo *si = &vmastat.stream_info[i];
+ if (si->size) {
+ fprintf(stderr, "image %s: size=%zd zeros=%zd saved=%zd\n",
+ si->devname, si->size, si->zero_bytes,
+ si->size - si->zero_bytes);
+ }
+ }
+ }
+
+ if (vmastat.status < 0) {
+ unlink(archivename);
+ g_error("creating vma archive failed");
+ }
+
+ g_list_free(backup_coroutines);
+ g_list_free(config_files);
+ vma_writer_destroy(vmaw);
+ return 0;
+}
+
+static int dump_config(int argc, char **argv)
+{
+ int c, ret = 0;
+ const char *filename;
+ const char *config_name = "qemu-server.conf";
+
+ for (;;) {
+ c = getopt(argc, argv, "hc:");
+ if (c == -1) {
+ break;
+ }
+ switch (c) {
+ case '?':
+ case 'h':
+ help();
+ break;
+ case 'c':
+ config_name = optarg;
+ break;
+ default:
+ help();
+ }
+ }
+
+ /* Get the filename */
+ if ((optind + 1) != argc) {
+ help();
+ }
+ filename = argv[optind++];
+
+ Error *errp = NULL;
+ VmaReader *vmar = vma_reader_create(filename, &errp);
+
+ if (!vmar) {
+ g_error("%s", error_get_pretty(errp));
+ }
+
+ int found = 0;
+ GList *l = vma_reader_get_config_data(vmar);
+ while (l && l->data) {
+ VmaConfigData *cdata = (VmaConfigData *)l->data;
+ l = g_list_next(l);
+ if (strcmp(cdata->name, config_name) == 0) {
+ found = 1;
+ fwrite(cdata->data, cdata->len, 1, stdout);
+ break;
+ }
+ }
+
+ vma_reader_destroy(vmar);
+
+ bdrv_close_all();
+
+ if (!found) {
+ fprintf(stderr, "unable to find configuration data '%s'\n", config_name);
+ return -1;
+ }
+
+ return ret;
+}
+
+int main(int argc, char **argv)
+{
+ const char *cmdname;
+ Error *main_loop_err = NULL;
+
+ error_init(argv[0]);
+ module_call_init(MODULE_INIT_TRACE);
+ qemu_init_exec_dir(argv[0]);
+
+ if (qemu_init_main_loop(&main_loop_err)) {
+ g_error("%s", error_get_pretty(main_loop_err));
+ }
+
+ bdrv_init();
+ module_call_init(MODULE_INIT_QOM);
+
+ if (argc < 2) {
+ help();
+ }
+
+ cmdname = argv[1];
+ argc--; argv++;
+
+
+ if (!strcmp(cmdname, "list")) {
+ return list_content(argc, argv);
+ } else if (!strcmp(cmdname, "create")) {
+ return create_archive(argc, argv);
+ } else if (!strcmp(cmdname, "extract")) {
+ return extract_content(argc, argv);
+ } else if (!strcmp(cmdname, "verify")) {
+ return verify_content(argc, argv);
+ } else if (!strcmp(cmdname, "config")) {
+ return dump_config(argc, argv);
+ }
+
+ help();
+ return 0;
+}
diff --git a/vma.h b/vma.h
new file mode 100644
index 0000000000..1b62859165
--- /dev/null
+++ b/vma.h
@@ -0,0 +1,150 @@
+/*
+ * VMA: Virtual Machine Archive
+ *
+ * Copyright (C) Proxmox Server Solutions
+ *
+ * Authors:
+ * Dietmar Maurer (dietmar@proxmox.com)
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef BACKUP_VMA_H
+#define BACKUP_VMA_H
+
+#include <uuid/uuid.h>
+#include "qapi/error.h"
+#include "block/block.h"
+
+#define VMA_BLOCK_BITS 12
+#define VMA_BLOCK_SIZE (1<<VMA_BLOCK_BITS)
+#define VMA_CLUSTER_BITS (VMA_BLOCK_BITS+4)
+#define VMA_CLUSTER_SIZE (1<<VMA_CLUSTER_BITS)
+
+#if VMA_CLUSTER_SIZE != 65536
+#error unexpected cluster size
+#endif
+
+#define VMA_EXTENT_HEADER_SIZE 512
+#define VMA_BLOCKS_PER_EXTENT 59
+#define VMA_MAX_CONFIGS 256
+
+#define VMA_MAX_EXTENT_SIZE \
+ (VMA_EXTENT_HEADER_SIZE+VMA_CLUSTER_SIZE*VMA_BLOCKS_PER_EXTENT)
+#if VMA_MAX_EXTENT_SIZE != 3867136
+#error unexpected VMA_EXTENT_SIZE
+#endif
+
+/* File Format Definitions */
+
+#define VMA_MAGIC (GUINT32_TO_BE(('V'<<24)|('M'<<16)|('A'<<8)|0x00))
+#define VMA_EXTENT_MAGIC (GUINT32_TO_BE(('V'<<24)|('M'<<16)|('A'<<8)|'E'))
+
+typedef struct VmaDeviceInfoHeader {
+ uint32_t devname_ptr; /* offset into blob_buffer table */
+ uint32_t reserved0;
+ uint64_t size; /* device size in bytes */
+ uint64_t reserved1;
+ uint64_t reserved2;
+} VmaDeviceInfoHeader;
+
+typedef struct VmaHeader {
+ uint32_t magic;
+ uint32_t version;
+ unsigned char uuid[16];
+ int64_t ctime;
+ unsigned char md5sum[16];
+
+ uint32_t blob_buffer_offset;
+ uint32_t blob_buffer_size;
+ uint32_t header_size;
+
+ unsigned char reserved[1984];
+
+ uint32_t config_names[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
+ uint32_t config_data[VMA_MAX_CONFIGS]; /* offset into blob_buffer table */
+
+ uint32_t reserved1;
+
+ VmaDeviceInfoHeader dev_info[256];
+} VmaHeader;
+
+typedef struct VmaExtentHeader {
+ uint32_t magic;
+ uint16_t reserved1;
+ uint16_t block_count;
+ unsigned char uuid[16];
+ unsigned char md5sum[16];
+ uint64_t blockinfo[VMA_BLOCKS_PER_EXTENT];
+} VmaExtentHeader;
+
+/* functions/definitions to read/write vma files */
+
+typedef struct VmaReader VmaReader;
+
+typedef struct VmaWriter VmaWriter;
+
+typedef struct VmaConfigData {
+ const char *name;
+ const void *data;
+ uint32_t len;
+} VmaConfigData;
+
+typedef struct VmaStreamInfo {
+ uint64_t size;
+ uint64_t cluster_count;
+ uint64_t transferred;
+ uint64_t zero_bytes;
+ int finished;
+ char *devname;
+} VmaStreamInfo;
+
+typedef struct VmaStatus {
+ int status;
+ bool closed;
+ char errmsg[8192];
+ char uuid_str[37];
+ VmaStreamInfo stream_info[256];
+} VmaStatus;
+
+typedef struct VmaDeviceInfo {
+ uint64_t size; /* device size in bytes */
+ const char *devname;
+} VmaDeviceInfo;
+
+VmaWriter *vma_writer_create(const char *filename, uuid_t uuid, Error **errp);
+int vma_writer_close(VmaWriter *vmaw, Error **errp);
+void vma_writer_error_propagate(VmaWriter *vmaw, Error **errp);
+void vma_writer_destroy(VmaWriter *vmaw);
+int vma_writer_add_config(VmaWriter *vmaw, const char *name, gpointer data,
+ size_t len);
+int vma_writer_register_stream(VmaWriter *vmaw, const char *devname,
+ size_t size);
+
+int64_t coroutine_fn vma_writer_write(VmaWriter *vmaw, uint8_t dev_id,
+ int64_t cluster_num,
+ const unsigned char *buf,
+ size_t *zero_bytes);
+
+int coroutine_fn vma_writer_close_stream(VmaWriter *vmaw, uint8_t dev_id);
+int coroutine_fn vma_writer_flush_output(VmaWriter *vmaw);
+
+int vma_writer_get_status(VmaWriter *vmaw, VmaStatus *status);
+void vma_writer_set_error(VmaWriter *vmaw, const char *fmt, ...);
+
+
+VmaReader *vma_reader_create(const char *filename, Error **errp);
+void vma_reader_destroy(VmaReader *vmar);
+VmaHeader *vma_reader_get_header(VmaReader *vmar);
+GList *vma_reader_get_config_data(VmaReader *vmar);
+VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id);
+int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id,
+ BlockBackend *target, bool write_zeroes,
+ bool skip, Error **errp);
+int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
+ Error **errp);
+int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp);
+
+#endif /* BACKUP_VMA_H */
Reference in New Issue View Git Blame Copy Permalink