c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2026-05-22 18:40:43 +03:00
Code Issues Packages Projects Releases Wiki Activity

zvol: ensure device minors are properly cleaned up

Browse Source 
Currently, if a minor is in use when we try to remove it, we'll skip it
and never come back to it again. Since the zvol state is hung off the
minor in the kernel, this can get us into weird situations if something
tries to use it after the removal fails. It's even worse at pool export,
as there's now a vestigial zvol state with no pool under it. It's
weirder again if the pool is subsequently reimported, as the zvol code
(reasonably) assumes the zvol state has been properly setup, when it's
actually left over from the previous import of the pool.

This commit attempts to tackle that by setting a flag on the zvol if its
minor can't be removed, and then checking that flag when a request is
made and rejecting it, thus stopping new work coming in.

The flag also causes a condvar to be signaled when the last client
finishes. For the case where a single minor is being removed (eg
changing volmode), it will wait for this signal before proceeding.
Meanwhile, when removing all minors, a background task is created for
each minor that couldn't be removed on the spot, and those tasks then
wake and clean up.

Since any new tasks are queued on to the pool's spa_zvol_taskq,
spa_export_common() will continue to wait at export until all minors are
removed.

Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #14872
Closes #16364
This commit is contained in:
Rob Norris
2024-07-18 13:24:05 +10:00
committed by Tony Hutter
parent f3a9746f21
commit cf80a803d5
4 changed files with 122 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
module/zfs/zvol.c
+93 -25
View File
@@ -37,6 +37,7 @@
* Copyright 2014 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2016 Actifio, Inc. All rights reserved.
* Copyright (c) 2012, 2019 by Delphix. All rights reserved.
* Copyright (c) 2024, Klara, Inc.
*/
/*
@@ -868,6 +869,9 @@ zvol_resume(zvol_state_t *zv)
*/
atomic_dec(&zv->zv_suspend_ref);
if (zv->zv_flags & ZVOL_REMOVING)
cv_broadcast(&zv->zv_removing_cv);
return (SET_ERROR(error));
}
@@ -903,6 +907,9 @@ zvol_last_close(zvol_state_t *zv)
ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
ASSERT(MUTEX_HELD(&zv->zv_state_lock));
if (zv->zv_flags & ZVOL_REMOVING)
cv_broadcast(&zv->zv_removing_cv);
zvol_shutdown_zv(zv);
dmu_objset_disown(zv->zv_objset, 1, zv);
@@ -1195,6 +1202,41 @@ zvol_create_minor(const char *name)
* Remove minors for specified dataset including children and snapshots.
*/
/*
* Remove the minor for a given zvol. This will do it all:
* - flag the zvol for removal, so new requests are rejected
* - wait until outstanding requests are completed
* - remove it from lists
* - free it
* It's also usable as a taskq task, and smells nice too.
*/
static void
zvol_remove_minor_task(void *arg)
{
zvol_state_t *zv = (zvol_state_t *)arg;
ASSERT(!RW_LOCK_HELD(&zvol_state_lock));
ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
mutex_enter(&zv->zv_state_lock);
while (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) {
zv->zv_flags |= ZVOL_REMOVING;
cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock);
}
mutex_exit(&zv->zv_state_lock);
rw_enter(&zvol_state_lock, RW_WRITER);
mutex_enter(&zv->zv_state_lock);
zvol_remove(zv);
zvol_os_clear_private(zv);
mutex_exit(&zv->zv_state_lock);
rw_exit(&zvol_state_lock);
zvol_os_free(zv);
}
static void
zvol_free_task(void *arg)
{
@@ -1207,11 +1249,13 @@ zvol_remove_minors_impl(const char *name)
zvol_state_t *zv, *zv_next;
int namelen = ((name) ? strlen(name) : 0);
taskqid_t t;
list_t free_list;
list_t delay_list, free_list;
if (zvol_inhibit_dev)
return;
list_create(&delay_list, sizeof (zvol_state_t),
offsetof(zvol_state_t, zv_next));
list_create(&free_list, sizeof (zvol_state_t),
offsetof(zvol_state_t, zv_next));
@@ -1230,9 +1274,24 @@ zvol_remove_minors_impl(const char *name)
* one is currently using this zv
*/
/* If in use, leave alone */
/*
* If in use, try to throw everyone off and try again
* later.
*/
if (zv->zv_open_count > 0 ||
atomic_read(&zv->zv_suspend_ref)) {
zv->zv_flags |= ZVOL_REMOVING;
t = taskq_dispatch(
zv->zv_objset->os_spa->spa_zvol_taskq,
zvol_remove_minor_task, zv, TQ_SLEEP);
if (t == TASKQID_INVALID) {
/*
* Couldn't create the task, so we'll
* do it in place once the loop is
* finished.
*/
list_insert_head(&delay_list, zv);
}
mutex_exit(&zv->zv_state_lock);
continue;
}
@@ -1259,7 +1318,11 @@ zvol_remove_minors_impl(const char *name)
}
rw_exit(&zvol_state_lock);
/* Drop zvol_state_lock before calling zvol_free() */
/* Wait for zvols that we couldn't create a remove task for */
while ((zv = list_remove_head(&delay_list)) != NULL)
zvol_remove_minor_task(zv);
/* Free any that we couldn't free in parallel earlier */
while ((zv = list_remove_head(&free_list)) != NULL)
zvol_os_free(zv);
}
@@ -1279,33 +1342,38 @@ zvol_remove_minor_impl(const char *name)
zv_next = list_next(&zvol_state_list, zv);
mutex_enter(&zv->zv_state_lock);
if (strcmp(zv->zv_name, name) == 0) {
/*
* By holding zv_state_lock here, we guarantee that no
* one is currently using this zv
*/
/* If in use, leave alone */
if (zv->zv_open_count > 0 ||
atomic_read(&zv->zv_suspend_ref)) {
mutex_exit(&zv->zv_state_lock);
continue;
}
zvol_remove(zv);
zvol_os_clear_private(zv);
mutex_exit(&zv->zv_state_lock);
if (strcmp(zv->zv_name, name) == 0)
/* Found, leave the the loop with zv_lock held */
break;
} else {
mutex_exit(&zv->zv_state_lock);
}
mutex_exit(&zv->zv_state_lock);
}
/* Drop zvol_state_lock before calling zvol_free() */
if (zv == NULL) {
rw_exit(&zvol_state_lock);
return;
}
ASSERT(MUTEX_HELD(&zv->zv_state_lock));
if (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) {
/*
* In use, so try to throw everyone off, then wait
* until finished.
*/
zv->zv_flags |= ZVOL_REMOVING;
mutex_exit(&zv->zv_state_lock);
rw_exit(&zvol_state_lock);
zvol_remove_minor_task(zv);
return;
}
zvol_remove(zv);
zvol_os_clear_private(zv);
mutex_exit(&zv->zv_state_lock);
rw_exit(&zvol_state_lock);
if (zv != NULL)
zvol_os_free(zv);
zvol_os_free(zv);
}
/*