diff --git a/include/sys/taskq.h b/include/sys/taskq.h index 7b44e8b8a..2c437f0e7 100644 --- a/include/sys/taskq.h +++ b/include/sys/taskq.h @@ -119,7 +119,7 @@ extern void taskq_init_ent(taskq_ent_t *); extern taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t); extern void taskq_destroy(taskq_t *); extern void taskq_wait_id(taskq_t *, taskqid_t); -extern void taskq_wait_all(taskq_t *, taskqid_t); +extern void taskq_wait_outstanding(taskq_t *, taskqid_t); extern void taskq_wait(taskq_t *); extern int taskq_cancel_id(taskq_t *, taskqid_t); extern int taskq_member(taskq_t *, void *); diff --git a/module/spl/spl-taskq.c b/module/spl/spl-taskq.c index 951298d9f..49bb40a25 100644 --- a/module/spl/spl-taskq.c +++ b/module/spl/spl-taskq.c @@ -327,6 +327,33 @@ taskq_find(taskq_t *tq, taskqid_t id, int *active) return (NULL); } +/* + * Theory for the taskq_wait_id(), taskq_wait_outstanding(), and + * taskq_wait() functions below. + * + * Taskq waiting is accomplished by tracking the lowest outstanding task + * id and the next available task id. As tasks are dispatched they are + * added to the tail of the pending, priority, or delay lists. As worker + * threads become available the tasks are removed from the heads of these + * lists and linked to the worker threads. This ensures the lists are + * kept sorted by lowest to highest task id. + * + * Therefore the lowest outstanding task id can be quickly determined by + * checking the head item from all of these lists. This value is stored + * with the taskq as the lowest id. It only needs to be recalculated when + * either the task with the current lowest id completes or is canceled. + * + * By blocking until the lowest task id exceeds the passed task id the + * taskq_wait_outstanding() function can be easily implemented. Similarly, + * by blocking until the lowest task id matches the next task id taskq_wait() + * can be implemented. + * + * Callers should be aware that when there are multiple worked threads it + * is possible for larger task ids to complete before smaller ones. Also + * when the taskq contains delay tasks with small task ids callers may + * block for a considerable length of time waiting for them to expire and + * execute. + */ static int taskq_wait_id_check(taskq_t *tq, taskqid_t id) { @@ -351,34 +378,8 @@ taskq_wait_id(taskq_t *tq, taskqid_t id) } EXPORT_SYMBOL(taskq_wait_id); -/* - * The taskq_wait() function will block until all previously submitted - * tasks have been completed. A previously submitted task is defined as - * a task with a lower task id than the current task queue id. Note that - * all task id's are assigned monotonically at dispatch time. - * - * Waiting for all previous tasks to complete is accomplished by tracking - * the lowest outstanding task id. As tasks are dispatched they are added - * added to the tail of the pending, priority, or delay lists. And as - * worker threads become available the tasks are removed from the heads - * of these lists and linked to the worker threads. This ensures the - * lists are kept in lowest to highest task id order. - * - * Therefore the lowest outstanding task id can be quickly determined by - * checking the head item from all of these lists. This value is stored - * with the task queue as the lowest id. It only needs to be recalculated - * when either the task with the current lowest id completes or is canceled. - * - * By blocking until the lowest task id exceeds the current task id when - * the function was called we ensure all previous tasks have completed. - * - * NOTE: When there are multiple worked threads it is possible for larger - * task ids to complete before smaller ones. Conversely when the task - * queue contains delay tasks with small task ids, you may block for a - * considerable length of time waiting for them to expire and execute. - */ static int -taskq_wait_check(taskq_t *tq, taskqid_t id) +taskq_wait_outstanding_check(taskq_t *tq, taskqid_t id) { int rc; @@ -389,26 +390,42 @@ taskq_wait_check(taskq_t *tq, taskqid_t id) return (rc); } +/* + * The taskq_wait_outstanding() function will block until all tasks with a + * lower taskqid than the passed 'id' have been completed. Note that all + * task id's are assigned monotonically at dispatch time. Zero may be + * passed for the id to indicate all tasks dispatch up to this point, + * but not after, should be waited for. + */ void -taskq_wait_all(taskq_t *tq, taskqid_t id) +taskq_wait_outstanding(taskq_t *tq, taskqid_t id) { - wait_event(tq->tq_wait_waitq, taskq_wait_check(tq, id)); + wait_event(tq->tq_wait_waitq, + taskq_wait_outstanding_check(tq, id ? id : tq->tq_next_id - 1)); } -EXPORT_SYMBOL(taskq_wait_all); +EXPORT_SYMBOL(taskq_wait_outstanding); +static int +taskq_wait_check(taskq_t *tq) +{ + int rc; + + spin_lock_irqsave(&tq->tq_lock, tq->tq_lock_flags); + rc = (tq->tq_lowest_id == tq->tq_next_id); + spin_unlock_irqrestore(&tq->tq_lock, tq->tq_lock_flags); + + return (rc); +} + +/* + * The taskq_wait() function will block until the taskq is empty. + * This means that if a taskq re-dispatches work to itself taskq_wait() + * callers will block indefinitely. + */ void taskq_wait(taskq_t *tq) { - taskqid_t id; - - ASSERT(tq); - - /* Wait for the largest outstanding taskqid */ - spin_lock_irqsave(&tq->tq_lock, tq->tq_lock_flags); - id = tq->tq_next_id - 1; - spin_unlock_irqrestore(&tq->tq_lock, tq->tq_lock_flags); - - taskq_wait_all(tq, id); + wait_event(tq->tq_wait_waitq, taskq_wait_check(tq)); } EXPORT_SYMBOL(taskq_wait); diff --git a/module/splat/splat-taskq.c b/module/splat/splat-taskq.c index 2787bf401..7d4ad5b69 100644 --- a/module/splat/splat-taskq.c +++ b/module/splat/splat-taskq.c @@ -588,10 +588,10 @@ splat_taskq_test4(struct file *file, void *arg) * next pending task as soon as it completes its current task. This * means that tasks do not strictly complete in order in which they * were dispatched (increasing task id). This is fine but we need to - * verify that taskq_wait_all() blocks until the passed task id and all - * lower task ids complete. We do this by dispatching the following + * verify taskq_wait_outstanding() blocks until the passed task id and + * all lower task ids complete. We do this by dispatching the following * specific sequence of tasks each of which block for N time units. - * We then use taskq_wait_all() to unblock at specific task id and + * We then use taskq_wait_outstanding() to unblock at specific task id and * verify the only the expected task ids have completed and in the * correct order. The two cases of interest are: * @@ -602,17 +602,17 @@ splat_taskq_test4(struct file *file, void *arg) * * The following table shows each task id and how they will be * scheduled. Each rows represent one time unit and each column - * one of the three worker threads. The places taskq_wait_all() + * one of the three worker threads. The places taskq_wait_outstanding() * must unblock for a specific id are identified as well as the * task ids which must have completed and their order. * - * +-----+ <--- taskq_wait_all(tq, 8) unblocks + * +-----+ <--- taskq_wait_outstanding(tq, 8) unblocks * | | Required Completion Order: 1,2,4,5,3,8,6,7 * +-----+ | * | | | * | | +-----+ * | | | 8 | - * | | +-----+ <--- taskq_wait_all(tq, 3) unblocks + * | | +-----+ <--- taskq_wait_outstanding(tq, 3) unblocks * | | 7 | | Required Completion Order: 1,2,4,5,3 * | +-----+ | * | 6 | | | @@ -755,13 +755,13 @@ splat_taskq_test5_impl(struct file *file, void *arg, boolean_t prealloc) splat_vprint(file, SPLAT_TASKQ_TEST5_NAME, "Taskq '%s' " "waiting for taskqid %d completion\n", tq_arg.name, 3); - taskq_wait_all(tq, 3); + taskq_wait_outstanding(tq, 3); if ((rc = splat_taskq_test_order(&tq_arg, order1))) goto out; splat_vprint(file, SPLAT_TASKQ_TEST5_NAME, "Taskq '%s' " "waiting for taskqid %d completion\n", tq_arg.name, 8); - taskq_wait_all(tq, 8); + taskq_wait_outstanding(tq, 8); rc = splat_taskq_test_order(&tq_arg, order2); out: @@ -923,7 +923,7 @@ splat_taskq_test6_impl(struct file *file, void *arg, boolean_t prealloc) splat_vprint(file, SPLAT_TASKQ_TEST6_NAME, "Taskq '%s' " "waiting for taskqid %d completion\n", tq_arg.name, SPLAT_TASKQ_ORDER_MAX); - taskq_wait_all(tq, SPLAT_TASKQ_ORDER_MAX); + taskq_wait_outstanding(tq, SPLAT_TASKQ_ORDER_MAX); rc = splat_taskq_test_order(&tq_arg, order); out: @@ -1030,7 +1030,7 @@ splat_taskq_test7_impl(struct file *file, void *arg, boolean_t prealloc) if (tq_arg->flag == 0) { splat_vprint(file, SPLAT_TASKQ_TEST7_NAME, "Taskq '%s' waiting\n", tq_arg->name); - taskq_wait_all(tq, SPLAT_TASKQ_DEPTH_MAX); + taskq_wait_outstanding(tq, SPLAT_TASKQ_DEPTH_MAX); } error = (tq_arg->depth == SPLAT_TASKQ_DEPTH_MAX ? 0 : -EINVAL);