Switch from _Noreturn to __attribute__((noreturn))

Parts of the Linux kernel build system struggle with _Noreturn.  This
results in the following warnings when building on RHEL 8.5, and likely
other environments.  Switch to using the __attribute__((noreturn)).

  warning: objtool: dbuf_free_range()+0x2b8:
    return with modified stack frame
  warning: objtool: dbuf_free_range()+0x0:
    stack state mismatch: cfa1=7+40 cfa2=7+8
  ...
  WARNING: EXPORT symbol "arc_buf_size" [zfs.ko] version generation
    failed, symbol will not be versioned.
  WARNING: EXPORT symbol "spa_open" [zfs.ko] version generation
    failed, symbol will not be versioned.
  ...

Additionally, __thread_exit() has been renamed spl_thread_exit() and
made a static inline function.  This was needed because the kernel
will generate a warning for symbols which are __attribute__((noreturn))
and then exported with EXPORT_SYMBOL.

While we could continue to use _Noreturn in user space I've also
switched it to __attribute__((noreturn)) purely for consistency
throughout the code base.

Reviewed-by: Ryan Moeller <freqlabs@FreeBSD.org>
Reviewed-by: Brian Atkinson <batkinson@lanl.gov>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #13238

cmd/raidz_test/raidz_test.c

@@ -839,7 +839,7 @@ static kcondvar_t sem_cv;
 static int max_free_slots;
 static int free_slots;
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 sweep_thread(void *arg)
 {
 	int err = 0;

cmd/zfs/zfs_main.c

@@ -499,7 +499,7 @@ usage_prop_cb(int prop, void *cb)
  * that command.  Otherwise, iterate over the entire command table and display
  * a complete usage message.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 usage(boolean_t requested)
 {
 	int i;

cmd/zgenhostid/zgenhostid.c

@@ -36,7 +36,7 @@
 #include <time.h>
 #include <unistd.h>
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 usage(void)
 {
 	(void) fprintf(stderr,

cmd/zhack/zhack.c

@@ -57,7 +57,7 @@ static importargs_t g_importargs;
 static char *g_pool;
 static boolean_t g_readonly;
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 usage(void)
 {
 	(void) fprintf(stderr,
@@ -87,7 +87,7 @@ usage(void)
 }
 
 
-static __attribute__((format(printf, 3, 4))) _Noreturn void
+static __attribute__((format(printf, 3, 4))) __attribute__((noreturn)) void
 fatal(spa_t *spa, void *tag, const char *fmt, ...)
 {
 	va_list ap;

cmd/zpool/zpool_main.c

@@ -518,7 +518,7 @@ print_vdev_prop_cb(int prop, void *cb)
  * that command.  Otherwise, iterate over the entire command table and display
  * a complete usage message.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 usage(boolean_t requested)
 {
 	FILE *fp = requested ? stdout : stderr;

cmd/ztest/ztest.c

@@ -558,7 +558,7 @@ enum ztest_object {
 	ZTEST_OBJECTS
 };
 
-static _Noreturn void usage(boolean_t);
+static __attribute__((noreturn)) void usage(boolean_t requested);
 static int ztest_scrub_impl(spa_t *spa);
 
 /*
@@ -622,7 +622,7 @@ static void sig_handler(int signo)
 
 char *fatal_msg;
 
-static __attribute__((format(printf, 2, 3))) _Noreturn void
+static __attribute__((format(printf, 2, 3))) __attribute__((noreturn)) void
 fatal(int do_perror, char *message, ...)
 {
 	va_list args;
@@ -831,7 +831,7 @@ fini_options(void)
 	short_opts = NULL;
 }
 
-static void
+static __attribute__((noreturn)) void
 usage(boolean_t requested)
 {
 	char option[80];
@@ -6994,7 +6994,7 @@ ztest_resume(spa_t *spa)
 	(void) zio_resume(spa);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 ztest_resume_thread(void *arg)
 {
 	spa_t *spa = arg;
@@ -7020,7 +7020,7 @@ ztest_resume_thread(void *arg)
 	thread_exit();
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 ztest_deadman_thread(void *arg)
 {
 	ztest_shared_t *zs = arg;
@@ -7098,7 +7098,7 @@ ztest_execute(int test, ztest_info_t *zi, uint64_t id)
 		    (double)functime / NANOSEC, zi->zi_funcname);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 ztest_thread(void *arg)
 {
 	int rand;

include/libuutil.h

@@ -85,13 +85,13 @@ extern void uu_warn(const char *, ...)
     __attribute__((format(printf, 1, 2)));
 extern void uu_vwarn(const char *, va_list)
     __attribute__((format(printf, 1, 0)));
-extern _Noreturn void uu_die(const char *, ...)
+extern __attribute__((noreturn)) void uu_die(const char *, ...)
     __attribute__((format(printf, 1, 2)));
-extern _Noreturn void uu_vdie(const char *, va_list)
+extern __attribute__((noreturn)) void uu_vdie(const char *, va_list)
     __attribute__((format(printf, 1, 0)));
-extern _Noreturn void uu_xdie(int, const char *, ...)
+extern __attribute__((noreturn)) void uu_xdie(int, const char *, ...)
     __attribute__((format(printf, 2, 3)));
-extern _Noreturn void uu_vxdie(int, const char *, va_list)
+extern __attribute__((noreturn)) void uu_vxdie(int, const char *, va_list)
     __attribute__((format(printf, 2, 0)));
 
 /*

include/os/linux/spl/sys/thread.h

@@ -53,7 +53,7 @@ typedef void (*thread_func_t)(void *);
 	__thread_create(stk, stksize, (thread_func_t)func, #func,	\
 	arg, len, pp, state, pri)
 
-#define	thread_exit()			__thread_exit()
+#define	thread_exit()			spl_thread_exit()
 #define	thread_join(t)			VERIFY(0)
 #define	curthread			current
 #define	getcomm()			current->comm
@@ -62,10 +62,16 @@ typedef void (*thread_func_t)(void *);
 extern kthread_t *__thread_create(caddr_t stk, size_t  stksize,
     thread_func_t func, const char *name, void *args, size_t len, proc_t *pp,
     int state, pri_t pri);
-extern __attribute__((noreturn)) void __thread_exit(void);
 extern struct task_struct *spl_kthread_create(int (*func)(void *),
     void *data, const char namefmt[], ...);
 
+static inline __attribute__((noreturn)) void
+spl_thread_exit(void)
+{
+	tsd_exit();
+	SPL_KTHREAD_COMPLETE_AND_EXIT(NULL, 0);
+}
+
 extern proc_t p0;
 
 #ifdef HAVE_SIGINFO

include/sys/zfs_context.h

@@ -152,8 +152,8 @@ extern void dprintf_setup(int *argc, char **argv);
 
 extern void cmn_err(int, const char *, ...);
 extern void vcmn_err(int, const char *, va_list);
-extern _Noreturn void panic(const char *, ...);
-extern _Noreturn void vpanic(const char *, va_list);
+extern __attribute__((noreturn)) void panic(const char *, ...);
+extern __attribute__((noreturn)) void vpanic(const char *, va_list);
 
 #define	fm_panic	panic
 

lib/libuutil/uu_pname.c

@@ -40,7 +40,7 @@
 
 static const char *pname;
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 uu_die_internal(int status, const char *format, va_list alist);
 
 int uu_exit_ok_value = EXIT_SUCCESS;
@@ -110,7 +110,7 @@ uu_warn(const char *format, ...)
 	va_end(alist);
 }
 
-static __attribute__((format(printf, 2, 0))) _Noreturn void
+static __attribute__((format(printf, 2, 0))) __attribute__((noreturn)) void
 uu_die_internal(int status, const char *format, va_list alist)
 {
 	uu_warn_internal(errno, format, alist);

lib/libzpool/kernel.c

@@ -633,7 +633,7 @@ __dprintf(boolean_t dprint, const char *file, const char *func,
 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 
-void
+__attribute__((noreturn)) void
 vpanic(const char *fmt, va_list adx)
 {
 	(void) fprintf(stderr, "error: ");
@@ -643,7 +643,7 @@ vpanic(const char *fmt, va_list adx)
 	abort();	/* think of it as a "user-level crash dump" */
 }
 
-void
+__attribute__((noreturn)) void
 panic(const char *fmt, ...)
 {
 	va_list adx;

lib/libzpool/taskq.c

@@ -211,7 +211,7 @@ taskq_wait_outstanding(taskq_t *tq, taskqid_t id)
 	taskq_wait(tq);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 taskq_thread(void *arg)
 {
 	taskq_t *tq = arg;

module/lua/ldo.c

@@ -91,7 +91,7 @@ static intptr_t stack_remaining(void) {
 typedef	struct _label_t { long long unsigned val[JMP_BUF_CNT]; } label_t;
 
 int setjmp(label_t *) __attribute__ ((__nothrow__));
-extern _Noreturn void longjmp(label_t *);
+extern __attribute__((noreturn)) void longjmp(label_t *);
 
 #define LUAI_THROW(L,c)		longjmp(&(c)->b)
 #define LUAI_TRY(L,c,a)		if (setjmp(&(c)->b) == 0) { a }

module/os/linux/spl/spl-thread.c

@@ -62,15 +62,6 @@ thread_generic_wrapper(void *arg)
 	return (0);
 }
 
-void
-__thread_exit(void)
-{
-	tsd_exit();
-	SPL_KTHREAD_COMPLETE_AND_EXIT(NULL, 0);
-	/* Unreachable */
-}
-EXPORT_SYMBOL(__thread_exit);
-
 /*
  * thread_create() may block forever if it cannot create a thread or
  * allocate memory.  This is preferable to returning a NULL which Solaris

module/zfs/arc.c

@@ -928,7 +928,7 @@ static unsigned long l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024;
 
 /* L2ARC persistence rebuild control routines. */
 void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen);
-static _Noreturn void l2arc_dev_rebuild_thread(void *arg);
+static __attribute__((noreturn)) void l2arc_dev_rebuild_thread(void *arg);
 static int l2arc_rebuild(l2arc_dev_t *dev);
 
 /* L2ARC persistence read I/O routines. */
@@ -9707,7 +9707,7 @@ l2arc_hdr_limit_reached(void)
  * This thread feeds the L2ARC at regular intervals.  This is the beating
  * heart of the L2ARC.
  */
-static _Noreturn void
+static  __attribute__((noreturn)) void
 l2arc_feed_thread(void *unused)
 {
 	(void) unused;
@@ -10146,7 +10146,7 @@ l2arc_spa_rebuild_start(spa_t *spa)
 /*
  * Main entry point for L2ARC rebuilding.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 l2arc_dev_rebuild_thread(void *arg)
 {
 	l2arc_dev_t *dev = arg;

module/zfs/dbuf.c

@@ -778,7 +778,7 @@ dbuf_evict_one(void)
  * of the dbuf cache is at or below the maximum size. Once the dbuf is aged
  * out of the cache it is destroyed and becomes eligible for arc eviction.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 dbuf_evict_thread(void *unused)
 {
 	(void) unused;

module/zfs/dmu_recv.c

@@ -2801,7 +2801,7 @@ receive_process_record(struct receive_writer_arg *rwa,
  * dmu_recv_stream's worker thread; pull records off the queue, and then call
  * receive_process_record  When we're done, signal the main thread and exit.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 receive_writer_thread(void *arg)
 {
 	struct receive_writer_arg *rwa = arg;

module/zfs/dmu_redact.c

@@ -351,7 +351,7 @@ redact_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	return (0);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 redact_traverse_thread(void *arg)
 {
 	struct redact_thread_arg *rt_arg = arg;
@@ -837,7 +837,7 @@ struct redact_merge_thread_arg {
 	int error_code;
 };
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 redact_merge_thread(void *arg)
 {
 	struct redact_merge_thread_arg *rmta = arg;

module/zfs/dmu_send.c

@@ -1234,7 +1234,7 @@ redact_list_cb(redact_block_phys_t *rb, void *arg)
  * error code of the thread in case something goes wrong, and pushes the End of
  * Stream record when the traverse_dataset call has finished.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 send_traverse_thread(void *arg)
 {
 	struct send_thread_arg *st_arg = arg;
@@ -1324,7 +1324,7 @@ get_next_range(bqueue_t *bq, struct send_range *prev)
 	return (next);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 redact_list_thread(void *arg)
 {
 	struct redact_list_thread_arg *rlt_arg = arg;
@@ -1519,7 +1519,7 @@ find_next_range(struct send_range **ranges, bqueue_t **qs, uint64_t *out_mask)
  * data from the redact_list_thread and use that to determine which blocks
  * should be redacted.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 send_merge_thread(void *arg)
 {
 	struct send_merge_thread_arg *smt_arg = arg;
@@ -1744,7 +1744,7 @@ enqueue_range(struct send_reader_thread_arg *srta, bqueue_t *q, dnode_t *dn,
  * some indirect blocks can be discarded because they're not holes. Second,
  * it issues prefetches for the data we need to send.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 send_reader_thread(void *arg)
 {
 	struct send_reader_thread_arg *srta = arg;

module/zfs/mmp.c

@@ -187,7 +187,7 @@ uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS;
 uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS;
 
 static void *const mmp_tag = "mmp_write_uberblock";
-static _Noreturn void mmp_thread(void *arg);
+static __attribute__((noreturn)) void mmp_thread(void *arg);
 
 void
 mmp_init(spa_t *spa)
@@ -217,14 +217,13 @@ mmp_thread_enter(mmp_thread_t *mmp, callb_cpr_t *cpr)
 	mutex_enter(&mmp->mmp_thread_lock);
 }
 
-static _Noreturn void
+static void
 mmp_thread_exit(mmp_thread_t *mmp, kthread_t **mpp, callb_cpr_t *cpr)
 {
 	ASSERT(*mpp != NULL);
 	*mpp = NULL;
 	cv_broadcast(&mmp->mmp_thread_cv);
 	CALLB_CPR_EXIT(cpr);		/* drops &mmp->mmp_thread_lock */
-	thread_exit();
 }
 
 void
@@ -537,7 +536,7 @@ mmp_write_uberblock(spa_t *spa)
 	zio_nowait(zio);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 mmp_thread(void *arg)
 {
 	spa_t *spa = (spa_t *)arg;
@@ -698,6 +697,8 @@ mmp_thread(void *arg)
 
 	mmp->mmp_zio_root = NULL;
 	mmp_thread_exit(mmp, &mmp->mmp_thread, &cpr);
+
+	thread_exit();
 }
 
 /*

module/zfs/spa.c

@@ -8139,7 +8139,7 @@ spa_async_autoexpand(spa_t *spa, vdev_t *vd)
 	spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 spa_async_thread(void *arg)
 {
 	spa_t *spa = (spa_t *)arg;

module/zfs/txg.c

@@ -108,8 +108,8 @@
  * now transition to the syncing state.
  */
 
-static _Noreturn void txg_sync_thread(void *arg);
-static _Noreturn void txg_quiesce_thread(void *arg);
+static __attribute__((noreturn)) void txg_sync_thread(void *arg);
+static __attribute__((noreturn)) void txg_quiesce_thread(void *arg);
 
 int zfs_txg_timeout = 5;	/* max seconds worth of delta per txg */
 
@@ -514,7 +514,7 @@ txg_has_quiesced_to_sync(dsl_pool_t *dp)
 	return (tx->tx_quiesced_txg != 0);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 txg_sync_thread(void *arg)
 {
 	dsl_pool_t *dp = arg;
@@ -605,7 +605,7 @@ txg_sync_thread(void *arg)
 	}
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 txg_quiesce_thread(void *arg)
 {
 	dsl_pool_t *dp = arg;

module/zfs/vdev_initialize.c

@@ -488,7 +488,7 @@ vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size)
 	vdev_xlate_walk(vd, &logical_rs, vdev_initialize_xlate_range_add, arg);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 vdev_initialize_thread(void *arg)
 {
 	vdev_t *vd = arg;

module/zfs/vdev_rebuild.c

@@ -133,7 +133,7 @@ static int zfs_rebuild_scrub_enabled = 1;
 /*
  * For vdev_rebuild_initiate_sync() and vdev_rebuild_reset_sync().
  */
-static _Noreturn void vdev_rebuild_thread(void *arg);
+static __attribute__((noreturn)) void vdev_rebuild_thread(void *arg);
 
 /*
  * Clear the per-vdev rebuild bytes value for a vdev tree.
@@ -736,7 +736,7 @@ vdev_rebuild_load(vdev_t *vd)
  * Each scan thread is responsible for rebuilding a top-level vdev.  The
  * rebuild progress in tracked on-disk in VDEV_TOP_ZAP_VDEV_REBUILD_PHYS.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 vdev_rebuild_thread(void *arg)
 {
 	vdev_t *vd = arg;

module/zfs/vdev_removal.c

@@ -140,7 +140,7 @@ int zfs_removal_suspend_progress = 0;
 
 #define	VDEV_REMOVAL_ZAP_OBJS	"lzap"
 
-static _Noreturn void spa_vdev_remove_thread(void *arg);
+static __attribute__((noreturn)) void spa_vdev_remove_thread(void *arg);
 static int spa_vdev_remove_cancel_impl(spa_t *spa);
 
 static void
@@ -1589,7 +1589,7 @@ spa_remove_max_segment(spa_t *spa)
  * TXG have completed (see spa_txg_zio) and writes the new mappings to disk
  * (see vdev_mapping_sync()).
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 spa_vdev_remove_thread(void *arg)
 {
 	spa_t *spa = arg;

module/zfs/vdev_trim.c

@@ -834,7 +834,7 @@ vdev_trim_range_add(void *arg, uint64_t start, uint64_t size)
  * by its ms_allocatable.  While a metaslab is undergoing trimming it is
  * not eligible for new allocations.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 vdev_trim_thread(void *arg)
 {
 	vdev_t *vd = arg;
@@ -1175,7 +1175,7 @@ vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size)
  * N.B. This behavior is different from a manual TRIM where a thread
  * is created for each leaf vdev, instead of each top-level vdev.
  */
-static _Noreturn void
+static __attribute__((noreturn)) void
 vdev_autotrim_thread(void *arg)
 {
 	vdev_t *vd = arg;
@@ -1514,7 +1514,7 @@ vdev_autotrim_restart(spa_t *spa)
 		vdev_autotrim(spa);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 vdev_trim_l2arc_thread(void *arg)
 {
 	vdev_t		*vd = arg;

module/zfs/zthr.c

@@ -231,7 +231,7 @@ struct zthr {
 	const char	*zthr_name;
 };
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 zthr_procedure(void *arg)
 {
 	zthr_t *t = arg;

tests/zfs-tests/cmd/mkbusy/mkbusy.c

@@ -31,14 +31,14 @@
 #include <string.h>
 
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 usage(char *progname)
 {
 	(void) fprintf(stderr, "Usage: %s <dirname|filename>\n", progname);
 	exit(1);
 }
 
-static _Noreturn void
+static __attribute__((noreturn)) void
 fail(char *err)
 {
 	perror(err);

tests/zfs-tests/cmd/mkfile/mkfile.c

@@ -44,7 +44,13 @@
 
 #define	FILE_MODE	(S_ISVTX + S_IRUSR + S_IWUSR)
 
-static _Noreturn void usage(void);
+static __attribute__((noreturn)) void
+usage(void)
+{
+	(void) fprintf(stderr, gettext(
+	    "Usage: mkfile [-nv] <size>[g|k|b|m] <name1> [<name2>] ...\n"));
+	exit(1);
+}
 
 int
 main(int argc, char **argv)
@@ -272,10 +278,3 @@ main(int argc, char **argv)
 	}
 	return (errors);
 }
-
-static void usage()
-{
-	(void) fprintf(stderr, gettext(
-	    "Usage: mkfile [-nv] <size>[g|k|b|m] <name1> [<name2>] ...\n"));
-	exit(1);
-}