From c25b8f99f8dcbe898b81728e6a9dab107df4fc0b Mon Sep 17 00:00:00 2001
From: Brian Behlendorf <behlendorf1@llnl.gov>
Date: Fri, 11 Aug 2017 08:51:44 -0700
Subject: [PATCH] Simplify threads, mutexs, cvs and rwlocks

* Simplify threads, mutexs, cvs and rwlocks

* Update the zk_thread_create() function to use the same trick
  as Illumos.  Specifically, cast the new pthread_t to a void
  pointer and return that as the kthread_t *.  This avoids the
  issues associated with managing a wrapper structure and is
  safe as long as the callers never attempt to dereference it.

* Update all function prototypes passed to pthread_create() to
  match the expected prototype.  We were getting away this with
  before since the function were explicitly cast.

* Replaced direct zk_thread_create() calls with thread_create()
  for code consistency.  All consumers of libzpool now use the
  proper wrappers.

* The mutex_held() calls were converted to MUTEX_HELD().

* Removed all mutex_owner() calls and retired the interface.
  Instead use MUTEX_HELD() which provides the same information
  and allows the implementation details to be hidden.  In this
  case the use of the pthread_equals() function.

* The kthread_t, kmutex_t, krwlock_t, and krwlock_t types had
  any non essential fields removed.  In the case of kthread_t
  and kcondvar_t they could be directly typedef'd to pthread_t
  and pthread_cond_t respectively.

* Removed all extra ASSERTS from the thread, mutex, rwlock, and
  cv wrapper functions.  In practice, pthreads already provides
  the vast majority of checks as long as we check the return
  code.  Removing this code from our wrappers help readability.

* Added TS_JOINABLE state flag to pass to request a joinable rather
  than detached thread.  This isn't a standard thread_create() state
  but it's the least invasive way to pass this information and is
  only used by ztest.

TEST_ZTEST_TIMEOUT=3600

Chunwei Chen <tuxoko@gmail.com>
Reviewed-by: Tom Caputi <tcaputi@datto.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #4547
Closes #5503
Closes #5523
Closes #6377
Closes #6495
---
 cmd/raidz_test/raidz_test.c |   6 +-
 cmd/ztest/ztest.c           |  46 +++---
 include/sys/zfs_context.h   | 122 ++++++---------
 lib/libzpool/kernel.c       | 292 ++++++++----------------------------
 module/zfs/arc.c            |   4 +-
 module/zfs/dbuf.c           |   2 +-
 module/zfs/mmp.c            |   5 +-
 module/zfs/spa.c            |  10 +-
 module/zfs/txg.c            |  10 +-
 module/zfs/zfs_sa.c         |   2 +-
 10 files changed, 143 insertions(+), 356 deletions(-)

diff --git a/cmd/raidz_test/raidz_test.c b/cmd/raidz_test/raidz_test.c
index a2b7e6cbfc..a05070399c 100644
--- a/cmd/raidz_test/raidz_test.c
+++ b/cmd/raidz_test/raidz_test.c
@@ -702,10 +702,8 @@ run_sweep(void)
 		opts->rto_dsize = size_v[s];
 		opts->rto_v = 0; /* be quiet */
 
-		VERIFY3P(zk_thread_create(NULL, 0,
-		    (thread_func_t)sweep_thread,
-		    (void *) opts, 0, NULL, TS_RUN, 0,
-		    PTHREAD_CREATE_JOINABLE), !=, NULL);
+		VERIFY3P(thread_create(NULL, 0, sweep_thread, (void *) opts,
+		    0, NULL, TS_RUN, defclsyspri), !=, NULL);
 	}
 
 exit:
diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c
index 788e6aa6ac..277782db01 100644
--- a/cmd/ztest/ztest.c
+++ b/cmd/ztest/ztest.c
@@ -2218,7 +2218,7 @@ ztest_lookup(ztest_ds_t *zd, ztest_od_t *od, int count)
 	int error;
 	int i;
 
-	ASSERT(mutex_held(&zd->zd_dirobj_lock));
+	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
 
 	for (i = 0; i < count; i++, od++) {
 		od->od_object = 0;
@@ -2259,7 +2259,7 @@ ztest_create(ztest_ds_t *zd, ztest_od_t *od, int count)
 	int missing = 0;
 	int i;
 
-	ASSERT(mutex_held(&zd->zd_dirobj_lock));
+	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
 
 	for (i = 0; i < count; i++, od++) {
 		if (missing) {
@@ -2305,7 +2305,7 @@ ztest_remove(ztest_ds_t *zd, ztest_od_t *od, int count)
 	int error;
 	int i;
 
-	ASSERT(mutex_held(&zd->zd_dirobj_lock));
+	ASSERT(MUTEX_HELD(&zd->zd_dirobj_lock));
 
 	od += count - 1;
 
@@ -6081,7 +6081,7 @@ ztest_resume(spa_t *spa)
 	(void) zio_resume(spa);
 }
 
-static void *
+static void
 ztest_resume_thread(void *arg)
 {
 	spa_t *spa = arg;
@@ -6105,8 +6105,6 @@ ztest_resume_thread(void *arg)
 	}
 
 	thread_exit();
-
-	return (NULL);
 }
 
 #define	GRACE	300
@@ -6140,7 +6138,7 @@ ztest_execute(int test, ztest_info_t *zi, uint64_t id)
 		    (double)functime / NANOSEC, zi->zi_funcname);
 }
 
-static void *
+static void
 ztest_thread(void *arg)
 {
 	int rand;
@@ -6180,8 +6178,6 @@ ztest_thread(void *arg)
 	}
 
 	thread_exit();
-
-	return (NULL);
 }
 
 static void
@@ -6309,10 +6305,10 @@ ztest_dataset_close(int d)
 static void
 ztest_run(ztest_shared_t *zs)
 {
-	kt_did_t *tid;
 	spa_t *spa;
 	objset_t *os;
 	kthread_t *resume_thread;
+	kthread_t **run_threads;
 	uint64_t object;
 	int error;
 	int t, d;
@@ -6373,9 +6369,8 @@ ztest_run(ztest_shared_t *zs)
 	/*
 	 * Create a thread to periodically resume suspended I/O.
 	 */
-	VERIFY3P((resume_thread = zk_thread_create(NULL, 0,
-	    (thread_func_t)ztest_resume_thread, spa, 0, NULL, TS_RUN, 0,
-	    PTHREAD_CREATE_JOINABLE)), !=, NULL);
+	resume_thread = thread_create(NULL, 0, ztest_resume_thread,
+	    spa, 0, NULL, TS_RUN | TS_JOINABLE, defclsyspri);
 
 #if 0
 	/*
@@ -6409,7 +6404,7 @@ ztest_run(ztest_shared_t *zs)
 	}
 	zs->zs_enospc_count = 0;
 
-	tid = umem_zalloc(ztest_opts.zo_threads * sizeof (kt_did_t),
+	run_threads = umem_zalloc(ztest_opts.zo_threads * sizeof (kthread_t *),
 	    UMEM_NOFAIL);
 
 	if (ztest_opts.zo_verbose >= 4)
@@ -6419,20 +6414,15 @@ ztest_run(ztest_shared_t *zs)
 	 * Kick off all the tests that run in parallel.
 	 */
 	for (t = 0; t < ztest_opts.zo_threads; t++) {
-		kthread_t *thread;
-
-		if (t < ztest_opts.zo_datasets &&
-		    ztest_dataset_open(t) != 0) {
-			umem_free(tid,
-			    ztest_opts.zo_threads * sizeof (kt_did_t));
+		if (t < ztest_opts.zo_datasets && ztest_dataset_open(t) != 0) {
+			umem_free(run_threads, ztest_opts.zo_threads *
+			    sizeof (kthread_t *));
 			return;
 		}
 
-		VERIFY3P(thread = zk_thread_create(NULL, 0,
-		    (thread_func_t)ztest_thread,
-		    (void *)(uintptr_t)t, 0, NULL, TS_RUN, 0,
-		    PTHREAD_CREATE_JOINABLE), !=, NULL);
-		tid[t] = thread->t_tid;
+		run_threads[t] = thread_create(NULL, 0, ztest_thread,
+		    (void *)(uintptr_t)t, 0, NULL, TS_RUN | TS_JOINABLE,
+		    defclsyspri);
 	}
 
 	/*
@@ -6440,7 +6430,7 @@ ztest_run(ztest_shared_t *zs)
 	 * so we don't close datasets while threads are still using them.
 	 */
 	for (t = ztest_opts.zo_threads - 1; t >= 0; t--) {
-		thread_join(tid[t]);
+		VERIFY0(thread_join(run_threads[t]));
 		if (t < ztest_opts.zo_datasets)
 			ztest_dataset_close(t);
 	}
@@ -6450,11 +6440,11 @@ ztest_run(ztest_shared_t *zs)
 	zs->zs_alloc = metaslab_class_get_alloc(spa_normal_class(spa));
 	zs->zs_space = metaslab_class_get_space(spa_normal_class(spa));
 
-	umem_free(tid, ztest_opts.zo_threads * sizeof (kt_did_t));
+	umem_free(run_threads, ztest_opts.zo_threads * sizeof (kthread_t *));
 
 	/* Kill the resume thread */
 	ztest_exiting = B_TRUE;
-	thread_join(resume_thread->t_tid);
+	VERIFY0(thread_join(resume_thread));
 	ztest_resume(spa);
 
 	/*
diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h
index 21a9a8481f..90e7954a18 100644
--- a/include/sys/zfs_context.h
+++ b/include/sys/zfs_context.h
@@ -207,15 +207,23 @@ extern int aok;
 	(unsigned long)i)
 
 /*
- * Threads.  TS_STACK_MIN is dictated by the minimum allowed pthread stack
- * size.  While TS_STACK_MAX is somewhat arbitrary, it was selected to be
- * large enough for the expected stack depth while small enough to avoid
- * exhausting address space with high thread counts.
+ * Threads.
  */
-#define	TS_MAGIC		0x72f158ab4261e538ull
-#define	TS_RUN			0x00000002
-#define	TS_STACK_MIN		MAX(PTHREAD_STACK_MIN, 32768)
-#define	TS_STACK_MAX		(256 * 1024)
+typedef pthread_t	kthread_t;
+
+#define	TS_RUN		0x00000002
+#define	TS_JOINABLE	0x00000004
+
+#define	curthread	((void *)(uintptr_t)pthread_self())
+#define	kpreempt(x)	yield()
+#define	getcomm()	"unknown"
+
+#define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
+	zk_thread_create(func, arg, stksize, state)
+#define	thread_exit()	pthread_exit(NULL)
+#define	thread_join(t)	pthread_join((pthread_t)(t), NULL)
+
+#define	newproc(f, a, cid, pri, ctp, pid)	(ENOSYS)
 
 /* in libzpool, p0 exists only to have its address taken */
 typedef struct proc {
@@ -225,100 +233,55 @@ typedef struct proc {
 extern struct proc p0;
 #define	curproc		(&p0)
 
-typedef void (*thread_func_t)(void *);
-typedef void (*thread_func_arg_t)(void *);
-typedef pthread_t kt_did_t;
-
-#define	kpreempt(x)	((void)0)
-
-typedef struct kthread {
-	kt_did_t	t_tid;
-	thread_func_t	t_func;
-	void *		t_arg;
-	pri_t		t_pri;
-} kthread_t;
-
-#define	curthread			zk_thread_current()
-#define	getcomm()			"unknown"
-#define	thread_exit			zk_thread_exit
-#define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
-	zk_thread_create(stk, stksize, (thread_func_t)func, arg,	\
-	    len, NULL, state, pri, PTHREAD_CREATE_DETACHED)
-#define	thread_join(t)			zk_thread_join(t)
-#define	newproc(f, a, cid, pri, ctp, pid)	(ENOSYS)
-
-extern kthread_t *zk_thread_current(void);
-extern void zk_thread_exit(void);
-extern kthread_t *zk_thread_create(caddr_t stk, size_t  stksize,
-	thread_func_t func, void *arg, uint64_t len,
-	proc_t *pp, int state, pri_t pri, int detachstate);
-extern void zk_thread_join(kt_did_t tid);
-
-#define	kpreempt_disable()	((void)0)
-#define	kpreempt_enable()	((void)0)
-
 #define	PS_NONE		-1
 
+extern kthread_t *zk_thread_create(void (*func)(void *), void *arg,
+    size_t stksize, int state);
+
 #define	issig(why)	(FALSE)
 #define	ISSIG(thr, why)	(FALSE)
 
+#define	kpreempt_disable()	((void)0)
+#define	kpreempt_enable()	((void)0)
+
 /*
  * Mutexes
  */
-#define	MTX_MAGIC	0x9522f51362a6e326ull
-#define	MTX_INIT	((void *)NULL)
-#define	MTX_DEST	((void *)-1UL)
-
 typedef struct kmutex {
-	void		*m_owner;
-	uint64_t	m_magic;
-	pthread_mutex_t	m_lock;
+	pthread_mutex_t		m_lock;
+	pthread_t		m_owner;
 } kmutex_t;
 
-#define	MUTEX_DEFAULT	0
-#define	MUTEX_NOLOCKDEP	MUTEX_DEFAULT
-#define	MUTEX_HELD(m)	((m)->m_owner == curthread)
-#define	MUTEX_NOT_HELD(m) (!MUTEX_HELD(m))
+#define	MUTEX_DEFAULT		0
+#define	MUTEX_NOLOCKDEP		MUTEX_DEFAULT
+#define	MUTEX_HELD(mp)		pthread_equal((mp)->m_owner, pthread_self())
+#define	MUTEX_NOT_HELD(mp)	!MUTEX_HELD(mp)
 
 extern void mutex_init(kmutex_t *mp, char *name, int type, void *cookie);
 extern void mutex_destroy(kmutex_t *mp);
 extern void mutex_enter(kmutex_t *mp);
 extern void mutex_exit(kmutex_t *mp);
 extern int mutex_tryenter(kmutex_t *mp);
-extern void *mutex_owner(kmutex_t *mp);
-extern int mutex_held(kmutex_t *mp);
 
 /*
  * RW locks
  */
-#define	RW_MAGIC	0x4d31fb123648e78aull
-#define	RW_INIT		((void *)NULL)
-#define	RW_DEST		((void *)-1UL)
-
 typedef struct krwlock {
-	void			*rw_owner;
-	void			*rw_wr_owner;
-	uint64_t		rw_magic;
 	pthread_rwlock_t	rw_lock;
+	pthread_t		rw_owner;
 	uint_t			rw_readers;
 } krwlock_t;
 
 typedef int krw_t;
 
-#define	RW_READER	0
-#define	RW_WRITER	1
-#define	RW_DEFAULT	RW_READER
-#define	RW_NOLOCKDEP	RW_READER
+#define	RW_READER		0
+#define	RW_WRITER		1
+#define	RW_DEFAULT		RW_READER
+#define	RW_NOLOCKDEP		RW_READER
 
-#define	RW_READ_HELD(x)		((x)->rw_readers > 0)
-#define	RW_WRITE_HELD(x)	((x)->rw_wr_owner == curthread)
-#define	RW_LOCK_HELD(x)		(RW_READ_HELD(x) || RW_WRITE_HELD(x))
-
-#undef RW_LOCK_HELD
-#define	RW_LOCK_HELD(x)		(RW_READ_HELD(x) || RW_WRITE_HELD(x))
-
-#undef RW_LOCK_HELD
-#define	RW_LOCK_HELD(x)		(RW_READ_HELD(x) || RW_WRITE_HELD(x))
+#define	RW_READ_HELD(rw)	((rw)->rw_readers > 0)
+#define	RW_WRITE_HELD(rw)	pthread_equal((rw)->rw_owner, pthread_self())
+#define	RW_LOCK_HELD(rw)	(RW_READ_HELD(rw) || RW_WRITE_HELD(rw))
 
 extern void rw_init(krwlock_t *rwlp, char *name, int type, void *arg);
 extern void rw_destroy(krwlock_t *rwlp);
@@ -328,6 +291,9 @@ extern int rw_tryupgrade(krwlock_t *rwlp);
 extern void rw_exit(krwlock_t *rwlp);
 #define	rw_downgrade(rwlp) do { } while (0)
 
+/*
+ * Credentials
+ */
 extern uid_t crgetuid(cred_t *cr);
 extern uid_t crgetruid(cred_t *cr);
 extern gid_t crgetgid(cred_t *cr);
@@ -337,14 +303,9 @@ extern gid_t *crgetgroups(cred_t *cr);
 /*
  * Condition variables
  */
-#define	CV_MAGIC	0xd31ea9a83b1b30c4ull
+typedef pthread_cond_t		kcondvar_t;
 
-typedef struct kcondvar {
-	uint64_t		cv_magic;
-	pthread_cond_t		cv;
-} kcondvar_t;
-
-#define	CV_DEFAULT	0
+#define	CV_DEFAULT		0
 #define	CALLOUT_FLAG_ABSOLUTE	0x2
 
 extern void cv_init(kcondvar_t *cv, char *name, int type, void *arg);
@@ -355,6 +316,7 @@ extern clock_t cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
     hrtime_t res, int flag);
 extern void cv_signal(kcondvar_t *cv);
 extern void cv_broadcast(kcondvar_t *cv);
+
 #define	cv_timedwait_sig(cv, mp, at)		cv_timedwait(cv, mp, at)
 #define	cv_wait_sig(cv, mp)			cv_wait(cv, mp)
 #define	cv_wait_io(cv, mp)			cv_wait(cv, mp)
diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c
index 3fc7337e8e..c4b600d038 100644
--- a/lib/libzpool/kernel.c
+++ b/lib/libzpool/kernel.c
@@ -64,104 +64,29 @@ struct proc p0;
  * =========================================================================
  * threads
  * =========================================================================
+ *
+ * TS_STACK_MIN is dictated by the minimum allowed pthread stack size.  While
+ * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for
+ * the expected stack depth while small enough to avoid exhausting address
+ * space with high thread counts.
  */
+#define	TS_STACK_MIN	MAX(PTHREAD_STACK_MIN, 32768)
+#define	TS_STACK_MAX	(256 * 1024)
 
-pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
-pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
-pthread_key_t kthread_key;
-int kthread_nr = 0;
-
-static void
-thread_init(void)
-{
-	kthread_t *kt;
-
-	VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
-
-	/* Create entry for primary kthread */
-	kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
-	kt->t_tid = pthread_self();
-	kt->t_func = NULL;
-
-	VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
-
-	/* Only the main thread should be running at the moment */
-	ASSERT3S(kthread_nr, ==, 0);
-	kthread_nr = 1;
-}
-
-static void
-thread_fini(void)
-{
-	kthread_t *kt = curthread;
-
-	ASSERT(pthread_equal(kt->t_tid, pthread_self()));
-	ASSERT3P(kt->t_func, ==, NULL);
-
-	umem_free(kt, sizeof (kthread_t));
-
-	/* Wait for all threads to exit via thread_exit() */
-	VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
-
-	kthread_nr--; /* Main thread is exiting */
-
-	while (kthread_nr > 0)
-		VERIFY0(pthread_cond_wait(&kthread_cond, &kthread_lock));
-
-	ASSERT3S(kthread_nr, ==, 0);
-	VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
-
-	VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
-}
-
+/*ARGSUSED*/
 kthread_t *
-zk_thread_current(void)
+zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state)
 {
-	kthread_t *kt = pthread_getspecific(kthread_key);
-
-	ASSERT3P(kt, !=, NULL);
-
-	return (kt);
-}
-
-void *
-zk_thread_helper(void *arg)
-{
-	kthread_t *kt = (kthread_t *)arg;
-
-	VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
-
-	VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
-	kthread_nr++;
-	VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
-	(void) setpriority(PRIO_PROCESS, 0, kt->t_pri);
-
-	kt->t_tid = pthread_self();
-	((thread_func_arg_t)kt->t_func)(kt->t_arg);
-
-	/* Unreachable, thread must exit with thread_exit() */
-	abort();
-
-	return (NULL);
-}
-
-kthread_t *
-zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
-    uint64_t len, proc_t *pp, int state, pri_t pri, int detachstate)
-{
-	kthread_t *kt;
 	pthread_attr_t attr;
+	pthread_t tid;
 	char *stkstr;
-
-	ASSERT0(state & ~TS_RUN);
-	ASSERT0(len);
-
-	kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
-	kt->t_func = func;
-	kt->t_arg = arg;
-	kt->t_pri = pri;
+	int detachstate = PTHREAD_CREATE_DETACHED;
 
 	VERIFY0(pthread_attr_init(&attr));
+
+	if (state & TS_JOINABLE)
+		detachstate = PTHREAD_CREATE_JOINABLE;
+
 	VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 
 	/*
@@ -183,6 +108,7 @@ zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 
 	VERIFY3S(stksize, >, 0);
 	stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
+
 	/*
 	 * If this ever fails, it may be because the stack size is not a
 	 * multiple of system page size.
@@ -190,36 +116,10 @@ zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 	VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 	VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 
-	VERIFY0(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt));
+	VERIFY0(pthread_create(&tid, &attr, (void *(*)(void *))func, arg));
 	VERIFY0(pthread_attr_destroy(&attr));
 
-	return (kt);
-}
-
-void
-zk_thread_exit(void)
-{
-	kthread_t *kt = curthread;
-
-	ASSERT(pthread_equal(kt->t_tid, pthread_self()));
-
-	umem_free(kt, sizeof (kthread_t));
-
-	VERIFY0(pthread_mutex_lock(&kthread_lock));
-	kthread_nr--;
-	VERIFY0(pthread_mutex_unlock(&kthread_lock));
-
-	VERIFY0(pthread_cond_broadcast(&kthread_cond));
-	pthread_exit((void *)TS_MAGIC);
-}
-
-void
-zk_thread_join(kt_did_t tid)
-{
-	void *ret;
-
-	pthread_join((pthread_t)tid, &ret);
-	VERIFY3P(ret, ==, (void *)TS_MAGIC);
+	return ((void *)(uintptr_t)tid);
 }
 
 /*
@@ -291,46 +191,34 @@ kstat_set_raw_ops(kstat_t *ksp,
 void
 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 {
-	ASSERT3S(type, ==, MUTEX_DEFAULT);
-	ASSERT3P(cookie, ==, NULL);
-	mp->m_owner = MTX_INIT;
-	mp->m_magic = MTX_MAGIC;
-	VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
+	VERIFY0(pthread_mutex_init(&mp->m_lock, NULL));
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
 }
 
 void
 mutex_destroy(kmutex_t *mp)
 {
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mp->m_owner, ==, MTX_INIT);
-	ASSERT0(pthread_mutex_destroy(&(mp)->m_lock));
-	mp->m_owner = MTX_DEST;
-	mp->m_magic = 0;
+	VERIFY0(pthread_mutex_destroy(&mp->m_lock));
 }
 
 void
 mutex_enter(kmutex_t *mp)
 {
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mp->m_owner, !=, MTX_DEST);
-	ASSERT3P(mp->m_owner, !=, curthread);
-	VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
-	ASSERT3P(mp->m_owner, ==, MTX_INIT);
-	mp->m_owner = curthread;
+	VERIFY0(pthread_mutex_lock(&mp->m_lock));
+	mp->m_owner = pthread_self();
 }
 
 int
 mutex_tryenter(kmutex_t *mp)
 {
-	int err;
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mp->m_owner, !=, MTX_DEST);
-	if (0 == (err = pthread_mutex_trylock(&mp->m_lock))) {
-		ASSERT3P(mp->m_owner, ==, MTX_INIT);
-		mp->m_owner = curthread;
+	int error;
+
+	error = pthread_mutex_trylock(&mp->m_lock);
+	if (error == 0) {
+		mp->m_owner = pthread_self();
 		return (1);
 	} else {
-		VERIFY3S(err, ==, EBUSY);
+		VERIFY3S(error, ==, EBUSY);
 		return (0);
 	}
 }
@@ -338,23 +226,8 @@ mutex_tryenter(kmutex_t *mp)
 void
 mutex_exit(kmutex_t *mp)
 {
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	ASSERT3P(mutex_owner(mp), ==, curthread);
-	mp->m_owner = MTX_INIT;
-	VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
-}
-
-void *
-mutex_owner(kmutex_t *mp)
-{
-	ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
-	return (mp->m_owner);
-}
-
-int
-mutex_held(kmutex_t *mp)
-{
-	return (mp->m_owner == curthread);
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	VERIFY0(pthread_mutex_unlock(&mp->m_lock));
 }
 
 /*
@@ -366,89 +239,60 @@ mutex_held(kmutex_t *mp)
 void
 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 {
-	ASSERT3S(type, ==, RW_DEFAULT);
-	ASSERT3P(arg, ==, NULL);
-	VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
-	rwlp->rw_owner = RW_INIT;
-	rwlp->rw_wr_owner = RW_INIT;
+	VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL));
 	rwlp->rw_readers = 0;
-	rwlp->rw_magic = RW_MAGIC;
+	rwlp->rw_owner = 0;
 }
 
 void
 rw_destroy(krwlock_t *rwlp)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-	ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
-	VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
-	rwlp->rw_magic = 0;
+	VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock));
 }
 
 void
 rw_enter(krwlock_t *rwlp, krw_t rw)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-	ASSERT3P(rwlp->rw_owner, !=, curthread);
-	ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
-
 	if (rw == RW_READER) {
-		VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
-		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
-
+		VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock));
 		atomic_inc_uint(&rwlp->rw_readers);
 	} else {
-		VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
-		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
-		ASSERT3U(rwlp->rw_readers, ==, 0);
-
-		rwlp->rw_wr_owner = curthread;
+		VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock));
+		rwlp->rw_owner = pthread_self();
 	}
-
-	rwlp->rw_owner = curthread;
 }
 
 void
 rw_exit(krwlock_t *rwlp)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-	ASSERT(RW_LOCK_HELD(rwlp));
-
 	if (RW_READ_HELD(rwlp))
 		atomic_dec_uint(&rwlp->rw_readers);
 	else
-		rwlp->rw_wr_owner = RW_INIT;
+		rwlp->rw_owner = 0;
 
-	rwlp->rw_owner = RW_INIT;
-	VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
+	VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock));
 }
 
 int
 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 {
-	int rv;
-
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
+	int error;
 
 	if (rw == RW_READER)
-		rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
+		error = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 	else
-		rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
-
-	if (rv == 0) {
-		ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
+		error = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 
+	if (error == 0) {
 		if (rw == RW_READER)
 			atomic_inc_uint(&rwlp->rw_readers);
-		else {
-			ASSERT3U(rwlp->rw_readers, ==, 0);
-			rwlp->rw_wr_owner = curthread;
-		}
+		else
+			rwlp->rw_owner = pthread_self();
 
-		rwlp->rw_owner = curthread;
 		return (1);
 	}
 
-	VERIFY3S(rv, ==, EBUSY);
+	VERIFY3S(error, ==, EBUSY);
 
 	return (0);
 }
@@ -456,8 +300,6 @@ rw_tryenter(krwlock_t *rwlp, krw_t rw)
 int
 rw_tryupgrade(krwlock_t *rwlp)
 {
-	ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
-
 	return (0);
 }
 
@@ -470,27 +312,21 @@ rw_tryupgrade(krwlock_t *rwlp)
 void
 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 {
-	ASSERT3S(type, ==, CV_DEFAULT);
-	cv->cv_magic = CV_MAGIC;
-	VERIFY0(pthread_cond_init(&cv->cv, NULL));
+	VERIFY0(pthread_cond_init(cv, NULL));
 }
 
 void
 cv_destroy(kcondvar_t *cv)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	VERIFY0(pthread_cond_destroy(&cv->cv));
-	cv->cv_magic = 0;
+	VERIFY0(pthread_cond_destroy(cv));
 }
 
 void
 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	ASSERT3P(mutex_owner(mp), ==, curthread);
-	mp->m_owner = MTX_INIT;
-	VERIFY0(pthread_cond_wait(&cv->cv, &mp->m_lock));
-	mp->m_owner = curthread;
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	VERIFY0(pthread_cond_wait(cv, &mp->m_lock));
+	mp->m_owner = pthread_self();
 }
 
 clock_t
@@ -501,8 +337,6 @@ cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 	timestruc_t ts;
 	clock_t delta;
 
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-
 	delta = abstime - ddi_get_lbolt();
 	if (delta <= 0)
 		return (-1);
@@ -516,10 +350,9 @@ cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 		ts.tv_nsec -= NANOSEC;
 	}
 
-	ASSERT3P(mutex_owner(mp), ==, curthread);
-	mp->m_owner = MTX_INIT;
-	error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
-	mp->m_owner = curthread;
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
+	mp->m_owner = pthread_self();
 
 	if (error == ETIMEDOUT)
 		return (-1);
@@ -548,7 +381,7 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 	if (delta <= 0)
 		return (-1);
 
-	VERIFY(gettimeofday(&tv, NULL) == 0);
+	VERIFY0(gettimeofday(&tv, NULL));
 
 	ts.tv_sec = tv.tv_sec + delta / NANOSEC;
 	ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC);
@@ -557,10 +390,9 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 		ts.tv_nsec -= NANOSEC;
 	}
 
-	ASSERT(mutex_owner(mp) == curthread);
-	mp->m_owner = MTX_INIT;
-	error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
-	mp->m_owner = curthread;
+	memset(&mp->m_owner, 0, sizeof (pthread_t));
+	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
+	mp->m_owner = pthread_self();
 
 	if (error == ETIMEDOUT)
 		return (-1);
@@ -573,15 +405,13 @@ cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 void
 cv_signal(kcondvar_t *cv)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	VERIFY0(pthread_cond_signal(&cv->cv));
+	VERIFY0(pthread_cond_signal(cv));
 }
 
 void
 cv_broadcast(kcondvar_t *cv)
 {
-	ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
-	VERIFY0(pthread_cond_broadcast(&cv->cv));
+	VERIFY0(pthread_cond_broadcast(cv));
 }
 
 /*
@@ -1188,7 +1018,6 @@ kernel_init(int mode)
 
 	VERIFY0(uname(&hw_utsname));
 
-	thread_init();
 	system_taskq_init();
 	icp_init();
 
@@ -1207,7 +1036,6 @@ kernel_fini(void)
 
 	icp_fini();
 	system_taskq_fini();
-	thread_fini();
 
 	random_fini();
 }
diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 481c381893..157a28d4b0 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -4214,7 +4214,7 @@ arc_kmem_reap_now(void)
  * using mutex_tryenter() from arc_reclaim_thread().
  */
 static void
-arc_reclaim_thread(void)
+arc_reclaim_thread(void *unused)
 {
 	fstrans_cookie_t	cookie = spl_fstrans_mark();
 	hrtime_t		growtime = 0;
@@ -7515,7 +7515,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
  * heart of the L2ARC.
  */
 static void
-l2arc_feed_thread(void)
+l2arc_feed_thread(void *unused)
 {
 	callb_cpr_t cpr;
 	l2arc_dev_t *dev;
diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c
index dc2c00495b..625e06701c 100644
--- a/module/zfs/dbuf.c
+++ b/module/zfs/dbuf.c
@@ -531,7 +531,7 @@ dbuf_evict_one(void)
  * out of the cache it is destroyed and becomes eligible for arc eviction.
  */
 static void
-dbuf_evict_thread(void)
+dbuf_evict_thread(void *unused)
 {
 	callb_cpr_t cpr;
 
diff --git a/module/zfs/mmp.c b/module/zfs/mmp.c
index 00478a39f2..a4771b677a 100644
--- a/module/zfs/mmp.c
+++ b/module/zfs/mmp.c
@@ -123,7 +123,7 @@ uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS;
  */
 uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS;
 
-static void mmp_thread(spa_t *spa);
+static void mmp_thread(void *arg);
 
 void
 mmp_init(spa_t *spa)
@@ -364,8 +364,9 @@ mmp_write_uberblock(spa_t *spa)
 }
 
 static void
-mmp_thread(spa_t *spa)
+mmp_thread(void *arg)
 {
+	spa_t *spa = (spa_t *)arg;
 	mmp_thread_t *mmp = &spa->spa_mmp;
 	boolean_t last_spa_suspended = spa_suspended(spa);
 	boolean_t last_spa_multihost = spa_multihost(spa);
diff --git a/module/zfs/spa.c b/module/zfs/spa.c
index f1f1444f1c..cb86c62008 100644
--- a/module/zfs/spa.c
+++ b/module/zfs/spa.c
@@ -1028,6 +1028,11 @@ spa_create_zio_taskqs(spa_t *spa)
 	}
 }
 
+/*
+ * Disabled until spa_thread() can be adapted for Linux.
+ */
+#undef HAVE_SPA_THREAD
+
 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD)
 static void
 spa_thread(void *arg)
@@ -3415,7 +3420,7 @@ spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy,
 	 * up calling spa_open() again.  The real fix is to figure out how to
 	 * avoid dsl_dir_open() calling this in the first place.
 	 */
-	if (mutex_owner(&spa_namespace_lock) != curthread) {
+	if (MUTEX_NOT_HELD(&spa_namespace_lock)) {
 		mutex_enter(&spa_namespace_lock);
 		locked = B_TRUE;
 	}
@@ -6068,8 +6073,9 @@ spa_async_autoexpand(spa_t *spa, vdev_t *vd)
 }
 
 static void
-spa_async_thread(spa_t *spa)
+spa_async_thread(void *arg)
 {
+	spa_t *spa = (spa_t *)arg;
 	int tasks, i;
 
 	ASSERT(spa->spa_sync_on);
diff --git a/module/zfs/txg.c b/module/zfs/txg.c
index 65bd7f93ac..8b1ec9c058 100644
--- a/module/zfs/txg.c
+++ b/module/zfs/txg.c
@@ -108,8 +108,8 @@
  * now transition to the syncing state.
  */
 
-static void txg_sync_thread(dsl_pool_t *dp);
-static void txg_quiesce_thread(dsl_pool_t *dp);
+static void txg_sync_thread(void *dp);
+static void txg_quiesce_thread(void *dp);
 
 int zfs_txg_timeout = 5;	/* max seconds worth of delta per txg */
 
@@ -477,8 +477,9 @@ txg_wait_callbacks(dsl_pool_t *dp)
 }
 
 static void
-txg_sync_thread(dsl_pool_t *dp)
+txg_sync_thread(void *arg)
 {
+	dsl_pool_t *dp = (dsl_pool_t *)arg;
 	spa_t *spa = dp->dp_spa;
 	tx_state_t *tx = &dp->dp_tx;
 	callb_cpr_t cpr;
@@ -561,8 +562,9 @@ txg_sync_thread(dsl_pool_t *dp)
 }
 
 static void
-txg_quiesce_thread(dsl_pool_t *dp)
+txg_quiesce_thread(void *arg)
 {
+	dsl_pool_t *dp = (dsl_pool_t *)arg;
 	tx_state_t *tx = &dp->dp_tx;
 	callb_cpr_t cpr;
 
diff --git a/module/zfs/zfs_sa.c b/module/zfs/zfs_sa.c
index 13e99c0587..08e881cc37 100644
--- a/module/zfs/zfs_sa.c
+++ b/module/zfs/zfs_sa.c
@@ -300,7 +300,7 @@ zfs_sa_upgrade(sa_handle_t *hdl, dmu_tx_t *tx)
 	 * Otherwise, we know we are doing the
 	 * sa_update() that caused us to enter this function.
 	 */
-	if (mutex_owner(&zp->z_lock) != curthread) {
+	if (MUTEX_NOT_HELD(&zp->z_lock)) {
 		if (mutex_tryenter(&zp->z_lock) == 0)
 			return;
 		else