Cleanup: Make memory barrier definitions consistent across kernels

We inherited membar_consumer() and membar_producer() from OpenSolaris,
but we had replaced membar_consumer() with Linux's smp_rmb() in
zfs_ioctl.c. The FreeBSD SPL consequently implemented a shim for the
Linux-only smp_rmb().

We reinstate membar_consumer() in platform independent code and fix the
FreeBSD SPL to implement membar_consumer() in a way analogous to Linux.

Reviewed-by: Konstantin Belousov <kib@FreeBSD.org>
Reviewed-by: Mateusz Guzik <mjguzik@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Neal Gompa <ngompa@datto.com>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Signed-off-by: Richard Yao <richard.yao@alumni.stonybrook.edu>
Closes #13843

include/os/freebsd/linux/compiler.h

@@ -83,7 +83,6 @@
 #define	__printf(a, b)			__printflike(a, b)
 
 #define	barrier()			__asm__ __volatile__("": : :"memory")
-#define	smp_rmb()		rmb()
 #define	___PASTE(a, b) a##b
 #define	__PASTE(a, b) ___PASTE(a, b)
 

include/os/freebsd/spl/sys/atomic.h

@@ -57,7 +57,8 @@ extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp,
     uint64_t newval);
 #endif
 
-#define	membar_producer	atomic_thread_fence_rel
+#define	membar_consumer()		atomic_thread_fence_acq()
+#define	membar_producer()		atomic_thread_fence_rel()
 
 static __inline uint32_t
 atomic_add_32_nv(volatile uint32_t *target, int32_t delta)

include/os/linux/spl/sys/vmsystm.h

@@ -44,7 +44,9 @@
 #define	zfs_totalhigh_pages	totalhigh_pages
 #endif
 
+#define	membar_consumer()		smp_rmb()
 #define	membar_producer()		smp_wmb()
+
 #define	physmem				zfs_totalram_pages
 
 #define	xcopyin(from, to, size)		copy_from_user(to, from, size)

module/zfs/zfs_ioctl.c

@@ -7482,7 +7482,7 @@ zfsdev_get_state(minor_t minor, enum zfsdev_state_type which)
 
 	for (zs = zfsdev_state_list; zs != NULL; zs = zs->zs_next) {
 		if (zs->zs_minor == minor) {
-			smp_rmb();
+			membar_consumer();
 			switch (which) {
 			case ZST_ONEXIT:
 				return (zs->zs_onexit);