From 0d6186cff3f917b6f3677cf927dda24466dbf1d5 Mon Sep 17 00:00:00 2001 From: "Adam D. Moss" Date: Thu, 11 Mar 2021 17:16:09 -0800 Subject: [PATCH] Microoptimizations for VERIFY() and friends Add branch hints and constify the intermediate evaluations of left/right params in VERIFY3*(). Reviewed-by: Ryan Moeller Reviewed-by: Brian Behlendorf Signed-off-by: Adam Moss Closes #11708 --- include/os/freebsd/spl/sys/debug.h | 80 +++++++++++++++--------------- include/os/linux/spl/sys/debug.h | 78 ++++++++++++++--------------- 2 files changed, 79 insertions(+), 79 deletions(-) diff --git a/include/os/freebsd/spl/sys/debug.h b/include/os/freebsd/spl/sys/debug.h index 2751f57801..1f820bc334 100644 --- a/include/os/freebsd/spl/sys/debug.h +++ b/include/os/freebsd/spl/sys/debug.h @@ -68,65 +68,65 @@ void spl_dumpstack(void); #define PANIC(fmt, a...) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a) -#define VERIFY(cond) \ - (void) (unlikely(!(cond)) && \ +#define VERIFY(cond) \ + (void) (unlikely(!(cond)) && \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "%s", "VERIFY(" #cond ") failed\n")) -#define VERIFY3B(LEFT, OP, RIGHT) do { \ - boolean_t _verify3_left = (boolean_t)(LEFT); \ - boolean_t _verify3_right = (boolean_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3B(LEFT, OP, RIGHT) do { \ + const boolean_t _verify3_left = (boolean_t)(LEFT); \ + const boolean_t _verify3_right = (boolean_t)(RIGHT);\ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%d " #OP " %d)\n", \ - (boolean_t) (_verify3_left), \ - (boolean_t) (_verify3_right)); \ + "failed (%d " #OP " %d)\n", \ + (boolean_t) (_verify3_left), \ + (boolean_t) (_verify3_right)); \ } while (0) -#define VERIFY3S(LEFT, OP, RIGHT) do { \ - int64_t _verify3_left = (int64_t)(LEFT); \ - int64_t _verify3_right = (int64_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3S(LEFT, OP, RIGHT) do { \ + const int64_t _verify3_left = (int64_t)(LEFT); \ + const int64_t _verify3_right = (int64_t)(RIGHT); \ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%lld " #OP " %lld)\n", \ - (long long) (_verify3_left), \ - (long long) (_verify3_right)); \ + "failed (%lld " #OP " %lld)\n", \ + (long long) (_verify3_left), \ + (long long) (_verify3_right)); \ } while (0) -#define VERIFY3U(LEFT, OP, RIGHT) do { \ - uint64_t _verify3_left = (uint64_t)(LEFT); \ - uint64_t _verify3_right = (uint64_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3U(LEFT, OP, RIGHT) do { \ + const uint64_t _verify3_left = (uint64_t)(LEFT); \ + const uint64_t _verify3_right = (uint64_t)(RIGHT); \ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%llu " #OP " %llu)\n", \ - (unsigned long long) (_verify3_left), \ - (unsigned long long) (_verify3_right)); \ + "failed (%llu " #OP " %llu)\n", \ + (unsigned long long) (_verify3_left), \ + (unsigned long long) (_verify3_right)); \ } while (0) -#define VERIFY3P(LEFT, OP, RIGHT) do { \ - uintptr_t _verify3_left = (uintptr_t)(LEFT); \ - uintptr_t _verify3_right = (uintptr_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3P(LEFT, OP, RIGHT) do { \ + const uintptr_t _verify3_left = (uintptr_t)(LEFT); \ + const uintptr_t _verify3_right = (uintptr_t)(RIGHT);\ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%px " #OP " %px)\n", \ - (void *) (_verify3_left), \ - (void *) (_verify3_right)); \ + "failed (%px " #OP " %px)\n", \ + (void *) (_verify3_left), \ + (void *) (_verify3_right)); \ } while (0) -#define VERIFY0(RIGHT) do { \ - int64_t _verify3_left = (int64_t)(0); \ - int64_t _verify3_right = (int64_t)(RIGHT); \ - if (!(_verify3_left == _verify3_right)) \ +#define VERIFY0(RIGHT) do { \ + const int64_t _verify3_left = (int64_t)(0); \ + const int64_t _verify3_right = (int64_t)(RIGHT); \ + if (unlikely(!(_verify3_left == _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ - "VERIFY3(0 == " #RIGHT ") " \ - "failed (0 == %lld)\n", \ - (long long) (_verify3_right)); \ + "VERIFY3(0 == " #RIGHT ") " \ + "failed (0 == %lld)\n", \ + (long long) (_verify3_right)); \ } while (0) -#define CTASSERT_GLOBAL(x) CTASSERT(x) +#define CTASSERT_GLOBAL(x) CTASSERT(x) /* * Debugging disabled (--disable-debug) @@ -154,11 +154,11 @@ void spl_dumpstack(void); #define ASSERT0 VERIFY0 #define ASSERT VERIFY #define IMPLY(A, B) \ - ((void)(((!(A)) || (B)) || \ + ((void)(likely((!(A)) || (B)) || \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "(" #A ") implies (" #B ")"))) #define EQUIV(A, B) \ - ((void)((!!(A) == !!(B)) || \ + ((void)(likely(!!(A) == !!(B)) || \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "(" #A ") is equivalent to (" #B ")"))) /* END CSTYLED */ diff --git a/include/os/linux/spl/sys/debug.h b/include/os/linux/spl/sys/debug.h index 46da5c7833..dc6b85eebf 100644 --- a/include/os/linux/spl/sys/debug.h +++ b/include/os/linux/spl/sys/debug.h @@ -62,63 +62,63 @@ void spl_dumpstack(void); #define PANIC(fmt, a...) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a) -#define VERIFY(cond) \ - (void) (unlikely(!(cond)) && \ +#define VERIFY(cond) \ + (void) (unlikely(!(cond)) && \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "%s", "VERIFY(" #cond ") failed\n")) -#define VERIFY3B(LEFT, OP, RIGHT) do { \ - boolean_t _verify3_left = (boolean_t)(LEFT); \ - boolean_t _verify3_right = (boolean_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3B(LEFT, OP, RIGHT) do { \ + const boolean_t _verify3_left = (boolean_t)(LEFT); \ + const boolean_t _verify3_right = (boolean_t)(RIGHT);\ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%d " #OP " %d)\n", \ - (boolean_t) (_verify3_left), \ - (boolean_t) (_verify3_right)); \ + "failed (%d " #OP " %d)\n", \ + (boolean_t) (_verify3_left), \ + (boolean_t) (_verify3_right)); \ } while (0) -#define VERIFY3S(LEFT, OP, RIGHT) do { \ - int64_t _verify3_left = (int64_t)(LEFT); \ - int64_t _verify3_right = (int64_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3S(LEFT, OP, RIGHT) do { \ + const int64_t _verify3_left = (int64_t)(LEFT); \ + const int64_t _verify3_right = (int64_t)(RIGHT); \ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%lld " #OP " %lld)\n", \ - (long long) (_verify3_left), \ - (long long) (_verify3_right)); \ + "failed (%lld " #OP " %lld)\n", \ + (long long) (_verify3_left), \ + (long long) (_verify3_right)); \ } while (0) -#define VERIFY3U(LEFT, OP, RIGHT) do { \ - uint64_t _verify3_left = (uint64_t)(LEFT); \ - uint64_t _verify3_right = (uint64_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3U(LEFT, OP, RIGHT) do { \ + const uint64_t _verify3_left = (uint64_t)(LEFT); \ + const uint64_t _verify3_right = (uint64_t)(RIGHT); \ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%llu " #OP " %llu)\n", \ - (unsigned long long) (_verify3_left), \ - (unsigned long long) (_verify3_right)); \ + "failed (%llu " #OP " %llu)\n", \ + (unsigned long long) (_verify3_left), \ + (unsigned long long) (_verify3_right)); \ } while (0) -#define VERIFY3P(LEFT, OP, RIGHT) do { \ - uintptr_t _verify3_left = (uintptr_t)(LEFT); \ - uintptr_t _verify3_right = (uintptr_t)(RIGHT); \ - if (!(_verify3_left OP _verify3_right)) \ +#define VERIFY3P(LEFT, OP, RIGHT) do { \ + const uintptr_t _verify3_left = (uintptr_t)(LEFT); \ + const uintptr_t _verify3_right = (uintptr_t)(RIGHT);\ + if (unlikely(!(_verify3_left OP _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ - "failed (%px " #OP " %px)\n", \ - (void *) (_verify3_left), \ - (void *) (_verify3_right)); \ + "failed (%px " #OP " %px)\n", \ + (void *) (_verify3_left), \ + (void *) (_verify3_right)); \ } while (0) -#define VERIFY0(RIGHT) do { \ - int64_t _verify3_left = (int64_t)(0); \ - int64_t _verify3_right = (int64_t)(RIGHT); \ - if (!(_verify3_left == _verify3_right)) \ +#define VERIFY0(RIGHT) do { \ + const int64_t _verify3_left = (int64_t)(0); \ + const int64_t _verify3_right = (int64_t)(RIGHT); \ + if (unlikely(!(_verify3_left == _verify3_right))) \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ - "VERIFY3(0 == " #RIGHT ") " \ - "failed (0 == %lld)\n", \ - (long long) (_verify3_right)); \ + "VERIFY3(0 == " #RIGHT ") " \ + "failed (0 == %lld)\n", \ + (long long) (_verify3_right)); \ } while (0) #define CTASSERT_GLOBAL(x) _CTASSERT(x, __LINE__) @@ -154,11 +154,11 @@ void spl_dumpstack(void); #define ASSERT0 VERIFY0 #define ASSERT VERIFY #define IMPLY(A, B) \ - ((void)(((!(A)) || (B)) || \ + ((void)(likely((!(A)) || (B)) || \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "(" #A ") implies (" #B ")"))) #define EQUIV(A, B) \ - ((void)((!!(A) == !!(B)) || \ + ((void)(likely(!!(A) == !!(B)) || \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \ "(" #A ") is equivalent to (" #B ")"))) /* END CSTYLED */