Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Microoptimizations for VERIFY() and friends 

Add branch hints and constify the intermediate evaluations of 
left/right params in VERIFY3*().

Reviewed-by: Ryan Moeller <ryan@iXsystems.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Adam Moss <c@yotes.com>
Closes #11708
This commit is contained in:
Adam D. Moss 2021-03-11 17:16:09 -08:00 committed by Tony Hutter
parent 6f349d33a5
commit 0d6186cff3
2 changed files with 79 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
include/os/freebsd/spl/sys/debug.h
View File

@ -68,65 +68,65 @@ void spl_dumpstack(void);
#define PANIC(fmt, a...) \ #define PANIC(fmt, a...) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a) spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a)
#define VERIFY(cond) \ #define VERIFY(cond) \
(void) (unlikely(!(cond)) && \ (void) (unlikely(!(cond)) && \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"%s", "VERIFY(" #cond ") failed\n")) "%s", "VERIFY(" #cond ") failed\n"))
#define VERIFY3B(LEFT, OP, RIGHT) do { \ #define VERIFY3B(LEFT, OP, RIGHT) do { \
boolean_t _verify3_left = (boolean_t)(LEFT); \ const boolean_t _verify3_left = (boolean_t)(LEFT); \
boolean_t _verify3_right = (boolean_t)(RIGHT); \ const boolean_t _verify3_right = (boolean_t)(RIGHT);\
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%d " #OP " %d)\n", \ "failed (%d " #OP " %d)\n", \
(boolean_t) (_verify3_left), \ (boolean_t) (_verify3_left), \
(boolean_t) (_verify3_right)); \ (boolean_t) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY3S(LEFT, OP, RIGHT) do { \ #define VERIFY3S(LEFT, OP, RIGHT) do { \
int64_t _verify3_left = (int64_t)(LEFT); \ const int64_t _verify3_left = (int64_t)(LEFT); \
int64_t _verify3_right = (int64_t)(RIGHT); \ const int64_t _verify3_right = (int64_t)(RIGHT); \
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%lld " #OP " %lld)\n", \ "failed (%lld " #OP " %lld)\n", \
(long long) (_verify3_left), \ (long long) (_verify3_left), \
(long long) (_verify3_right)); \ (long long) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY3U(LEFT, OP, RIGHT) do { \ #define VERIFY3U(LEFT, OP, RIGHT) do { \
uint64_t _verify3_left = (uint64_t)(LEFT); \ const uint64_t _verify3_left = (uint64_t)(LEFT); \
uint64_t _verify3_right = (uint64_t)(RIGHT); \ const uint64_t _verify3_right = (uint64_t)(RIGHT); \
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%llu " #OP " %llu)\n", \ "failed (%llu " #OP " %llu)\n", \
(unsigned long long) (_verify3_left), \ (unsigned long long) (_verify3_left), \
(unsigned long long) (_verify3_right)); \ (unsigned long long) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY3P(LEFT, OP, RIGHT) do { \ #define VERIFY3P(LEFT, OP, RIGHT) do { \
uintptr_t _verify3_left = (uintptr_t)(LEFT); \ const uintptr_t _verify3_left = (uintptr_t)(LEFT); \
uintptr_t _verify3_right = (uintptr_t)(RIGHT); \ const uintptr_t _verify3_right = (uintptr_t)(RIGHT);\
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%px " #OP " %px)\n", \ "failed (%px " #OP " %px)\n", \
(void *) (_verify3_left), \ (void *) (_verify3_left), \
(void *) (_verify3_right)); \ (void *) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY0(RIGHT) do { \ #define VERIFY0(RIGHT) do { \
int64_t _verify3_left = (int64_t)(0); \ const int64_t _verify3_left = (int64_t)(0); \
int64_t _verify3_right = (int64_t)(RIGHT); \ const int64_t _verify3_right = (int64_t)(RIGHT); \
if (!(_verify3_left == _verify3_right)) \ if (unlikely(!(_verify3_left == _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(0 == " #RIGHT ") " \ "VERIFY3(0 == " #RIGHT ") " \
"failed (0 == %lld)\n", \ "failed (0 == %lld)\n", \
(long long) (_verify3_right)); \ (long long) (_verify3_right)); \
} while (0) } while (0)
#define CTASSERT_GLOBAL(x) CTASSERT(x) #define CTASSERT_GLOBAL(x) CTASSERT(x)
/* /*
* Debugging disabled (--disable-debug) * Debugging disabled (--disable-debug)
@ -154,11 +154,11 @@ void spl_dumpstack(void);
#define ASSERT0 VERIFY0 #define ASSERT0 VERIFY0
#define ASSERT VERIFY #define ASSERT VERIFY
#define IMPLY(A, B) \ #define IMPLY(A, B) \
((void)(((!(A)) || (B)) || \ ((void)(likely((!(A)) || (B)) || \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"(" #A ") implies (" #B ")"))) "(" #A ") implies (" #B ")")))
#define EQUIV(A, B) \ #define EQUIV(A, B) \
((void)((!!(A) == !!(B)) || \ ((void)(likely(!!(A) == !!(B)) || \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"(" #A ") is equivalent to (" #B ")"))) "(" #A ") is equivalent to (" #B ")")))
/* END CSTYLED */ /* END CSTYLED */

78
include/os/linux/spl/sys/debug.h
View File

@ -62,63 +62,63 @@ void spl_dumpstack(void);
#define PANIC(fmt, a...) \ #define PANIC(fmt, a...) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a) spl_panic(__FILE__, __FUNCTION__, __LINE__, fmt, ## a)
#define VERIFY(cond) \ #define VERIFY(cond) \
(void) (unlikely(!(cond)) && \ (void) (unlikely(!(cond)) && \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"%s", "VERIFY(" #cond ") failed\n")) "%s", "VERIFY(" #cond ") failed\n"))
#define VERIFY3B(LEFT, OP, RIGHT) do { \ #define VERIFY3B(LEFT, OP, RIGHT) do { \
boolean_t _verify3_left = (boolean_t)(LEFT); \ const boolean_t _verify3_left = (boolean_t)(LEFT); \
boolean_t _verify3_right = (boolean_t)(RIGHT); \ const boolean_t _verify3_right = (boolean_t)(RIGHT);\
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%d " #OP " %d)\n", \ "failed (%d " #OP " %d)\n", \
(boolean_t) (_verify3_left), \ (boolean_t) (_verify3_left), \
(boolean_t) (_verify3_right)); \ (boolean_t) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY3S(LEFT, OP, RIGHT) do { \ #define VERIFY3S(LEFT, OP, RIGHT) do { \
int64_t _verify3_left = (int64_t)(LEFT); \ const int64_t _verify3_left = (int64_t)(LEFT); \
int64_t _verify3_right = (int64_t)(RIGHT); \ const int64_t _verify3_right = (int64_t)(RIGHT); \
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%lld " #OP " %lld)\n", \ "failed (%lld " #OP " %lld)\n", \
(long long) (_verify3_left), \ (long long) (_verify3_left), \
(long long) (_verify3_right)); \ (long long) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY3U(LEFT, OP, RIGHT) do { \ #define VERIFY3U(LEFT, OP, RIGHT) do { \
uint64_t _verify3_left = (uint64_t)(LEFT); \ const uint64_t _verify3_left = (uint64_t)(LEFT); \
uint64_t _verify3_right = (uint64_t)(RIGHT); \ const uint64_t _verify3_right = (uint64_t)(RIGHT); \
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%llu " #OP " %llu)\n", \ "failed (%llu " #OP " %llu)\n", \
(unsigned long long) (_verify3_left), \ (unsigned long long) (_verify3_left), \
(unsigned long long) (_verify3_right)); \ (unsigned long long) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY3P(LEFT, OP, RIGHT) do { \ #define VERIFY3P(LEFT, OP, RIGHT) do { \
uintptr_t _verify3_left = (uintptr_t)(LEFT); \ const uintptr_t _verify3_left = (uintptr_t)(LEFT); \
uintptr_t _verify3_right = (uintptr_t)(RIGHT); \ const uintptr_t _verify3_right = (uintptr_t)(RIGHT);\
if (!(_verify3_left OP _verify3_right)) \ if (unlikely(!(_verify3_left OP _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \ "VERIFY3(" #LEFT " " #OP " " #RIGHT ") " \
"failed (%px " #OP " %px)\n", \ "failed (%px " #OP " %px)\n", \
(void *) (_verify3_left), \ (void *) (_verify3_left), \
(void *) (_verify3_right)); \ (void *) (_verify3_right)); \
} while (0) } while (0)
#define VERIFY0(RIGHT) do { \ #define VERIFY0(RIGHT) do { \
int64_t _verify3_left = (int64_t)(0); \ const int64_t _verify3_left = (int64_t)(0); \
int64_t _verify3_right = (int64_t)(RIGHT); \ const int64_t _verify3_right = (int64_t)(RIGHT); \
if (!(_verify3_left == _verify3_right)) \ if (unlikely(!(_verify3_left == _verify3_right))) \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"VERIFY3(0 == " #RIGHT ") " \ "VERIFY3(0 == " #RIGHT ") " \
"failed (0 == %lld)\n", \ "failed (0 == %lld)\n", \
(long long) (_verify3_right)); \ (long long) (_verify3_right)); \
} while (0) } while (0)
#define CTASSERT_GLOBAL(x) _CTASSERT(x, __LINE__) #define CTASSERT_GLOBAL(x) _CTASSERT(x, __LINE__)
@ -154,11 +154,11 @@ void spl_dumpstack(void);
#define ASSERT0 VERIFY0 #define ASSERT0 VERIFY0
#define ASSERT VERIFY #define ASSERT VERIFY
#define IMPLY(A, B) \ #define IMPLY(A, B) \
((void)(((!(A)) || (B)) || \ ((void)(likely((!(A)) || (B)) || \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"(" #A ") implies (" #B ")"))) "(" #A ") implies (" #B ")")))
#define EQUIV(A, B) \ #define EQUIV(A, B) \
((void)((!!(A) == !!(B)) || \ ((void)(likely(!!(A) == !!(B)) || \
spl_panic(__FILE__, __FUNCTION__, __LINE__, \ spl_panic(__FILE__, __FUNCTION__, __LINE__, \
"(" #A ") is equivalent to (" #B ")"))) "(" #A ") is equivalent to (" #B ")")))
/* END CSTYLED */ /* END CSTYLED */