Extend zloop.sh for automated testing
In order to debug issues encountered by ztest during automated testing it's important that as much debugging information as possible by dumped at the time of the failure. The following changes extend the zloop.sh script in order to make it easier to integrate with buildbot. * Add the `-m <maximum cores>` option to zloop.sh to place a limit of the number of core dumps generated. By default, the existing behavior is maintained and no limit is set. * Add the `-l` option to create a 'ztest.core.N' symlink in the current directory to the core directory. This functionality is provided primarily for buildbot which expects log files to have well known names. * Rename 'ztest.ddt' to 'ztest.zdb' and extend it to dump additional basic information on failure for latter analysis. Reviewed-by: Tim Chase <tim@chase2k.com> Reviewed by: Thomas Caputi <tcaputi@datto.com> Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov> Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Closes #6999
This commit is contained in:
Brian Behlendorf
parent
bb25362553
commit
cd0a89ded9
|
|
@ -52,6 +52,8 @@ function usage
|
|||
" -s Size of vdev devices.\n" \
|
||||
" -f Specify working directory for ztest vdev files.\n" \
|
||||
" -c Specify a core dump directory to use.\n" \
|
||||
" -m Max number of core dumps to allow before exiting.\n" \
|
||||
" -l Create 'ztest.core.N' symlink to core directory.\n" \
|
||||
" -h Print this help message.\n" \
|
||||
"" >&2
|
||||
}
|
||||
|
|
@ -105,14 +107,24 @@ function store_core
|
|||
coreid=$(date "+zloop-%y%m%d-%H%M%S")
|
||||
foundcrashes=$((foundcrashes + 1))
|
||||
|
||||
# zdb debugging
|
||||
zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
|
||||
zdbdebug=$($zdbcmd 2>&1)
|
||||
echo -e "$zdbcmd\n" >>ztest.zdb
|
||||
echo "$zdbdebug" >>ztest.zdb
|
||||
|
||||
dest=$coredir/$coreid
|
||||
or_die mkdir -p "$dest"
|
||||
or_die mkdir -p "$dest/vdev"
|
||||
|
||||
if [[ $symlink -ne 0 ]]; then
|
||||
or_die ln -sf "$dest" ztest.core.$foundcrashes
|
||||
fi
|
||||
|
||||
echo "*** ztest crash found - moving logs to $dest"
|
||||
|
||||
or_die mv ztest.history "$dest/"
|
||||
or_die mv ztest.ddt "$dest/"
|
||||
or_die mv ztest.zdb "$dest/"
|
||||
or_die mv ztest.out "$dest/"
|
||||
or_die mv "$workdir/ztest*" "$dest/vdev/"
|
||||
|
||||
|
|
@ -123,7 +135,7 @@ function store_core
|
|||
# check for core
|
||||
if [[ -f "$core" ]]; then
|
||||
coreprog=$(core_prog "$core")
|
||||
corestatus=$($GDB --batch --quiet \
|
||||
coredebug=$($GDB --batch --quiet \
|
||||
-ex "set print thread-events off" \
|
||||
-ex "printf \"*\n* Backtrace \n*\n\"" \
|
||||
-ex "bt" \
|
||||
|
|
@ -135,17 +147,25 @@ function store_core
|
|||
-ex "thread apply all bt" \
|
||||
-ex "printf \"*\n* Backtraces (full) \n*\n\"" \
|
||||
-ex "thread apply all bt full" \
|
||||
-ex "quit" "$coreprog" "$core" | grep -v "New LWP")
|
||||
-ex "quit" "$coreprog" "$core" 2>&1 | \
|
||||
grep -v "New LWP")
|
||||
|
||||
# Dump core + logs to stored directory
|
||||
echo "$corestatus" >>"$dest/status"
|
||||
echo "$coredebug" >>"$dest/ztest.gdb"
|
||||
or_die mv "$core" "$dest/"
|
||||
|
||||
# Record info in cores logfile
|
||||
echo "*** core @ $coredir/$coreid/$core:" | \
|
||||
tee -a ztest.cores
|
||||
fi
|
||||
echo "continuing..."
|
||||
|
||||
if [[ $coremax -gt 0 ]] &&
|
||||
[[ $foundcrashes -ge $coremax ]]; then
|
||||
echo "exiting... max $coremax allowed cores"
|
||||
exit 1
|
||||
else
|
||||
echo "continuing..."
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -156,12 +176,16 @@ basedir=$DEFAULTWORKDIR
|
|||
rundir="zloop-run"
|
||||
timeout=0
|
||||
size="512m"
|
||||
while getopts ":ht:s:c:f:" opt; do
|
||||
coremax=0
|
||||
symlink=0
|
||||
while getopts ":ht:m:s:c:f:l" opt; do
|
||||
case $opt in
|
||||
t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
|
||||
m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
|
||||
s ) [[ $OPTARG ]] && size=$OPTARG ;;
|
||||
c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
|
||||
f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
|
||||
l ) symlink=1 ;;
|
||||
h ) usage
|
||||
exit 2
|
||||
;;
|
||||
|
|
@ -180,6 +204,7 @@ export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0
|
|||
if [[ -f "$(core_file)" ]]; then
|
||||
echo -n "There's a core dump here you might want to look at first... "
|
||||
core_file
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -194,7 +219,7 @@ if [[ ! -w $coredir ]]; then
|
|||
fi
|
||||
|
||||
or_die rm -f ztest.history
|
||||
or_die rm -f ztest.ddt
|
||||
or_die rm -f ztest.zdb
|
||||
or_die rm -f ztest.cores
|
||||
|
||||
ztrc=0 # ztest return value
|
||||
|
|
@ -245,7 +270,6 @@ while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do
|
|||
$cmd >>ztest.out 2>&1
|
||||
ztrc=$?
|
||||
grep -E '===|WARNING' ztest.out >>ztest.history
|
||||
$ZDB -U "$workdir/zpool.cache" -DD ztest >>ztest.ddt 2>&1
|
||||
|
||||
store_core
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue