Allow to limit zed's syslog chattiness

Some usage patterns like send/recv of replication streams can
produce a large number of events. In such a case, the current
all-syslog.sh zedlet will hold up to its name, and flood the
logs with mostly redundant information. Two mitigate this
situation, this changeset introduces to new variables
ZED_SYSLOG_SUBCLASS_INCLUDE and ZED_SYSLOG_SUBCLASS_EXCLUDE
to zed.rc that give more control over which event classes end
up in the syslog.

Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Daniel Kobras <d.kobras@science-computing.de>
Closes #6886
Closes #7260

cmd/zed/zed.d/all-debug.sh

@@ -10,6 +10,8 @@
 
 : "${ZED_DEBUG_LOG:="${TMPDIR:="/tmp"}/zed.debug.log"}"
 
+zed_exit_if_ignoring_this_event
+
 lockfile="$(basename -- "${ZED_DEBUG_LOG}").lock"
 
 umask 077

cmd/zed/zed.d/all-syslog.sh

@@ -5,6 +5,8 @@
 [ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
 . "${ZED_ZEDLET_DIR}/zed-functions.sh"
 
+zed_exit_if_ignoring_this_event
+
 zed_log_msg "eid=${ZEVENT_EID}" "class=${ZEVENT_SUBCLASS}" \
     "${ZEVENT_POOL_GUID:+"pool_guid=${ZEVENT_POOL_GUID}"}" \
     "${ZEVENT_VDEV_PATH:+"vdev_path=${ZEVENT_VDEV_PATH}"}" \

cmd/zed/zed.d/zed-functions.sh

@@ -438,3 +438,23 @@ zed_guid_to_pool()
 		$ZPOOL get -H -ovalue,name guid | awk '$1=='"$guid"' {print $2}'
 	fi
 }
+
+# zed_exit_if_ignoring_this_event
+#
+# Exit the script if we should ignore this event, as determined by
+# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc.
+# This function assumes you've imported the normal zed variables.
+zed_exit_if_ignoring_this_event()
+{
+	if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then
+	    eval "case ${ZEVENT_SUBCLASS} in
+	    ${ZED_SYSLOG_SUBCLASS_INCLUDE});;
+	    *) exit 0;;
+	    esac"
+	elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then
+	    eval "case ${ZEVENT_SUBCLASS} in
+	    ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;;
+	    *);;
+	    esac"
+	fi
+}

cmd/zed/zed.d/zed.rc

@@ -100,3 +100,14 @@ ZED_USE_ENCLOSURE_LEDS=1
 #
 #ZED_SYSLOG_TAG="zed"
 
+##
+# Which set of event subclasses to log
+# By default, events from all subclasses are logged.
+# If ZED_SYSLOG_SUBCLASS_INCLUDE is set, only subclasses
+# matching the pattern are logged. Use the pipe symbol (|)
+# or shell wildcards (*, ?) to match multiple subclasses.
+# Otherwise, if ZED_SYSLOG_SUBCLASS_EXCLUDE is set, the
+# matching subclasses are excluded from logging.
+#ZED_SYSLOG_SUBCLASS_INCLUDE="checksum|scrub_*|vdev.*"
+#ZED_SYSLOG_SUBCLASS_EXCLUDE="statechange|config_*|history_event"
+

tests/runfiles/linux.run

@@ -413,7 +413,7 @@ tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
 tags = ['functional', 'devices']
 
 [tests/functional/events]
-tests = ['events_001_pos', 'events_002_pos']
+tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter']
 tags = ['functional', 'events']
 
 [tests/functional/exec]

tests/zfs-tests/include/libtest.shlib

@@ -3398,6 +3398,21 @@ function wait_scrubbed
 	return 1
 }
 
+# Backup the zed.rc in our test directory so that we can edit it for our test.
+#
+# Returns: Backup file name.  You will need to pass this to zed_rc_restore().
+function zed_rc_backup
+{
+	zedrc_backup="$(mktemp)"
+	cp $ZEDLET_DIR/zed.rc $zedrc_backup
+	echo $zedrc_backup
+}
+
+function zed_rc_restore
+{
+	mv $1 $ZEDLET_DIR/zed.rc
+}
+
 #
 # Setup custom environment for the ZED.
 #
@@ -3536,6 +3551,23 @@ function zed_events_drain
        done
 }
 
+# Set a variable in zed.rc to something, un-commenting it in the process.
+#
+# $1 variable
+# $2 value
+function zed_rc_set
+{
+	var="$1"
+	val="$2"
+	# Remove the line
+	cmd="'/$var/d'"
+	eval sed -i $cmd $ZEDLET_DIR/zed.rc
+
+	# Add it at the end
+	echo "$var=$val" >> $ZEDLET_DIR/zed.rc
+}
+
+
 #
 # Check is provided device is being active used as a swap device.
 #

tests/zfs-tests/tests/functional/events/Makefile.am

@@ -5,4 +5,5 @@ dist_pkgdata_SCRIPTS = \
 	events.cfg \
 	events_common.kshlib \
 	events_001_pos.ksh \
-	events_002_pos.ksh
+	events_002_pos.ksh \
+	zed_rc_filter.ksh

tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh

@@ -0,0 +1,104 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+# DESCRIPTION:
+# Verify zed.rc ZED_SYSLOG_SUBCLASS_INCLUDE/EXCLUDE event filtering works.
+#
+# STRATEGY:
+# 1. Execute zpool sub-commands on a pool.
+# 2. Test different combinations of ZED_SYSLOG_SUBCLASS_INCLUDE filtering.
+# 3. Execute zpool sub-commands on a pool.
+# 4. Test different combinations of ZED_SYSLOG_SUBCLASS_EXCLUDE filtering.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	log_must zed_stop
+	zed_rc_restore $zedrc_backup
+}
+
+log_assert "Verify zpool sub-commands generate expected events"
+log_onexit cleanup
+
+log_must zpool events -c
+log_must zed_start
+
+# Backup our zed.rc
+zedrc_backup=$(zed_rc_backup)
+
+log_note "Include a single event type"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE history_event
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Include a single event type with wildcards"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE '*history_event*'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Test a filter of a non-match and a match"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'foobar|*history_event*'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Include multiple events"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start|scrub_finish'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.scrub_start" \
+    -e "sysevent.fs.zfs.scrub_finish" \
+    "zpool scrub $TESTPOOL && wait_scrubbed $TESTPOOL"
+
+# We can't use run_and_verify() for exclusions, so run the rest of the tests
+# manually.
+log_note "Test one exclusion"
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'history_event'
+truncate -s 0 $ZED_DEBUG_LOG
+log_must zfs set compression=off $TESTPOOL/$TESTFS
+log_must file_wait $ZED_DEBUG_LOG 3
+log_mustnot grep -q history_event $ZED_DEBUG_LOG
+
+log_note "Test one exclusion with wildcards"
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE '*history_event*'
+truncate -s 0 $ZED_DEBUG_LOG
+log_must zfs set compression=off $TESTPOOL/$TESTFS
+log_must file_wait $ZED_DEBUG_LOG 3
+log_mustnot grep -q history_event $ZED_DEBUG_LOG
+
+log_note "Test one inclusion and one exclusion"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start'
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'scrub_finish'
+truncate -s 0 $ZED_DEBUG_LOG
+zpool scrub $TESTPOOL
+wait_scrubbed $TESTPOOL
+log_must file_wait $ZED_DEBUG_LOG 3
+log_must grep -q scrub_start $ZED_DEBUG_LOG
+log_mustnot grep -q scrub_finish $ZED_DEBUG_LOG
+
+log_pass "zed.rc event filtering works correctly."

tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh

@@ -32,16 +32,15 @@
 log_assert "Testing the scrub after resilver zedlet"
 
 # Backup our zed.rc
-zedrc_backup="$(mktemp)"
+zedrc_backup=$(zed_rc_backup)
-log_must cp $ZEDLET_DIR/zed.rc $zedrc_backup
 
-# Enable ZED_SCRUB_AFTER_RESILVER
+# Enable ZED_SCRUB_AFTER_RESILVER in zed.rc
-eval "sed -i 's/\#ZED_SCRUB_AFTER_RESILVER/ZED_SCRUB_AFTER_RESILVER/g' $ZEDLET_DIR/zed.rc"
+zed_rc_set ZED_SCRUB_AFTER_RESILVER 1
 
 function cleanup
 {
 	# Restore our zed.rc
-	log_must mv $zedrc_backup $ZEDLET_DIR/zed.rc
+	log_must zed_rc_restore $zedrc_backup
 	default_cleanup_noexit
 }