improve handling of mail IDs

We now use the swaks transaction to grep for the QUEUE id (if possible)
and apply more robust matching this way.

test/helper/sending.bash

@@ -128,7 +128,7 @@ function _send_email() {
 #
 # 1. this function waits for the mail to be processed; there is no asynchronicity
 #    because filtering the logs in a synchronous way is easier and safer;
-# 2. this function takes the name of a variable and inserts ID(s) one can later
+# 2. this function takes the name of a variable and inserts IDs one can later
 #    filter by to check logs.
 #
 # No. 2 is especially useful in case you send more than one email in a single
@@ -138,19 +138,14 @@ function _send_email() {
 # in. The second argument **can** be the flag `--unchecked`; if this flag is supplied,
 # the function uses `_send_email_unchecked` instead of `_send_email`. This avoids the
 # `assert_success`. Be warned though this is only required in special situations where
-# it is still possible to `grep` for the mail IDs - otherwise this function fails.
+# it is still possible to `grep` for the Message-ID that Postfix generated, -
-#
+# otherwise this function fails. The rest of the arguments are the same as `_send_email`.
-# The rest of the arguments are the same as `_send_email`.
 #
 # ## Attention
 #
 # This function assumes `CONTAINER_NAME` to be properly set (to the container
 # name the command should be executed in)!
 #
-# Moreover, if `--data <DATA>` is specified, the additional header added implicitly
-# (with `--add-header`) may get lost, so pay attention to the data having the token
-# to place additonal headers into.
-#
 # ## Safety
 #
 # This functions assumes **no concurrent sending of emails to the same container**!
@@ -159,51 +154,42 @@ function _send_email() {
 # is UNDEFINED BEHAVIOR!
 function _send_email_and_get_id() {
   # Get the name of the variable that the ID is stored in
-  local ID_NAME="${1:?Mail ID must be set for _send_email_and_get_id}"
+  local ID_ENV_VAR_NAME="${1:?Mail ID must be set for _send_email_and_get_id}"
-  # Get a "reference" to the content of ID_NAME so we can manipulate the content
+  # Get a "reference" to the content of ID_ENV_VAR_NAME so we can manipulate the content
-  local -n MAIL_ID=${ID_NAME}
+  local -n ID_ENV_VAR_REF=${ID_ENV_VAR_NAME}
   # Export the variable so everyone has access
   # shellcheck disable=SC2163
-  export "${ID_NAME}"
+  export "${ID_ENV_VAR_NAME}"
   shift 1
 
+  local QUEUE_ID MESSAGE_ID
+  # The unique ID Postfix (and other services) use may be different in length
+  # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a
+  # range to safely capture it.
+  local QUEUE_ID_REGEX='[A-Z0-9]{9,12}'
+  local MESSAGE_ID_REGEX="[0-9]{14}\\.${QUEUE_ID_REGEX}"
+
   _wait_for_empty_mail_queue_in_container
-  if [[ ${1:-} == --unchecked ]]; then
+  if [[ ${1} == --unchecked ]]; then
     shift 1
-    _send_email_unchecked "${@}" --add-header "Message-Id: ${ID_NAME}"
+    local OUTPUT=$(_send_email_unchecked "${@}")
+    QUEUE_ID=$(_exec_in_container tac /var/log/mail.log       \
+      | grep -E "postfix/smtpd.*: ${QUEUE_ID_REGEX}: client=" \
+      | grep -E -m 1 -o '[A-Z0-9]{9,12}' || :)
   else
-    _send_email "${@}" --add-header "Message-Id: ${ID_NAME}"
+    local OUTPUT=$(_send_email "${@}")
+    QUEUE_ID=$(grep -F 'queued as' <<< "${OUTPUT}" | grep -E -o "${QUEUE_ID_REGEX}$")
   fi
   _wait_for_empty_mail_queue_in_container
 
-  # The unique IDs Postfix (and other services) use may be different in length
+  MESSAGE_ID=$(_exec_in_container tac /var/log/mail.log \
-  # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a
+    | grep -E "message-id=<${MESSAGE_ID_REGEX}@"        \
-  # range to safely capture it.
+    | grep -E -m 1 -o "${MESSAGE_ID_REGEX}" || :)
-  #
+
-  # First, we define the regular expressions we need for capturing the IDs.
+  ID_ENV_VAR_REF="${QUEUE_ID}|${MESSAGE_ID}"
-  local REGEX_ID_PART_ONE='[A-Z0-9]{9,12}'
-  local REGEX_ID_PART_TWO="$(date +'%Y%m%d')[0-9]+\\.[0-9]+"
-  # The first line Postfix logs looks something like this:
-  #
-  # Jan  4 16:09:19 mail postfix/cleanup[1188]: 07B29249A7: message-id=MAIL_ID_HEADER
-  #
-  # where 07B29249A7 is one of the IDs we are searching for and MAIL_ID_HEADER is what ID_NAME
-  # is set to. Note that we are searching the log in reverse, which is important to get the correct ID.
-  MAIL_ID=$(_exec_in_container tac /var/log/mail.log \
-    | grep -F -m 1 "message-id=${ID_NAME}" \
-    | grep -E -o "${REGEX_ID_PART_ONE}")
-  # We additionally grep for another ID that Postfix (and later mechanisms like Sieve) use (additionally).
-  # The corresponding line looks something like this:
-  #
-  # Jan  4 16:09:19 mail postfix/cleanup[1188]: 07B29249A7: message-id=<20240104160919.001289@mail.example.test>
-  #
-  # where 20240104160919 is the other ID we are searching for. Note that the date is encoded by this ID.
-  # We exploit the fact that MAIL_ID is already on the line (07B29249A7), so we can search for it efficiently.
-  MAIL_ID+="|$(_exec_in_container grep -F "${MAIL_ID}: message-id=" /var/log/mail.log \
-    | grep -E -o "${REGEX_ID_PART_TWO}")"
 
   # Last but not least, we perform plausibility checks on the IDs.
-  assert_not_equal "${MAIL_ID}" ''
+  assert_not_equal "${ID_ENV_VAR_REF}" ''
-  run echo "${MAIL_ID}"
+  run echo "${ID_ENV_VAR_REF}"
-  assert_line --regexp "^${REGEX_ID_PART_ONE}|${REGEX_ID_PART_TWO}$"
+  assert_line --regexp "^${QUEUE_ID_REGEX}|${MESSAGE_ID_REGEX}$"
 }

test/tests/parallel/set1/spam_virus/rspamd_full.bats

@@ -47,12 +47,15 @@ function setup_file() {
   # 1. The first one should pass just fine
   _send_email_and_get_id MAIL_ID_PASS
   # 2. The second one should be rejected due to spam (GTube pattern)
-  _send_email_and_get_id MAIL_ID_SPAM --unchecked --body 'XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X'
+  _send_email_and_get_id MAIL_ID_SPAM --unchecked \
+    --body 'XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X'
   # 3. Te third one should be rejected due to a virus (ClamAV Eicar pattern)
   # shellcheck disable=SC2016
-  _send_email_and_get_id MAIL_ID_VIRUS --unchecked --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'
+  _send_email_and_get_id MAIL_ID_VIRUS --unchecked \
+    --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'
   # 4. The fourth one will receive an added header (GTube pattern)
-  _send_email_and_get_id MAIL_ID_HEADER --unchecked --body 'YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X'
+  _send_email_and_get_id MAIL_ID_HEADER \
+    --body 'YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X'
 
   _run_in_container cat /var/log/mail.log
   assert_success