Merge branch 'master' into mta-sts-support

.gitignore

@@ -3,6 +3,7 @@
 #################################################
 
 .env
+compose.override.yaml
 docs/site/
 docker-data/
 

CHANGELOG.md

@@ -6,8 +6,22 @@ All notable changes to this project will be documented in this file. The format
 
 > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes.
 
+### Features
+
+  - **Authentication with OIDC / OAuth 2.0** 🎉
+    - DMS now supports authentication via OAuth2 (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) from capable services (_like Roundcube_).
+    - This does not replace the need for an `ACCOUNT_PROVISIONER` (`FILE` / `LDAP`), which is required for an account to receive or send mail.
+    - Successful authentication (_via Dovecot PassDB_) still requires an existing account (_lookup via Dovecot UserDB_).
+
 ### Updates
 
+- **Tests**:
+  - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747)):
+    - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2.
+    - `swaks` version is now the latest from Github releases instead of the Debian package.
+    - `_nc_wrapper`, `_send_mail` and related helpers expect the `.txt` filepath extension again.
+    - `sending.bash` helper methods were refactored to better integrate `swaks` and accommodate different usage contexts.
+    - `test/files/emails/existing/` files were removed similar to previous removal of SMTP auth files as they became redundant with `swaks`.
 - **Internal:**
   - tests: Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752))
   - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750))

Dockerfile

@@ -108,6 +108,13 @@ EOF
 COPY target/rspamd/local.d/ /etc/rspamd/local.d/
 COPY target/rspamd/scores.d/* /etc/rspamd/scores.d/
 
+# -----------------------------------------------
+# --- OAUTH2 ------------------------------------
+# -----------------------------------------------
+
+COPY target/dovecot/auth-oauth2.conf.ext /etc/dovecot/conf.d
+COPY target/dovecot/dovecot-oauth2.conf.ext /etc/dovecot
+
 # -----------------------------------------------
 # --- LDAP & SpamAssassin's Cron ----------------
 # -----------------------------------------------

README.md

@@ -48,3 +48,4 @@ If you have issues, please search through [the documentation][documentation::web
 - Support for [LetsEncrypt](https://letsencrypt.org/), manual and self-signed certificates
 - A [setup script](https://docker-mailserver.github.io/docker-mailserver/latest/config/setup.sh) for easy configuration and maintenance
 - SASLauthd with LDAP authentication
+- OAuth2 authentication (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_)

docs/content/config/advanced/auth-oauth2.md

@@ -0,0 +1,69 @@
+---
+title: 'Advanced | Basic OAuth2 Authentication'
+---
+
+## Introduction
+
+!!! warning "This is only a supplement to the existing account provisioners"
+
+    Accounts must still be managed via the configured [`ACCOUNT_PROVISIONER`][env::account-provisioner] (FILE or LDAP).
+
+    Reasoning for this can be found in [#3480][gh-pr::oauth2]. Future iterations on this feature may allow it to become a full account provisioner.
+
+[gh-pr::oauth2]: https://github.com/docker-mailserver/docker-mailserver/pull/3480
+[env::account-provisioner]: ../environment.md#account_provisioner
+
+The present OAuth2 support provides the capability for 3rd-party applications such as Roundcube to authenticate with DMS (dovecot) by using a token obtained from an OAuth2 provider, instead of passing passwords around.
+
+## Example (Authentik & Roundcube)
+
+This example assumes you have:
+
+- A working DMS server set up
+- An Authentik server set up ([documentation](https://goauthentik.io/docs/installation/))
+- A Roundcube server set up (either [docker](https://hub.docker.com/r/roundcube/roundcubemail/) or [bare metal](https://github.com/roundcube/roundcubemail/wiki/Installation))
+
+!!! example "Setup Instructions"
+
+    === "1. Docker Mailserver"
+        Edit the following values in `mailserver.env`:
+        ```env
+        # -----------------------------------------------
+        # --- OAUTH2 Section ----------------------------
+        # -----------------------------------------------
+
+        # empty => OAUTH2 authentication is disabled
+        # 1 => OAUTH2 authentication is enabled
+        ENABLE_OAUTH2=1
+
+        # Specify the user info endpoint URL of the oauth2 provider
+        OAUTH2_INTROSPECTION_URL=https://authentik.example.com/application/o/userinfo/
+        ```
+
+    === "2. Authentik"
+        1. Create a new OAuth2 provider
+        2. Note the client id and client secret
+        3. Set the allowed redirect url to the equivalent of `https://roundcube.example.com/index.php/login/oauth` for your RoundCube instance.
+
+    === "3. Roundcube"
+        Add the following to `oauth2.inc.php` ([documentation](https://github.com/roundcube/roundcubemail/wiki/Configuration)):
+
+        ```php
+        $config['oauth_provider'] = 'generic';
+        $config['oauth_provider_name'] = 'Authentik';
+        $config['oauth_client_id'] = '<insert client id here>';
+        $config['oauth_client_secret'] = '<insert client secret here>';
+        $config['oauth_auth_uri'] = 'https://authentik.example.com/application/o/authorize/';
+        $config['oauth_token_uri'] = 'https://authentik.example.com/application/o/token/';
+        $config['oauth_identity_uri'] = 'https://authentik.example.com/application/o/userinfo/';
+
+        // Optional: disable SSL certificate check on HTTP requests to OAuth server. For possible values, see:
+        // http://docs.guzzlephp.org/en/stable/request-options.html#verify
+        $config['oauth_verify_peer'] = false;
+
+        $config['oauth_scope'] = 'email openid profile';
+        $config['oauth_identity_fields'] = ['email'];
+
+        // Boolean: automatically redirect to OAuth login when opening Roundcube without a valid session
+        $config['oauth_login_redirect'] = false;
+        ```

docs/content/config/environment.md

@@ -54,7 +54,15 @@ The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage m
 
 Configures the provisioning source of user accounts (including aliases) for user queries and authentication by services managed by DMS (_Postfix and Dovecot_).
 
-User provisioning via OIDC is planned for the future, see [this tracking issue](https://github.com/docker-mailserver/docker-mailserver/issues/2713).
+!!! tip "OAuth2 Support"
+
+    Presently DMS supports OAuth2 only as an supplementary authentication method. 
+
+    - A third-party service must provide a valid token for the user which Dovecot validates with the authentication service provider. To enable this feature reference the [OAuth2 configuration example guide][docs::auth::oauth2-config-guide].
+    - User accounts must be provisioned to receive mail via one of the supported `ACCOUNT_PROVISIONER` providers.
+    - User provisioning via OIDC is planned for the future, see [this tracking issue](https://github.com/docker-mailserver/docker-mailserver/issues/2713).
+
+[docs::auth::oauth2-config-guide]: ./advanced/auth-oauth2.md
 
 - **empty** => use FILE
 - LDAP => use LDAP authentication
@@ -725,10 +733,20 @@ Enable or disable `getmail`.
 
 - **5** => `getmail` The number of minutes for the interval. Min: 1; Max: 30; Default: 5.
 
+
+#### OAUTH2
+
+##### ENABLE_OAUTH2
+
+- **empty** => OAUTH2 authentication is disabled
+- 1 => OAUTH2 authentication is enabled
+
+##### OAUTH2_INTROSPECTION_URL
+
+- => Specify the user info endpoint URL of the oauth2 provider (_eg: `https://oauth2.example.com/userinfo/`_)
+
 #### LDAP
 
-
-
 ##### LDAP_START_TLS
 
 - **empty** => no

docs/mkdocs.yml

@@ -143,6 +143,7 @@ nav:
           - 'Postfix': config/advanced/override-defaults/postfix.md
           - 'Modifications via Script': config/advanced/override-defaults/user-patches.md
         - 'LDAP Authentication': config/advanced/auth-ldap.md
+        - 'OAuth2 Authentication': config/advanced/auth-oauth2.md
         - 'Email Filtering with Sieve': config/advanced/mail-sieve.md
         - 'Email Gathering with Fetchmail': config/advanced/mail-fetchmail.md
         - 'Email Gathering with Getmail': config/advanced/mail-getmail.md

mailserver.env

@@ -425,6 +425,18 @@ ENABLE_GETMAIL=0
 # The number of minutes for the interval. Min: 1; Max: 30.
 GETMAIL_POLL=5
 
+# -----------------------------------------------
+# --- OAUTH2 Section ----------------------------
+# -----------------------------------------------
+
+# empty => OAUTH2 authentication is disabled
+# 1 => OAUTH2 authentication is enabled
+ENABLE_OAUTH2=
+
+# Specify the user info endpoint URL of the oauth2 provider
+# Example: https://oauth2.example.com/userinfo/
+OAUTH2_INTROSPECTION_URL=
+
 # -----------------------------------------------
 # --- LDAP Section ------------------------------
 # -----------------------------------------------

target/dovecot/10-auth.conf

@@ -123,6 +123,7 @@ auth_mechanisms = plain login
 #!include auth-sql.conf.ext
 #!include auth-ldap.conf.ext
 !include auth-passwdfile.inc
+#!include auth-oauth2.conf.ext
 #!include auth-checkpassword.conf.ext
 #!include auth-vpopmail.conf.ext
 #!include auth-static.conf.ext

target/dovecot/auth-oauth2.conf.ext

@@ -0,0 +1,7 @@
+auth_mechanisms = $auth_mechanisms oauthbearer xoauth2
+
+passdb {
+    driver = oauth2
+    mechanisms = xoauth2 oauthbearer
+    args = /etc/dovecot/dovecot-oauth2.conf.ext
+}

target/dovecot/dovecot-oauth2.conf.ext

@@ -0,0 +1,4 @@
+introspection_url =
+# Dovecot defaults:
+introspection_mode = auth
+username_attribute = email

target/scripts/build/packages.sh

@@ -80,7 +80,7 @@ function _install_packages() {
   # `bind9-dnsutils` provides the `dig` command
   # `iputils-ping` provides the `ping` command
   DEBUG_PACKAGES=(
-    bind9-dnsutils iputils-ping less nano swaks
+    bind9-dnsutils iputils-ping less nano
   )
 
   apt-get "${QUIET}" --no-install-recommends install \
@@ -192,7 +192,15 @@ function _install_getmail() {
 
 function _install_utils() {
   _log 'debug' 'Installing utils sourced from Github'
+  _log 'trace' 'Installing jaq'
   curl -sL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq
+
+  _log 'trace' 'Installing swaks'
+  local SWAKS_VERSION='20240103.0'
+  local SWAKS_RELEASE="swaks-${SWAKS_VERSION}"
+  curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" | tar -xz
+  mv "${SWAKS_RELEASE}/swaks" /usr/local/bin
+  rm -r "${SWAKS_RELEASE}"
 }
 
 function _remove_data_after_package_installations() {

target/scripts/start-mailserver.sh

@@ -71,6 +71,11 @@ function _register_functions() {
       ;;
   esac
 
+  if [[ ${ENABLE_OAUTH2} -eq 1 ]]; then
+      _environment_variables_oauth2
+      _register_setup_function '_setup_oauth2'
+  fi
+
   if [[ ${ENABLE_SASLAUTHD} -eq 1 ]]; then
     _environment_variables_saslauthd
     _register_setup_function '_setup_saslauthd'

target/scripts/startup/setup.d/oauth2.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+function _setup_oauth2() {
+  _log 'debug' 'Setting up OAUTH2'
+
+  # Enable OAuth2 PassDB (Authentication):
+  sedfile -i -e '/\!include auth-oauth2\.conf\.ext/s/^#//' /etc/dovecot/conf.d/10-auth.conf
+  _replace_by_env_in_file 'OAUTH2_' '/etc/dovecot/dovecot-oauth2.conf.ext'
+
+  return 0
+}

target/scripts/startup/variables-stack.sh

@@ -151,6 +151,12 @@ function __environment_variables_general_setup() {
   VARS[UPDATE_CHECK_INTERVAL]="${UPDATE_CHECK_INTERVAL:=1d}"
 }
 
+function _environment_variables_oauth2() {
+  _log 'debug' 'Setting OAUTH2-related environment variables now'
+
+  VARS[OAUTH2_INTROSPECTION_URL]="${OAUTH2_INTROSPECTION_URL:=}"
+}
+
 # This function handles environment variables related to LDAP.
 # NOTE: SASLAuthd and Dovecot LDAP support inherit these common ENV.
 function _environment_variables_ldap() {

test/config/oauth2/provider.py

@@ -0,0 +1,56 @@
+# OAuth2 mock service
+#
+# Dovecot will query this service with the token it was provided.
+# If the session for the token is valid, a response provides an attribute to perform a UserDB lookup on (default: email).
+
+import json
+import base64
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+# OAuth2.0 Bearer token (paste into https://jwt.io/ to check it's contents).
+# You should never need to edit this unless you REALLY need to change the issuer.
+token = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vcHJvdmlkZXIuZXhhbXBsZS50ZXN0OjgwMDAvIiwic3ViIjoiODJjMWMzMzRkY2M2ZTMxMWFlNGFhZWJmZTk0NmM1ZTg1OGYwNTVhZmYxY2U1YTM3YWE3Y2M5MWFhYjE3ZTM1YyIsImF1ZCI6Im1haWxzZXJ2ZXIiLCJ1aWQiOiI4OU4zR0NuN1M1Y090WkZNRTVBeVhNbmxURFdVcnEzRmd4YWlyWWhFIn0.zuCytArbphhJn9XT_y9cBdGqDCNo68tBrtOwPIsuKNyF340SaOuZa0xarZofygytdDpLtYr56QlPTKImi-n1ZWrHkRZkwrQi5jQ-j_n2hEAL0vUToLbDnXYfc5q2w7z7X0aoCmiK8-fV7Kx4CVTM7riBgpElf6F3wNAIcX6R1ijUh6ISCL0XYsdogf8WUNZipXY-O4R7YHXdOENuOp3G48hWhxuUh9PsUqE5yxDwLsOVzCTqg9S5gxPQzF2eCN9J0I2XiIlLKvLQPIZ2Y_K7iYvVwjpNdgb4xhm9wuKoIVinYkF_6CwIzAawBWIDJAbix1IslkUPQMGbupTDtOgTiQ"
+
+# This is the string the user-facing client (e.g. Roundcube) should send via IMAP to Dovecot.
+# We include the user and the above token separated by '\1' chars as per the XOAUTH2 spec.
+xoauth2 = base64.b64encode(f"user=user1@localhost.localdomain\1auth=Bearer {token}\1\1".encode("utf-8"))
+# If changing the user above, use the new output from the below line with the contents of the AUTHENTICATE command in test/test-files/auth/imap-oauth2-auth.txt
+print("XOAUTH2 string: " + str(xoauth2))
+
+
+class HTTPRequestHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        auth = self.headers.get("Authorization")
+        if auth is None:
+            self.send_response(401)
+            self.end_headers()
+            return
+        if len(auth.split()) != 2:
+            self.send_response(401)
+            self.end_headers()
+            return
+        auth = auth.split()[1]
+        # Valid session, respond with JSON containing the expected `email` claim to match as Dovecot username:
+        if auth == token:
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+            self.wfile.write(json.dumps({
+                "email": "user1@localhost.localdomain",
+                "email_verified": True,
+                "sub": "82c1c334dcc6e311ae4aaebfe946c5e858f055aff1ce5a37aa7cc91aab17e35c"
+            }).encode("utf-8"))
+        else:
+            self.send_response(401)
+        self.end_headers()
+
+server = HTTPServer(('', 80), HTTPRequestHandler)
+print("Starting server", flush=True)
+
+try:
+    server.serve_forever()
+except KeyboardInterrupt:
+    print()
+    print("Received keyboard interrupt")
+finally:
+    print("Exiting")

test/files/auth/imap-oauth2-auth.txt

@@ -0,0 +1,4 @@
+a0 NOOP See test/config/oauth2/provider.py to generate the below XOAUTH2 string
+a1 AUTHENTICATE XOAUTH2 dXNlcj11c2VyMUBsb2NhbGhvc3QubG9jYWxkb21haW4BYXV0aD1CZWFyZXIgZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBjM01pT2lKb2RIUndPaTh2Y0hKdmRtbGtaWEl1WlhoaGJYQnNaUzUwWlhOME9qZ3dNREF2SWl3aWMzVmlJam9pT0RKak1XTXpNelJrWTJNMlpUTXhNV0ZsTkdGaFpXSm1aVGswTm1NMVpUZzFPR1l3TlRWaFptWXhZMlUxWVRNM1lXRTNZMk01TVdGaFlqRTNaVE0xWXlJc0ltRjFaQ0k2SW0xaGFXeHpaWEoyWlhJaUxDSjFhV1FpT2lJNE9VNHpSME51TjFNMVkwOTBXa1pOUlRWQmVWaE5ibXhVUkZkVmNuRXpSbWQ0WVdseVdXaEZJbjAuenVDeXRBcmJwaGhKbjlYVF95OWNCZEdxRENObzY4dEJydE93UElzdUtOeUYzNDBTYU91WmEweGFyWm9meWd5dGREcEx0WXI1NlFsUFRLSW1pLW4xWldySGtSWmt3clFpNWpRLWpfbjJoRUFMMHZVVG9MYkRuWFlmYzVxMnc3ejdYMGFvQ21pSzgtZlY3S3g0Q1ZUTTdyaUJncEVsZjZGM3dOQUljWDZSMWlqVWg2SVNDTDBYWXNkb2dmOFdVTlppcFhZLU80UjdZSFhkT0VOdU9wM0c0OGhXaHh1VWg5UHNVcUU1eXhEd0xzT1Z6Q1RxZzlTNWd4UFF6RjJlQ045SjBJMlhpSWxMS3ZMUVBJWjJZX0s3aVl2VndqcE5kZ2I0eGhtOXd1S29JVmluWWtGXzZDd0l6QWF3QldJREpBYml4MUlzbGtVUFFNR2J1cFREdE9nVGlRAQE=
+a2 EXAMINE INBOX
+a3 LOGOUT

test/files/emails/existing/added.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <added@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-added.txt
-This is a test mail.

test/files/emails/existing/alias-external.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <alias1@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-alias-external.txt
-This is a test mail.

test/files/emails/existing/alias-local.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local Alias <alias2@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-alias-local.txt
-This is a test mail.

test/files/emails/existing/alias-recipient-delimiter.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local Alias With Delimiter <alias1+test@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-alias-recipient-delimiter.txt
-This is a test mail.

test/files/emails/existing/catchall-local.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <wildcard@localdomain2.com>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-catchall-local.txt
-This is a test mail.

test/files/emails/existing/regexp-alias-external.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <bounce-always@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-regexp-alias-external.txt
-This is a test mail.

test/files/emails/existing/regexp-alias-local.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <test123@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-regexp-alias-local.txt
-This is a test mail.

test/files/emails/existing/user-and-cc-local-alias.txt

@@ -1,6 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <user1@localhost.localdomain>
-Cc: Existing Local Alias <alias2@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-user-and-cc-local-alias.txt
-This is a test mail.

test/files/emails/existing/user1.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <user1@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message existing-user1.txt
-This is a test mail.

test/files/emails/non-existing-user.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <dockermailserver@external.tld>
-To: Existing Local User <user1@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message non-existing-user.txt
-This is a test mail.

test/files/emails/rspamd/pass.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <pass@example.test>
-To: Existing Local User <user1@localhost.localdomain>
-Date: Sat, 22 May 2010 07:43:25 -0400
-Subject: Test Message rspamd/pass.txt
-This mail should pass and Rspamd should not mark it.

test/files/emails/rspamd/spam-header.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <spam-header@example.test>
-To: Existing Local User <user1@localhost.localdomain>
-Date: Sat, 21 Jan 2023 11:11:11 +0000
-Subject: Test Message rspamd-spam-header.txt
-YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X

test/files/emails/rspamd/spam.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <spam@example.test>
-To: Existing Local User <user1@localhost.localdomain>
-Date: Sat, 21 Jan 2023 11:11:11 +0000
-Subject: Test Message rspamd-spam.txt
-XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X

test/files/emails/rspamd/virus.txt

@@ -1,5 +0,0 @@
-From: Docker Mail Server <virus@example.test>
-To: Existing Local User <user1@localhost.localdomain>
-Date: Sat, 21 Jan 2023 11:11:11 +0000
-Subject: Test Message rspamd-virus.txt
-X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*

test/helper/common.bash

@@ -466,7 +466,7 @@ function _print_mail_log_for_id() {
   local MAIL_ID=${1:?Mail ID must be provided}
   local CONTAINER_NAME=$(__handle_container_name "${2:-}")
 
-  _run_in_container grep -F "${MAIL_ID}" /var/log/mail.log
+  _run_in_container grep -E "${MAIL_ID}" /var/log/mail.log
 }
 
 # A simple wrapper for netcat (`nc`). This is useful when sending
@@ -480,7 +480,7 @@ function _nc_wrapper() {
 
   [[ -v CONTAINER_NAME ]] || return 1
 
-  _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}.txt"
+  _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}"
 }
 
 # ? << Miscellaneous helper functions

test/helper/sending.bash

@@ -7,68 +7,118 @@
 # ! ATTENTION: This file is loaded by `common.sh` - do not load it yourself!
 # ! ATTENTION: This file requires helper functions from `common.sh`!
 
-# Sends a mail from localhost (127.0.0.1) to a container. To send
+# Sends an e-mail from the container named by the environment variable `CONTAINER_NAME`
-# a custom email, create a file at `test/files/<TEST FILE>`,
+# to the same or another container.
-# and provide `<TEST FILE>` as an argument to this function.
 #
-# Parameters include all options that one can supply to `swaks`
+# To send a custom email, you can
-# itself. The `--data` parameter expects a relative path from `emails/`
+#
-# where the contents will be implicitly provided to `swaks` via STDIN.
+# 1. create a file at `test/files/<TEST FILE>` and provide `<TEST FILE>` via `--data` as an argument to this function;
+# 2. use this function without the `--data` argument, in which case we provide a default;
+# 3. provide data inline (`--data <INLINE DATA>`).
+#
+# The very first parameter **may** be `--expect-rejection` - use it of you expect the mail transaction to not finish
+# successfully. All other (following) parameters include all options that one can supply to `swaks` itself.
+# As mentioned before, the `--data` parameter expects a value of either:
+#
+# - A relative path from `test/files/emails/`
+# - An "inline" data string (e.g., `Date: 1 Jan 2024\nSubject: This is a test`)
+#
+# ## Output
+#
+# This functions prints the output of the transaction that `swaks` prints.
 #
 # ## Attention
 #
 # This function assumes `CONTAINER_NAME` to be properly set (to the container
 # name the command should be executed in)!
 #
-# This function will just send the email in an "asynchronous" fashion, i.e. it will
+# This function will send the email in an "asynchronous" fashion,
-# send the email but it will not make sure the mail queue is empty after the mail
+# it will return without waiting for the Postfix mail queue to be emptied.
-# has been sent.
 function _send_email() {
-  [[ -v CONTAINER_NAME ]] || return 1
+  local RETURN_VALUE=0
+  local COMMAND_STRING
 
-  # Parameter defaults common to our testing needs:
+  function __parse_arguments() {
-  local EHLO='mail.external.tld'
+    [[ -v CONTAINER_NAME ]] || return 1
-  local FROM='user@external.tld'
-  local TO='user1@localhost.localdomain'
-  local SERVER='0.0.0.0'
-  local PORT=25
-  # Extra options for `swaks` that aren't covered by the default options above:
-  local ADDITIONAL_SWAKS_OPTIONS=()
-  # Specifically for handling `--data` option below:
-  local FINAL_SWAKS_OPTIONS=()
 
-  while [[ ${#} -gt 0 ]]; do
+    # Parameter defaults common to our testing needs:
-    case "${1}" in
+    local EHLO='mail.external.tld'
-      ( '--ehlo' )   EHLO=${2:?--ehlo given but no argument}     ; shift 2 ;;
+    local FROM='user@external.tld'
-      ( '--from' )   FROM=${2:?--from given but no argument}     ; shift 2 ;;
+    local TO='user1@localhost.localdomain'
-      ( '--to' )     TO=${2:?--to given but no argument}         ; shift 2 ;;
+    local SERVER='0.0.0.0'
-      ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;;
+    local PORT=25
-      ( '--port' )   PORT=${2:?--port given but no argument}     ; shift 2 ;;
+    # Extra options for `swaks` that aren't covered by the default options above:
-      ( '--data' )
+    local ADDITIONAL_SWAKS_OPTIONS=()
-        local TEMPLATE_FILE="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}.txt"
+    local DATA_WAS_SUPPLIED=0
-        FINAL_SWAKS_OPTIONS+=('--data')
-        FINAL_SWAKS_OPTIONS+=('-')
-        FINAL_SWAKS_OPTIONS+=('<')
-        FINAL_SWAKS_OPTIONS+=("${TEMPLATE_FILE}")
-        shift 2
-        ;;
-      ( * ) ADDITIONAL_SWAKS_OPTIONS+=("${1}") ; shift 1 ;;
-    esac
-  done
 
-  _run_in_container_bash "swaks --server ${SERVER} --port ${PORT} --ehlo ${EHLO} --from ${FROM} --to ${TO} ${ADDITIONAL_SWAKS_OPTIONS[*]} ${FINAL_SWAKS_OPTIONS[*]}"
+    while [[ ${#} -gt 0 ]]; do
+      case "${1}" in
+        ( '--ehlo' )   EHLO=${2:?--ehlo given but no argument}     ; shift 2 ;;
+        ( '--from' )   FROM=${2:?--from given but no argument}     ; shift 2 ;;
+        ( '--to' )     TO=${2:?--to given but no argument}         ; shift 2 ;;
+        ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;;
+        ( '--port' )   PORT=${2:?--port given but no argument}     ; shift 2 ;;
+        ( '--data' )
+          ADDITIONAL_SWAKS_OPTIONS+=('--data')
+          local FILE_PATH="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}"
+          if _exec_in_container_bash "[[ -e ${FILE_PATH} ]]"; then
+            ADDITIONAL_SWAKS_OPTIONS+=("@${FILE_PATH}")
+          else
+            ADDITIONAL_SWAKS_OPTIONS+=("'${2}'")
+          fi
+          shift 2
+          DATA_WAS_SUPPLIED=1
+          ;;
+        ( * ) ADDITIONAL_SWAKS_OPTIONS+=("'${1}'") ; shift 1 ;;
+      esac
+    done
+
+    if [[ ${DATA_WAS_SUPPLIED} -eq 0 ]]; then
+      # Fallback template (without the implicit `Message-Id` + `X-Mailer` headers from swaks):
+      # NOTE: It is better to let Postfix generate and append the `Message-Id` header itself,
+      #       as it will contain the Queue ID for tracking in logs (which is also returned in swaks output).
+      ADDITIONAL_SWAKS_OPTIONS+=('--data')
+      ADDITIONAL_SWAKS_OPTIONS+=("'Date: %DATE%\nTo: %TO_ADDRESS%\nFrom: %FROM_ADDRESS%\nSubject: test %DATE%\n%NEW_HEADERS%\n%BODY%\n'")
+    fi
+
+    echo "swaks --server '${SERVER}' --port '${PORT}' --ehlo '${EHLO}' --from '${FROM}' --to '${TO}' ${ADDITIONAL_SWAKS_OPTIONS[*]}"
+  }
+
+  if [[ ${1:-} == --expect-rejection ]]; then
+    shift 1
+    COMMAND_STRING=$(__parse_arguments "${@}")
+    _run_in_container_bash "${COMMAND_STRING}"
+    RETURN_VALUE=${?}
+  else
+    COMMAND_STRING=$(__parse_arguments "${@}")
+    _run_in_container_bash "${COMMAND_STRING}"
+    assert_success
+  fi
+
+  # shellcheck disable=SC2154
+  echo "${output}"
+  return "${RETURN_VALUE}"
 }
 
 # Like `_send_email` with two major differences:
 #
 # 1. this function waits for the mail to be processed; there is no asynchronicity
-#    because filtering the logs in a synchronous way is easier and safer!
+#    because filtering the logs in a synchronous way is easier and safer;
-# 2. this function prints an ID one can later filter by to check logs
+# 2. this function takes the name of a variable and inserts IDs one can later
+#    filter by to check logs.
 #
 # No. 2 is especially useful in case you send more than one email in a single
 # test file and need to assert certain log entries for each mail individually.
 #
-# This function takes the same arguments as `_send_mail`.
+# The first argument has to be the name of the variable that the e-mail ID is stored in.
+# The second argument **can** be the flag `--expect-rejection`.
+#
+# - If this flag is supplied, the function does not check whether the whole mail delivery
+#   transaction was successful. Additionally the queue ID will be retrieved differently.
+# - CAUTION: It must still be possible to `grep` for the Message-ID that Postfix
+#   generated in the mail log; otherwise this function fails.
+#
+# The rest of the arguments are the same as `_send_email`.
 #
 # ## Attention
 #
@@ -82,20 +132,35 @@ function _send_email() {
 # chosen. Sending more than one mail at any given point in time with this function
 # is UNDEFINED BEHAVIOR!
 function _send_email_and_get_id() {
-  [[ -v CONTAINER_NAME ]] || return 1
+  # Export the variable denoted by ${1} so everyone has access
+  export "${1:?Mail ID must be set for _send_email_and_get_id}"
+  # Get a "reference" to the content of the variable denoted by ${1} so we can manipulate the content
+  local -n ID_ENV_VAR_REF=${1:?}
+  # Prepare the message ID header here because we will shift away ${1} later
+  local MID="<${1}@dms-tests>"
+  # Get rid of ${1} so only the arguments for swaks remain
+  shift 1
 
-  _wait_for_empty_mail_queue_in_container
+  local QUEUE_ID
-  _send_email "${@}"
-  _wait_for_empty_mail_queue_in_container
-
-  local MAIL_ID
   # The unique ID Postfix (and other services) use may be different in length
   # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a
   # range to safely capture it.
-  MAIL_ID=$(_exec_in_container tac /var/log/mail.log              \
+  local QUEUE_ID_REGEX='[A-Z0-9]{9,12}'
-    | grep -E -m 1 'postfix/smtpd.*: [A-Z0-9]+: client=localhost' \
-    | grep -E -o '[A-Z0-9]{9,12}' || true)
 
-  assert_not_equal "${MAIL_ID}" ''
+  _wait_for_empty_mail_queue_in_container
-  echo "${MAIL_ID}"
+  local OUTPUT=$(_send_email "${@}" --header "Message-Id: ${MID}")
+  _wait_for_empty_mail_queue_in_container
+
+  # We store Postfix's queue ID first
+  ID_ENV_VAR_REF=$(_exec_in_container tac /var/log/mail.log                    \
+    | grep -E "postfix/cleanup.*: ${QUEUE_ID_REGEX}:.*message-id=${MID}" \
+    | grep -E --only-matching --max-count 1 "${QUEUE_ID_REGEX}" || :)
+  # But we also requre potential Dovecot sieve output, which requires the mesage ID,
+  # so we need to provide the message ID too.
+  ID_ENV_VAR_REF+="|${MID}"
+
+  # Last but not least, we perform plausibility checks on the IDs.
+  assert_not_equal "${ID_ENV_VAR_REF}" ''
+  run echo "${ID_ENV_VAR_REF}"
+  assert_line --regexp "^${QUEUE_ID_REGEX}\|${MID}$"
 }

test/tests/parallel/set1/dovecot/dovecot_quotas.bats

@@ -225,12 +225,9 @@ function teardown_file() { _default_teardown ; }
   sleep 10
 
   # send some big emails
-  _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded'
+  _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt'
-  assert_success
+  _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt'
-  _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded'
+  _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt'
-  assert_success
-  _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded'
-  assert_success
   # check for quota warn message existence
   run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/
   assert_success

test/tests/parallel/set1/dovecot/dovecot_sieve.bats

@@ -26,11 +26,9 @@ function setup_file() {
   _wait_for_smtp_port_in_container
 
   # Single mail sent from 'spam@spam.com' that is handled by User (relocate) and Global (copy) sieves for user1:
-  _send_email --data 'sieve/spam-folder'
+  _send_email --data 'sieve/spam-folder.txt'
-  assert_success
   # Mail for user2 triggers the sieve-pipe:
-  _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe'
+  _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe.txt'
-  assert_success
 
   _wait_for_empty_mail_queue_in_container
 }

test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats

@@ -26,8 +26,7 @@ function teardown() { _default_teardown ; }
   _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
   _wait_for_smtp_port_in_container
 
-  _send_email --data 'existing/user1'
+  _send_email
-  assert_success
   _wait_for_empty_mail_queue_in_container
 
   # Mail received should be stored as `u.1` (one file per message)
@@ -48,8 +47,7 @@ function teardown() { _default_teardown ; }
   _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
   _wait_for_smtp_port_in_container
 
-  _send_email --data 'existing/user1'
+  _send_email
-  assert_success
   _wait_for_empty_mail_queue_in_container
 
   # Mail received should be stored in `m.1` (1 or more messages)

test/tests/parallel/set1/dovecot/special_use_folders.bats

@@ -14,8 +14,7 @@ function setup_file() {
 function teardown_file() { _default_teardown ; }
 
 @test 'normal delivery works' {
-  _send_email --data 'existing/user1'
+  _send_email
-  assert_success
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new 1
 }
 
@@ -27,7 +26,7 @@ function teardown_file() { _default_teardown ; }
 }
 
 @test "(IMAP) special-use folders should be created when necessary" {
-  _nc_wrapper 'nc/imap_special_use_folders' '-w 8 0.0.0.0 143'
+  _nc_wrapper 'nc/imap_special_use_folders.txt' '-w 8 0.0.0.0 143'
   assert_output --partial 'Drafts'
   assert_output --partial 'Junk'
   assert_output --partial 'Trash'

test/tests/parallel/set1/spam_virus/clamav.bats

@@ -25,8 +25,7 @@ function setup_file() {
 
   _wait_for_service postfix
   _wait_for_smtp_port_in_container
-  _send_email --from 'virus@external.tld' --data 'amavis/virus'
+  _send_email --from 'virus@external.tld' --data 'amavis/virus.txt'
-  assert_success
   _wait_for_empty_mail_queue_in_container
 }
 

test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats

@@ -18,8 +18,7 @@ function setup_file() {
   _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
   _wait_for_smtp_port_in_container
 
-  _send_email --data 'existing/user1'
+  _send_email
-  assert_success
   _wait_for_empty_mail_queue_in_container
 }
 

test/tests/parallel/set1/spam_virus/fail2ban.bats

@@ -74,7 +74,7 @@ function teardown_file() {
   CONTAINER1_IP=$(_get_container_ip "${CONTAINER1_NAME}")
   # Trigger a ban by failing to login twice:
   for _ in {1..2}; do
-    CONTAINER_NAME=${CONTAINER2_NAME} _send_email \
+    CONTAINER_NAME=${CONTAINER2_NAME} _send_email --expect-rejection \
       --server "${CONTAINER1_IP}" \
       --port 465 \
       --auth PLAIN \

test/tests/parallel/set1/spam_virus/postgrey_enabled.bats

@@ -51,7 +51,7 @@ function teardown_file() { _default_teardown ; }
   _reload_postfix
 
   # Send test mail (it should fail to deliver):
-  _send_email --from 'user@external.tld' --port 25 --data 'postgrey'
+  _send_email --expect-rejection --from 'user@external.tld' --port 25 --data 'postgrey.txt'
   assert_failure
   assert_output --partial 'Recipient address rejected: Delayed by Postgrey'
 
@@ -67,8 +67,7 @@ function teardown_file() { _default_teardown ; }
   # Wait until `$POSTGREY_DELAY` seconds pass before trying again:
   sleep 3
   # Retry delivering test mail (it should be trusted this time):
-  _send_email --from 'user@external.tld' --port 25 --data 'postgrey'
+  _send_email --from 'user@external.tld' --port 25 --data 'postgrey.txt'
-  assert_success
 
   # Confirm postgrey permitted delivery (triplet is now trusted):
   _should_have_log_entry \
@@ -87,7 +86,7 @@ function teardown_file() { _default_teardown ; }
 #   - It'd also cause the earlier greylist test to fail.
 # - TODO: Actually confirm whitelist feature works correctly as these test cases are using a workaround:
 @test "should whitelist sender 'user@whitelist.tld'" {
-  _nc_wrapper 'nc/postgrey_whitelist' '-w 0 0.0.0.0 10023'
+  _nc_wrapper 'nc/postgrey_whitelist.txt' '-w 0 0.0.0.0 10023'
 
   _should_have_log_entry \
     'action=pass' \
@@ -96,7 +95,7 @@ function teardown_file() { _default_teardown ; }
 }
 
 @test "should whitelist recipient 'user2@otherdomain.tld'" {
-  _nc_wrapper 'nc/postgrey_whitelist_recipients' '-w 0 0.0.0.0 10023'
+  _nc_wrapper 'nc/postgrey_whitelist_recipients.txt' '-w 0 0.0.0.0 10023'
 
   _should_have_log_entry \
     'action=pass' \

test/tests/parallel/set1/spam_virus/postscreen.bats

@@ -44,7 +44,7 @@ function teardown_file() {
 # Use `nc` to send all SMTP commands at once instead (emulate a misbehaving client that should be rejected)
 # NOTE: Postscreen only runs on port 25, avoid implicit ports in test methods
 @test 'should fail send when talking out of turn' {
-  CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen' "${CONTAINER1_IP} 25"
+  CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen.txt' "${CONTAINER1_IP} 25"
   # Expected postscreen log entry:
   assert_output --partial 'Protocol error'
 
@@ -56,14 +56,10 @@ function teardown_file() {
 @test "should successfully pass postscreen and get postfix greeting message (respecting postscreen_greet_wait time)" {
   # Configure `send_email()` to send from the mail client container (CONTAINER2_NAME) via ENV override,
   # mail is sent to the DMS server container (CONTAINER1_NAME) via `--server` parameter:
-  CONTAINER_NAME=${CONTAINER2_NAME} _send_email --server "${CONTAINER1_IP}" --port 25 --data 'postscreen'
+  # TODO: Use _send_email_and_get_id when proper resolution of domain names is possible:
-  # NOTE: Cannot assert_success due to sender address not being resolvable.
+  CONTAINER_NAME=${CONTAINER2_NAME} _send_email --expect-rejection --server "${CONTAINER1_IP}" --port 25 --data 'postscreen.txt'
-  # TODO: Uncomment when proper resolution of domain names is possible:
+  # CONTAINER_NAME=${CONTAINER2_NAME} _send_email_and_get_id MAIL_ID_POSTSCREEN --server "${CONTAINER1_IP}" --data 'postscreen.txt'
-  # assert_success
+  # _print_mail_log_for_id "${MAIL_ID_POSTSCREEN}"
-
-  # TODO: Prefer this approach when `_send_email_and_get_id()` can support separate client and server containers:
-  # local MAIL_ID=$(_send_email_and_get_id --port 25 --data 'postscreen')
-  # _print_mail_log_for_id "${MAIL_ID}"
   # assert_output --partial "stored mail into mailbox 'INBOX'"
 
   _run_in_container cat /var/log/mail.log

test/tests/parallel/set1/spam_virus/rspamd_full.bats

@@ -43,16 +43,24 @@ function setup_file() {
   _wait_for_service postfix
   _wait_for_smtp_port_in_container
 
-  # We will send 3 emails: the first one should pass just fine; the second one should
+  # ref: https://rspamd.com/doc/gtube_patterns.html
-  # be rejected due to spam; the third one should be rejected due to a virus.
+  local GTUBE_SUFFIX='*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X'
-  export MAIL_ID1=$(_send_email_and_get_id --from 'rspamd-pass@example.test' --data 'rspamd/pass')
-  export MAIL_ID2=$(_send_email_and_get_id --from 'rspamd-spam@example.test' --data 'rspamd/spam')
-  export MAIL_ID3=$(_send_email_and_get_id --from 'rspamd-virus@example.test' --data 'rspamd/virus')
-  export MAIL_ID4=$(_send_email_and_get_id --from 'rspamd-spam-header@example.test' --data 'rspamd/spam-header')
 
-  for ID in MAIL_ID{1,2,3,4}; do
+  # We will send 4 emails:
-    [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; }
+  # 1. The first one should pass just fine
-  done
+  _send_email_and_get_id MAIL_ID_PASS
+  # 2. The second one should be rejected (GTUBE pattern)
+  _send_email_and_get_id MAIL_ID_REJECT --expect-rejection --body "XJS${GTUBE_SUFFIX}"
+  # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern)
+  # shellcheck disable=SC2016
+  _send_email_and_get_id MAIL_ID_VIRUS --expect-rejection \
+    --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'
+  # 4. The fourth one will receive an added header (GTUBE pattern)
+  _send_email_and_get_id MAIL_ID_HEADER --body "YJS${GTUBE_SUFFIX}"
+
+  _run_in_container cat /var/log/mail.log
+  assert_success
+  refute_output --partial 'inet:localhost:11332: Connection refused'
 }
 
 function teardown_file() { _default_teardown ; }
@@ -104,7 +112,7 @@ function teardown_file() { _default_teardown ; }
 @test 'normal mail passes fine' {
   _service_log_should_contain_string 'rspamd' 'F \(no action\)'
 
-  _print_mail_log_for_id "${MAIL_ID1}"
+  _print_mail_log_for_id "${MAIL_ID_PASS}"
   assert_output --partial "stored mail into mailbox 'INBOX'"
 
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
@@ -114,7 +122,7 @@ function teardown_file() { _default_teardown ; }
   _service_log_should_contain_string 'rspamd' 'S \(reject\)'
   _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"'
 
-  _print_mail_log_for_id "${MAIL_ID2}"
+  _print_mail_log_for_id "${MAIL_ID_REJECT}"
   assert_output --partial 'milter-reject'
   assert_output --partial '5.7.1 Gtube pattern'
 
@@ -125,7 +133,7 @@ function teardown_file() { _default_teardown ; }
   _service_log_should_contain_string 'rspamd' 'T \(reject\)'
   _service_log_should_contain_string 'rspamd' 'reject "ClamAV FOUND VIRUS "Eicar-Signature"'
 
-  _print_mail_log_for_id "${MAIL_ID3}"
+  _print_mail_log_for_id "${MAIL_ID_VIRUS}"
   assert_output --partial 'milter-reject'
   assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"'
   refute_output --partial "stored mail into mailbox 'INBOX'"
@@ -214,7 +222,7 @@ function teardown_file() { _default_teardown ; }
   _service_log_should_contain_string 'rspamd' 'S \(add header\)'
   _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"'
 
-  _print_mail_log_for_id "${MAIL_ID4}"
+  _print_mail_log_for_id "${MAIL_ID_HEADER}"
   assert_output --partial "fileinto action: stored mail into mailbox 'Junk'"
 
   _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1
@@ -256,7 +264,7 @@ function teardown_file() { _default_teardown ; }
 
   # Move an email to the "Junk" folder from "INBOX"; the first email we
   # sent should pass fine, hence we can now move it.
-  _nc_wrapper 'nc/rspamd_imap_move_to_junk' '0.0.0.0 143'
+  _nc_wrapper 'nc/rspamd_imap_move_to_junk.txt' '0.0.0.0 143'
   sleep 1 # wait for the transaction to finish
 
   _run_in_container cat /var/log/mail/mail.log
@@ -270,7 +278,7 @@ function teardown_file() { _default_teardown ; }
   # Move an email to the "INBOX" folder from "Junk"; there should be two mails
   # in the "Junk" folder, since the second email we sent during setup should
   # have landed in the Junk folder already.
-  _nc_wrapper 'nc/rspamd_imap_move_to_inbox' '0.0.0.0 143'
+  _nc_wrapper 'nc/rspamd_imap_move_to_inbox.txt' '0.0.0.0 143'
   sleep 1 # wait for the transaction to finish
 
   _run_in_container cat /var/log/mail/mail.log

test/tests/parallel/set1/spam_virus/spam_junk_folder.bats

@@ -95,7 +95,7 @@ function teardown() { _default_teardown ; }
 function _should_send_spam_message() {
   _wait_for_smtp_port_in_container
   _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis
-  _send_email --from 'spam@external.tld' --data 'amavis/spam'
+  _send_email --from 'spam@external.tld' --data 'amavis/spam.txt'
 }
 
 function _should_be_received_by_amavis() {

test/tests/parallel/set3/container_configuration/hostname.bats

@@ -207,7 +207,7 @@ function _should_have_correct_mail_headers() {
   # (eg: OVERRIDE_HOSTNAME or `--hostname mail --domainname example.test`)
   local EXPECTED_HOSTNAME=${3:-${EXPECTED_FQDN}}
 
-  _send_email --from 'user@external.tld' --data 'existing/user1'
+  _send_email --from 'user@external.tld'
   _wait_for_empty_mail_queue_in_container
   _count_files_in_directory_in_container '/var/mail/localhost.localdomain/user1/new/' '1'
 

test/tests/parallel/set3/mta/dsn.bats

@@ -49,9 +49,9 @@ function teardown_file() {
 
   # TODO replace with _send_email as soon as it supports DSN
   # TODO ref: https://github.com/jetmore/swaks/issues/41
-  _nc_wrapper 'emails/nc_raw/dsn/unauthenticated'
+  _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt'
-  _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465'
+  _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465'
-  _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587'
+  _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587'
   _wait_for_empty_mail_queue_in_container
 
   _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log
@@ -62,7 +62,7 @@ function teardown_file() {
 @test "should only send a DSN when requested from ports 465/587" {
   export CONTAINER_NAME=${CONTAINER2_NAME}
 
-  _nc_wrapper 'emails/nc_raw/dsn/unauthenticated'
+  _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt'
   _wait_for_empty_mail_queue_in_container
 
   # DSN requests can now only be made on ports 465 and 587,
@@ -74,8 +74,8 @@ function teardown_file() {
   assert_failure
 
   # These ports are excluded via master.cf.
-  _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465'
+  _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465'
-  _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587'
+  _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587'
   _wait_for_empty_mail_queue_in_container
 
   _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log
@@ -85,9 +85,9 @@ function teardown_file() {
 @test "should never send a DSN" {
   export CONTAINER_NAME=${CONTAINER3_NAME}
 
-  _nc_wrapper 'emails/nc_raw/dsn/unauthenticated'
+  _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt'
-  _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465'
+  _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465'
-  _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587'
+  _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587'
   _wait_for_empty_mail_queue_in_container
 
   # DSN requests are rejected regardless of origin.

test/tests/parallel/set3/mta/lmtp_ip.bats

@@ -38,7 +38,7 @@ function teardown_file() { _default_teardown ; }
 
 @test "delivers mail to existing account" {
   _wait_for_smtp_port_in_container
-  _send_email --data 'existing/user1' # send a test email
+  _send_email
 
   # Verify delivery was successful, log line should look similar to:
   # postfix/lmtp[1274]: 0EA424ABE7D9: to=<user1@localhost.localdomain>, relay=127.0.0.1[127.0.0.1]:24, delay=0.13, delays=0.07/0.01/0.01/0.05, dsn=2.0.0, status=sent (250 2.0.0 <user1@localhost.localdomain> ixPpB+Zvv2P7BAAAUi6ngw Saved)

test/tests/parallel/set3/mta/privacy.bats

@@ -26,11 +26,10 @@ function teardown_file() { _default_teardown ; }
 # this test covers https://github.com/docker-mailserver/docker-mailserver/issues/681
 @test "(Postfix) remove privacy details of the sender" {
   _send_email \
-    --port 587 -tls --auth LOGIN \
+    --port 587 -tls --auth PLAIN \
     --auth-user user1@localhost.localdomain \
     --auth-password mypassword \
-    --data 'privacy'
+    --data 'privacy.txt'
-  assert_success
 
   _run_until_success_or_timeout 120 _exec_in_container_bash '[[ -d /var/mail/localhost.localdomain/user1/new ]]'
   assert_success

test/tests/parallel/set3/mta/smtp_delivery.bats

@@ -63,46 +63,43 @@ function setup_file() {
 
   # TODO: Move to clamav tests (For use when ClamAV is enabled):
   # _repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /var/run/clamav/clamd.ctl
-  # _send_email --from 'virus@external.tld' --data 'amavis/virus'
+  # _send_email --from 'virus@external.tld' --data 'amavis/virus.txt'
 
   # Required for 'delivers mail to existing alias':
-  _send_email --to alias1@localhost.localdomain --data 'existing/alias-external'
+  _send_email --to alias1@localhost.localdomain --header "Subject: Test Message existing-alias-external"
   # Required for 'delivers mail to existing alias with recipient delimiter':
-  _send_email --to alias1~test@localhost.localdomain --data 'existing/alias-recipient-delimiter'
+  _send_email --to alias1~test@localhost.localdomain --header 'Subject: Test Message existing-alias-recipient-delimiter'
   # Required for 'delivers mail to existing catchall':
-  _send_email --to wildcard@localdomain2.com --data 'existing/catchall-local'
+  _send_email --to wildcard@localdomain2.com --header 'Subject: Test Message existing-catchall-local'
   # Required for 'delivers mail to regexp alias':
-  _send_email --to test123@localhost.localdomain --data 'existing/regexp-alias-local'
+  _send_email --to test123@localhost.localdomain --header 'Subject: Test Message existing-regexp-alias-local'
 
   # Required for 'rejects mail to unknown user':
-  _send_email --to nouser@localhost.localdomain --data 'non-existing-user'
+  _send_email --expect-rejection --to nouser@localhost.localdomain
+  assert_failure
   # Required for 'redirects mail to external aliases':
-  _send_email --to bounce-always@localhost.localdomain --data 'existing/regexp-alias-external'
+  _send_email --to bounce-always@localhost.localdomain
-  _send_email --to alias2@localhost.localdomain --data 'existing/alias-local'
+  _send_email --to alias2@localhost.localdomain
   # Required for 'rejects spam':
-  _send_email --from 'spam@external.tld' --data 'amavis/spam'
+  _send_email --from 'spam@external.tld' --data 'amavis/spam.txt'
 
   # Required for 'delivers mail to existing account':
-  _send_email --data 'existing/user1'
+  _send_email --header 'Subject: Test Message existing-user1'
-  assert_success
   _send_email --to user2@otherdomain.tld
-  assert_success
   _send_email --to user3@localhost.localdomain
-  assert_success
+  _send_email --to added@localhost.localdomain --header 'Subject: Test Message existing-added'
-  _send_email --to added@localhost.localdomain --data 'existing/added'
+  _send_email \
-  assert_success
+    --to user1@localhost.localdomain \
-  _send_email --to user1@localhost.localdomain --data 'existing/user-and-cc-local-alias'
+    --header 'Subject: Test Message existing-user-and-cc-local-alias' \
-  assert_success
+    --cc 'alias2@localhost.localdomain'
-  _send_email --data 'sieve/spam-folder'
+  _send_email --data 'sieve/spam-folder.txt'
-  assert_success
+  _send_email --to user2@otherdomain.tld --data 'sieve/pipe.txt'
-  _send_email --to user2@otherdomain.tld --data 'sieve/pipe'
-  assert_success
   _run_in_container_bash 'sendmail root < /tmp/docker-mailserver-test/emails/sendmail/root-email.txt'
   assert_success
 }
 
 function _unsuccessful() {
-  _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword
+  _send_email --expect-rejection --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword --quit-after AUTH
   assert_failure
   assert_output --partial 'authentication failed'
   assert_output --partial 'No authentication type succeeded'
@@ -110,7 +107,6 @@ function _unsuccessful() {
 
 function _successful() {
   _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password mypassword --quit-after AUTH
-  assert_success
   assert_output --partial 'Authentication successful'
 }
 

test/tests/serial/mail_pop3.bats

@@ -24,12 +24,12 @@ function teardown_file() { _default_teardown ; }
 }
 
 @test 'authentication works' {
-  _nc_wrapper 'auth/pop3-auth' '-w 1 0.0.0.0 110'
+  _nc_wrapper 'auth/pop3-auth.txt' '-w 1 0.0.0.0 110'
   assert_success
 }
 
 @test 'added user authentication works' {
-  _nc_wrapper 'auth/added-pop3-auth' '-w 1 0.0.0.0 110'
+  _nc_wrapper 'auth/added-pop3-auth.txt' '-w 1 0.0.0.0 110'
   assert_success
 }
 

test/tests/serial/mail_with_imap.bats

@@ -21,7 +21,7 @@ function setup_file() {
 function teardown_file() { _default_teardown ; }
 
 @test '(Dovecot) LDAP RIMAP connection and authentication works' {
-  _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143'
+  _nc_wrapper 'auth/imap-auth.txt' '-w 1 0.0.0.0 143'
   assert_success
 }
 
@@ -31,8 +31,8 @@ function teardown_file() { _default_teardown ; }
 }
 
 @test '(SASLauthd) RIMAP SMTP authentication works' {
-  _send_email \
+  _send_email --expect-rejection \
-    --auth LOGIN \
+    --auth PLAIN \
     --auth-user user1@localhost.localdomain \
     --auth-password mypassword \
     --quit-after AUTH
@@ -41,20 +41,18 @@ function teardown_file() { _default_teardown ; }
 
   _send_email \
     --port 465 \
-    --auth LOGIN \
+    --auth PLAIN \
     --auth-user user1@localhost.localdomain \
     --auth-password mypassword \
     --quit-after AUTH
-  assert_success
   assert_output --partial 'Authentication successful'
 
   _send_email \
     --port 587 \
-    --auth LOGIN \
+    --auth PLAIN \
     --auth-user user1@localhost.localdomain \
     --auth-password mypassword \
     --quit-after AUTH
-  assert_success
   assert_output --partial 'Authentication successful'
 }
 

test/tests/serial/mail_with_ldap.bats

@@ -248,7 +248,7 @@ function teardown() {
 
 # dovecot
 @test "dovecot: ldap imap connection and authentication works" {
-  _nc_wrapper 'auth/imap-ldap-auth' '-w 1 0.0.0.0 143'
+  _nc_wrapper 'auth/imap-ldap-auth.txt' '-w 1 0.0.0.0 143'
   assert_success
 }
 
@@ -326,25 +326,26 @@ function teardown() {
 @test "spoofing (with LDAP): rejects sender forging" {
   _wait_for_smtp_port_in_container_to_respond dms-test_ldap
 
-  _send_email \
+  _send_email --expect-rejection \
-    --port 465 -tlsc --auth LOGIN \
+    --port 465 -tlsc --auth PLAIN \
     --auth-user some.user@localhost.localdomain \
     --auth-password secret \
     --ehlo mail \
     --from ldap@localhost.localdomain \
-    --data 'auth/ldap-smtp-auth-spoofed'
+    --data 'auth/ldap-smtp-auth-spoofed.txt'
+  assert_failure
   assert_output --partial 'Sender address rejected: not owned by user'
 }
 
 @test "spoofing (with LDAP): accepts sending as alias" {
   _send_email \
-    --port 465 -tlsc --auth LOGIN \
+    --port 465 -tlsc --auth PLAIN \
     --auth-user some.user@localhost.localdomain \
     --auth-password secret \
     --ehlo mail \
     --from postmaster@localhost.localdomain \
     --to some.user@localhost.localdomain \
-    --data 'auth/ldap-smtp-auth-spoofed-alias'
+    --data 'auth/ldap-smtp-auth-spoofed-alias.txt'
   assert_output --partial 'End data with'
 }
 
@@ -353,20 +354,21 @@ function teardown() {
   # Template used has invalid AUTH: https://github.com/docker-mailserver/docker-mailserver/pull/3006#discussion_r1073321432
   skip 'TODO: This test seems to have been broken from the start (?)'
 
-  _send_email \
+  _send_email --expect-rejection \
-    --port 465 -tlsc --auth LOGIN \
+    --port 465 -tlsc --auth PLAIN \
     --auth-user some.user.email@localhost.localdomain \
     --auth-password secret \
     --ehlo mail \
     --from randomspoofedaddress@localhost.localdomain \
     --to some.user@localhost.localdomain \
-    --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception'
+    --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt'
+  assert_failure
   assert_output --partial 'Sender address rejected: not owned by user'
 }
 
 @test "saslauthd: ldap smtp authentication" {
-  _send_email \
+  _send_email --expect-rejection \
-    --auth LOGIN \
+    --auth PLAIN \
     --auth-user some.user@localhost.localdomain \
     --auth-password wrongpassword \
     --quit-after AUTH
@@ -379,12 +381,11 @@ function teardown() {
     --auth-user some.user@localhost.localdomain \
     --auth-password secret \
     --quit-after AUTH
-  assert_success
   assert_output --partial 'Authentication successful'
 
   _send_email \
     --port 587 -tls \
-    --auth LOGIN \
+    --auth PLAIN \
     --auth-user some.user@localhost.localdomain \
     --auth-password secret \
     --quit-after AUTH

test/tests/serial/mail_with_oauth2.bats

@@ -0,0 +1,66 @@
+load "${REPOSITORY_ROOT}/test/helper/setup"
+load "${REPOSITORY_ROOT}/test/helper/common"
+
+BATS_TEST_NAME_PREFIX='[OAuth2] '
+CONTAINER1_NAME='dms-test_oauth2'
+CONTAINER2_NAME='dms-test_oauth2_provider'
+
+function setup_file() {
+  export DMS_TEST_NETWORK='test-network-oauth2'
+  export DMS_DOMAIN='example.test'
+  export FQDN_MAIL="mail.${DMS_DOMAIN}"
+  export FQDN_OAUTH2="oauth2.${DMS_DOMAIN}"
+
+  # Link the test containers to separate network:
+  # NOTE: If the network already exists, test will fail to start.
+  docker network create "${DMS_TEST_NETWORK}"
+
+  # Setup local oauth2 provider service:
+  docker run --rm -d --name "${CONTAINER2_NAME}" \
+    --hostname "${FQDN_OAUTH2}" \
+    --network "${DMS_TEST_NETWORK}" \
+    --volume "${REPOSITORY_ROOT}/test/config/oauth2/:/app/" \
+    docker.io/library/python:latest \
+    python /app/provider.py
+
+  _run_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER2_NAME} 2>&1 | grep 'Starting server'"
+
+  #
+  # Setup DMS container
+  #
+
+  # Add OAUTH2 configuration so that Dovecot can reach out to our mock provider (CONTAINER2)
+  local ENV_OAUTH2_CONFIG=(
+    --env ENABLE_OAUTH2=1
+    --env OAUTH2_INTROSPECTION_URL=http://oauth2.example.test/userinfo/
+  )
+
+  export CONTAINER_NAME=${CONTAINER1_NAME}
+  local CUSTOM_SETUP_ARGUMENTS=(
+    "${ENV_OAUTH2_CONFIG[@]}"
+
+    --hostname "${FQDN_MAIL}"
+    --network "${DMS_TEST_NETWORK}"
+  )
+
+  _init_with_defaults
+  _common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
+  _wait_for_tcp_port_in_container 143
+
+  # Set default implicit container fallback for helpers:
+  export CONTAINER_NAME=${CONTAINER1_NAME}
+}
+
+function teardown_file() {
+  docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}"
+  docker network rm "${DMS_TEST_NETWORK}"
+}
+
+
+@test "oauth2: imap connect and authentication works" {
+  # An initial connection needs to be made first, otherwise the auth attempt fails
+  _run_in_container_bash 'nc -vz 0.0.0.0 143'
+
+  _nc_wrapper 'auth/imap-oauth2-auth.txt' '-w 1 0.0.0.0 143'
+  assert_output --partial 'Examine completed'
+}

test/tests/serial/tests.bats

@@ -80,12 +80,12 @@ function teardown_file() { _default_teardown ; }
 }
 
 @test "imap: authentication works" {
-  _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143'
+  _nc_wrapper 'auth/imap-auth.txt' '-w 1 0.0.0.0 143'
   assert_success
 }
 
 @test "imap: added user authentication works" {
-  _nc_wrapper 'auth/added-imap-auth' '-w 1 0.0.0.0 143'
+  _nc_wrapper 'auth/added-imap-auth.txt' '-w 1 0.0.0.0 143'
   assert_success
 }
 
@@ -293,13 +293,13 @@ EOF
 
   # An authenticated user cannot use an envelope sender (MAIL FROM)
   # address they do not own according to `main.cf:smtpd_sender_login_maps` lookup
-  _send_email \
+  _send_email --expect-rejection \
-    --port 465 -tlsc --auth LOGIN \
+    --port 465 -tlsc --auth PLAIN \
     --auth-user added@localhost.localdomain \
     --auth-password mypassword \
     --ehlo mail \
     --from user2@localhost.localdomain \
-    --data 'auth/added-smtp-auth-spoofed'
+    --data 'auth/added-smtp-auth-spoofed.txt'
   assert_output --partial 'Sender address rejected: not owned by user'
 }
 
@@ -310,12 +310,12 @@ EOF
   # to each table. Address is authorized when a result that maps to
   # the DMS account is returned.
   _send_email \
-    --port 465 -tlsc --auth LOGIN \
+    --port 465 -tlsc --auth PLAIN \
     --auth-user user1@localhost.localdomain \
     --auth-password mypassword \
     --ehlo mail \
     --from alias1@localhost.localdomain \
-    --data 'auth/added-smtp-auth-spoofed-alias'
+    --data 'auth/added-smtp-auth-spoofed-alias.txt'
   assert_success
   assert_output --partial 'End data with'
 }

test/tests/serial/vmail-id.bats

@@ -20,7 +20,7 @@ function setup_file() {
 function teardown_file() { _default_teardown ; }
 
 @test 'should successfully deliver mail' {
-  _send_email --data 'existing/user1'
+  _send_email --header 'Subject: Test Message existing-user1'
   _wait_for_empty_mail_queue_in_container
 
   # Should be successfully sent (received) by Postfix:
@@ -31,7 +31,7 @@ function teardown_file() { _default_teardown ; }
 
   # Verify successful delivery via Dovecot to `/var/mail` account by searching for the subject:
   _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep -R \
-    'Subject: Test Message existing-user1.txt' \
+    'Subject: Test Message existing-user1' \
     '/var/mail/localhost.localdomain/user1/new/'
   assert_success
   _should_output_number_of_lines 1