Merge branch 'master' into tests/normalizations

CHANGELOG.md

@@ -10,6 +10,11 @@ All notable changes to this project will be documented in this file. The format
 
 - **Tests**:
   - existing tests were streamlined and simplified, which is a follow-up of [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732)
+- **Internal:**
+  - tests: Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752))
+  - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750))
+- **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)):
+  - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996)
 
 ## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0)
 

Dockerfile

@@ -106,6 +106,7 @@ EOF
 # -----------------------------------------------
 
 COPY target/rspamd/local.d/ /etc/rspamd/local.d/
+COPY target/rspamd/scores.d/* /etc/rspamd/scores.d/
 
 # -----------------------------------------------
 # --- LDAP & SpamAssassin's Cron ----------------

target/postfix/main.cf

@@ -5,6 +5,9 @@ biff = no
 append_dot_mydomain = no
 readme_directory = no
 
+# Disabled as not compatible with Dovecot
+smtputf8_enable = no
+
 # Basic configuration
 # myhostname =
 alias_maps = hash:/etc/aliases

target/rspamd/local.d/actions.conf

@@ -1,9 +1,12 @@
 # documentation: https://rspamd.com/doc/configuration/metrics.html#actions
 # and            https://rspamd.com/doc/configuration/metrics.html
 
-#greylist = 4;
+# These values work in conjunction with the symbol scores in
-#add_header = 6;
+# `scores.d/*.conf`. When adjusting them, make sure to understand
-#rewrite_subject = 7;
+# and to be able to explain the impact on the whole system.
-#reject = 15;
+greylist = 4;
+add_header = 6;
+rewrite_subject = 7;
+reject = 11;
 
 subject = "***SPAM*** %s"

target/rspamd/scores.d/policies_group.conf

@@ -0,0 +1,108 @@
+# Please refer to
+# https://github.com/docker-mailserver/docker-mailserver/issues/3690
+# for understanding this file and its scores' values.
+
+symbols = {
+    # SPF
+    "R_SPF_ALLOW" {
+        weight = -1;
+        description = "SPF verification allows sending";
+        groups = ["spf"];
+    }
+    "R_SPF_NA" {
+        weight = 1.5;
+        description = "Missing SPF record";
+        one_shot = true;
+        groups = ["spf"];
+    }
+    "R_SPF_SOFTFAIL" {
+        weight = 2.5;
+        description = "SPF verification soft-failed";
+        groups = ["spf"];
+    }
+    "R_SPF_FAIL" {
+        weight = 4.5;
+        description = "SPF verification failed";
+        groups = ["spf"];
+    }
+
+    "R_SPF_NEUTRAL" { # == R_SPF_NA
+        weight = 1.5;
+        description = "SPF policy is neutral";
+        groups = ["spf"];
+    }
+    "R_SPF_DNSFAIL" { # == R_SPF_SOFTFAIL
+        weight = 2.5;
+        description = "SPF DNS failure";
+        groups = ["spf"];
+    }
+    "R_SPF_PERMFAIL" { # == R_SPF_FAIL
+        weight = 4.5;
+        description = "SPF record is malformed or persistent DNS error";
+        groups = ["spf"];
+    }
+
+    # DKIM
+    "R_DKIM_ALLOW" {
+        weight = -1;
+        description = "DKIM verification succeed";
+        one_shot = true;
+        groups = ["dkim"];
+    }
+    "R_DKIM_NA" {
+        weight = 0;
+        description = "Missing DKIM signature";
+        one_shot = true;
+        groups = ["dkim"];
+    }
+    "R_DKIM_TEMPFAIL" {
+        weight = 1.5;
+        description = "DKIM verification soft-failed";
+        groups = ["dkim"];
+    }
+    "R_DKIM_PERMFAIL" {
+        weight = 4.5;
+        description = "DKIM verification hard-failed (invalid)";
+        groups = ["dkim"];
+    }
+
+    "R_DKIM_REJECT" { # == R_DKIM_PERMFAIL
+        weight = 4.5;
+        description = "DKIM verification failed";
+        one_shot = true;
+        groups = ["dkim"];
+    }
+
+    # DMARC
+    "DMARC_NA" {
+        weight = 1;
+        description = "No DMARC record";
+        groups = ["dmarc"];
+    }
+    "DMARC_POLICY_QUARANTINE" {
+        weight = 1.5;
+        description = "DMARC quarantine policy";
+        groups = ["dmarc"];
+    }
+    "DMARC_POLICY_REJECT" {
+        weight = 2;
+        description = "DMARC reject policy";
+        groups = ["dmarc"];
+    }
+
+    "DMARC_POLICY_ALLOW" { # no equivalent
+        weight = -1;
+        description = "DMARC permit policy";
+        groups = ["dmarc"];
+    }
+    "DMARC_POLICY_ALLOW_WITH_FAILURES" { # no equivalent
+        weight = -0.5;
+        description = "DMARC permit policy with DKIM/SPF failure";
+        groups = ["dmarc"];
+    }
+    "DMARC_POLICY_SOFTFAIL" { # == DMARC_POLICY_QUARANTINE
+        weight = 1.5;
+        description = "DMARC soft-failed";
+        groups = ["dmarc"];
+    }
+}

target/scripts/startup/setup.d/postfix.sh

@@ -19,9 +19,6 @@ function _setup_postfix_early() {
     postconf "inet_protocols = ${POSTFIX_INET_PROTOCOLS}"
   fi
 
-  __postfix__log 'trace' "Disabling SMTPUTF8 support"
-  postconf 'smtputf8_enable = no'
-
   __postfix__log 'trace' "Configuring SASLauthd"
   if [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && [[ ! -f /etc/postfix/sasl/smtpd.conf ]]; then
     cat >/etc/postfix/sasl/smtpd.conf << EOF

test/tests/parallel/set1/spam_virus/clamav.bats

@@ -32,7 +32,7 @@ function setup_file() {
 function teardown_file() { _default_teardown ; }
 
 @test 'log files exist at /var/log/mail directory' {
-  _run_in_container_bash "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log' | wc -l"
+  _run_in_container_bash "ls -1 /var/log/mail/ | grep -c -E 'clamav|freshclam|mail.log'"
   assert_success
   assert_output 3
 }

test/tests/parallel/set3/mta/smtp_delivery.bats

@@ -270,6 +270,10 @@ function _successful() {
     --protocol ESMTP \
     --server mail.example.test \
     --quit-after FIRST-EHLO
+
+  # Ensure the output is actually related to what we want to refute against:
+  assert_output --partial 'EHLO mail.external.tld'
+  assert_output --partial '221 2.0.0 Bye'
   refute_output --partial 'SMTPUTF8'
 }
 

test/tests/parallel/set3/scripts/setup_cli.bats

@@ -204,12 +204,12 @@ function teardown_file() { _default_teardown ; }
   run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user2 51M
   assert_failure
 
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1"
   assert_success
 
   run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user@example.com 26M
   assert_success
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:26M\$' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:26M\$' | grep 1"
   assert_success
 
   run grep "quota_user2@example.com" "${TEST_TMP_CONFIG}/dovecot-quotas.cf"
@@ -220,12 +220,12 @@ function teardown_file() { _default_teardown ; }
 @test "delquota" {
   run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user@example.com 12M
   assert_success
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1"
   assert_success
 
   run ./setup.sh -c "${CONTAINER_NAME}" quota del unknown@domain.com
   assert_failure
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1"
   assert_success
 
   run ./setup.sh -c "${CONTAINER_NAME}" quota del quota_user@example.com
@@ -260,13 +260,13 @@ function teardown_file() { _default_teardown ; }
   ./setup.sh -c "${CONTAINER_NAME}" relay add-domain example3.org smtp.relay.com 587
 
   # check adding
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example1.org\s\+\[smtp.relay1.com\]:2525' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example1.org\s\+\[smtp.relay1.com\]:2525' | grep 1"
   assert_success
   # test default port
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example2.org\s\+\[smtp.relay2.com\]:25' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example2.org\s\+\[smtp.relay2.com\]:25' | grep 1"
   assert_success
   # test modifying
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example3.org\s\+\[smtp.relay.com\]:587' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example3.org\s\+\[smtp.relay.com\]:587' | grep 1"
   assert_success
 }
 
@@ -276,16 +276,16 @@ function teardown_file() { _default_teardown ; }
   ./setup.sh -c "${CONTAINER_NAME}" relay add-auth example2.org smtp_user2 smtp_pass_new
 
   # test adding
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -e '^@example.org\s\+smtp_user:smtp_pass' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -c -e '^@example.org\s\+smtp_user:smtp_pass' | grep 1"
   assert_success
   # test updating
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -e '^@example2.org\s\+smtp_user2:smtp_pass_new' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -c -e '^@example2.org\s\+smtp_user2:smtp_pass_new' | grep 1"
   assert_success
 }
 
 @test "relay exclude-domain" {
   ./setup.sh -c "${CONTAINER_NAME}" relay exclude-domain example.org
 
-  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example.org\s*$' | wc -l | grep 1"
+  run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example.org\s*$' | grep 1"
   assert_success
 }

test/tests/serial/tests.bats

@@ -264,7 +264,7 @@ EOF
 #
 
 @test "amavis: config overrides" {
-  _run_in_container_bash "grep 'Test Verification' /etc/amavis/conf.d/50-user | wc -l"
+  _run_in_container_bash "grep -c 'Test Verification' /etc/amavis/conf.d/50-user"
   assert_success
   assert_output 1
 }