From f28fce9cc432f1f447bd963d9e54e44bcf2c27dd Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:43:21 +0200 Subject: [PATCH 001/125] rspamd: disable checks for authenticated users (#3440) Co-authored-by: Casper Co-authored-by: William Desportes --- docs/content/config/environment.md | 9 +++++++++ docs/content/config/security/rspamd.md | 11 ++++++----- mailserver.env | 7 +++++++ target/rspamd/local.d/settings.conf | 12 ++++++++++++ .../scripts/startup/setup.d/security/rspamd.sh | 18 +++++++++++++++++- target/scripts/startup/variables-stack.sh | 1 + .../parallel/set1/spam_virus/rspamd_full.bats | 17 +++++++++++++++-- .../set1/spam_virus/rspamd_partly.bats | 10 ++++++++++ 8 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 target/rspamd/local.d/settings.conf diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index ab9171ac..b81cef12 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -338,6 +338,15 @@ The purpose of this setting is to opt-out of starting an internal Redis instance - 0 => Disabled - 1 => Enabled +##### RSPAMD_CHECK_AUTHENTICATED + +This settings controls whether checks should be performed on emails coming from authenticated users (i.e. most likely outgoing emails). The default value is `0` in order to align better with SpamAssassin. **We recommend** reading through [the Rspamd documentation on scanning outbound emails][rspamd-scanning-outbound] though to decide for yourself whether you need and want this feature. + +- **0** => No checks will be performed for authenticated users +- 1 => All default checks will be performed for authenticated users + +[rspamd-scanning-outbound]: https://rspamd.com/doc/tutorials/scanning_outbound.html + ##### RSPAMD_GREYLISTING Controls whether the [Rspamd Greylisting module][rspamd-greylisting-module] is enabled. This module can further assist in avoiding spam emails by [greylisting] e-mails with a certain spam score. diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 44674e9e..d1d0987e 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -21,11 +21,12 @@ The following environment variables are related to Rspamd: 1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) 2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) -3. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) -4. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) -5. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) -6. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -7. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +3. [`RSPAMD_CHECK_AUTHENTICATED`](../environment.md#rspamd_check_authenticated) +4. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) +5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) +6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) +7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) +8. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. diff --git a/mailserver.env b/mailserver.env index 038e23b1..cb040b9f 100644 --- a/mailserver.env +++ b/mailserver.env @@ -142,6 +142,13 @@ ENABLE_RSPAMD_REDIS= # 1 => enabled RSPAMD_LEARN=0 +# This settings controls whether checks should be performed on emails coming +# from authenticated users (i.e. most likely outgoing emails). The default value +# is `0` in order to align better with SpamAssassin. We recommend reading +# through https://rspamd.com/doc/tutorials/scanning_outbound.html though to +# decide for yourself whether you need and want this feature. +RSPAMD_CHECK_AUTHENTICATED=0 + # Controls whether the Rspamd Greylisting module is enabled. # This module can further assist in avoiding spam emails by greylisting # e-mails with a certain spam score. diff --git a/target/rspamd/local.d/settings.conf b/target/rspamd/local.d/settings.conf new file mode 100644 index 00000000..4f635e74 --- /dev/null +++ b/target/rspamd/local.d/settings.conf @@ -0,0 +1,12 @@ +# documentation: https://rspamd.com/doc/configuration/settings.html + +# DMS::SED_TAG::1::START +# Disable all checks for authenticated users +authenticated { + priority = high; + authenticated = yes; + apply { + groups_enabled = []; + } +} +# DMS::SED_TAG::1::END diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 400e1a82..4ece646b 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -14,6 +14,7 @@ function _setup_rspamd() { __rspamd__setup_learning __rspamd__setup_greylisting __rspamd__setup_hfilter_group + __rspamd__setup_check_authenticated __rspamd__handle_user_modules_adjustments # must run last __rspamd__log 'trace' '---------- Setup finished ----------' @@ -250,7 +251,8 @@ function __rspamd__setup_hfilter_group() { if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]]; then __rspamd__log 'debug' 'Hfilter (group) module is enabled' # Check if we received a number first - if _env_var_expect_integer 'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' && [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]]; then + if _env_var_expect_integer 'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' \ + && [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]]; then __rspamd__log 'trace' "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}" sed -i -E \ "s|(.*score =).*(# __TAG__HFILTER_HOSTNAME_UNKNOWN)|\1 ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}; \2|g" \ @@ -264,6 +266,20 @@ function __rspamd__setup_hfilter_group() { fi } +function __rspamd__setup_check_authenticated() { + local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf" + if _env_var_expect_zero_or_one 'RSPAMD_CHECK_AUTHENTICATED' \ + && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]] + then + __rspamd__log 'debug' 'Content checks for authenticated users are disabled' + else + __rspamd__log 'debug' 'Enabling content checks for authenticated users' + sed -i -E \ + '/DMS::SED_TAG::1::START/{:a;N;/DMS::SED_TAG::1::END/!ba};/authenticated/d' \ + "${MODULE_FILE}" + fi +} + # Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. # To get a detailed explanation of the commands and how the file works, visit # https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index c2a52d5c..d6c5453c 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -51,6 +51,7 @@ function __environment_variables_general_setup() { VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" + VARS[RSPAMD_CHECK_AUTHENTICATED]="${RSPAMD_CHECK_AUTHENTICATED:=0}" VARS[RSPAMD_GREYLISTING]="${RSPAMD_GREYLISTING:=0}" VARS[RSPAMD_HFILTER]="${RSPAMD_HFILTER:=1}" VARS[RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE]="${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE:=6}" diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 536ce43d..3fbf59d2 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -25,6 +25,7 @@ function setup_file() { --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=1 --env RSPAMD_LEARN=1 + --env RSPAMD_CHECK_AUTHENTICATED=0 --env RSPAMD_GREYLISTING=1 --env RSPAMD_HFILTER=1 --env RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=7 @@ -292,10 +293,22 @@ function teardown_file() { _default_teardown ; } } @test 'hfilter group module is configured correctly' { - _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' + local MODULE_FILE='/etc/rspamd/local.d/hfilter_group.conf' + _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" assert_success - _run_in_container grep '__TAG__HFILTER_HOSTNAME_UNKNOWN' /etc/rspamd/local.d/hfilter_group.conf + _run_in_container grep '__TAG__HFILTER_HOSTNAME_UNKNOWN' "${MODULE_FILE}" assert_success assert_output --partial 'score = 7;' } + +@test 'checks on authenticated users are disabled' { + local MODULE_FILE='/etc/rspamd/local.d/settings.conf' + _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" + assert_success + + _run_in_container grep -E -A 6 'authenticated \{' "${MODULE_FILE}" + assert_success + assert_output --partial 'authenticated = yes;' + assert_output --partial 'groups_enabled = [];' +} diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats index 0c7983f8..9fc8af31 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -24,6 +24,7 @@ function setup_file() { --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=0 --env RSPAMD_LEARN=0 + --env RSPAMD_CHECK_AUTHENTICATED=1 --env RSPAMD_GREYLISTING=0 --env RSPAMD_HFILTER=0 ) @@ -85,3 +86,12 @@ function teardown_file() { _default_teardown ; } _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' assert_failure } + +@test 'checks on authenticated users are enabled' { + local MODULE_FILE='/etc/rspamd/local.d/settings.conf' + _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" + assert_success + + _run_in_container grep -E 'authenticated \{' "${MODULE_FILE}" + assert_failure +} From 8f971713360c4620046c7f223fe66df9ea800253 Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 14 Aug 2023 01:58:54 +0200 Subject: [PATCH 002/125] compose.yaml: Add comment about disabled authentication on port 25 (#3464) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 37c42c7c..e9c49596 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,7 +9,7 @@ services: # https://docker-mailserver.github.io/docker-mailserver/latest/config/security/understanding-the-ports/ # To avoid conflicts with yaml base-60 float, DO NOT remove the quotation marks. ports: - - "25:25" # SMTP (explicit TLS => STARTTLS) + - "25:25" # SMTP (explicit TLS => STARTTLS, Authentication is DISABLED => use port 465/587 instead) - "143:143" # IMAP4 (explicit TLS => STARTTLS) - "465:465" # ESMTP (implicit TLS) - "587:587" # ESMTP (explicit TLS => STARTTLS) From 5bada0a83b0875b36934e1de856b724979c83346 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 17 Aug 2023 14:33:34 +1200 Subject: [PATCH 003/125] tests: Refactor LDAP tests to current conventions (#3483) * tests: Switch to setup helper and conventions * tests: Adapt run command to new conventions - We have two helper methods with implicit `CONTAINER_NAME` reference, which is a bit more DRY and improves readability. - `wc -l` + `assert_output 1` converted to use helper `_should_output_number_of_lines 1` - `DOMAIN` var changed from `my-domain.com` to local testing domain `example.test`. * tests: Refactor `setup_file()` - Test wide ENV defined at the top - OpenLDAP build and run logic grouped together. Added notes, network alias and tty not required. - Extracted out special LDAP Postfix/Dovecot ENV into separate array. LDAP specific provisioning / auth ENV also included, with comments + linebreak to better group related ENV. - Likewise additional ENV to support test cases has been extracted to a separate array with additional comments for context. - Those two arrays are expanded back into the main `CUSTOM_SETUP_ARGUMENTS` that configure hostname and network for the DMS container. * tests: Refactor the LDAP account table query testcase - Covers 3 accounts to test from LDAP. - 2 are the same query against users/aliases/groups tables in Postfix, only differing by account queried (and expected as returned result). - 1 separate test to ensure a difference in config is supported correctly. - Extracted repeated test logic into a helper method. - Added additional context in comments about the creation source of these LDAP accounts and their related Postfix config / interaction. Direct reference to special case PR (since `git blame` will be less useful). * tests: Use iteration for `grep` setting checks More DRY approach. With a bit more helpful failure context via `assert_output` (_and only grepping the key_). Simpler to grok what's being covered. * tests: DRY test email delivery A bit more verbose with the new helper method. `test-email.txt` template is only used by the LDAP test, as is the `sendmail` command. Helper will take two args to support the testcases, but at a later date should be refactored to be consistent with the `_send_email()` helper (_which presently uses `nc` that is required for plain-text mail/auth, otherwise we'd have used `openssl`, bigger refactor required_). * tests: Slight revisions and relocating testcases - Dovecot quota plugin testcase revised to check files exist instead of rely on `ls` failure. - Moved Postfix quota plugin testcase into prior dovecot testcase for quota plugin check. Better error output by only querying the `smtpd_recipient_restrictions` setting (_which should be the only one configured for the service_). - Moved the saslauthd and pflogsumm testcases (_no changes beyond revised comments_) above the `ATTENTION` comment, and one testcase below the comment that belonged to that group. * tests: Simplify openldap `docker build` command - `--no-cache` was creating a new image on the Docker host each time the test is run. Shouldn't be any need to build without cache. - No need to use `pushd` + `popd`, can just provide the path context directly, and the `./Dockerfile` is an implicit default thus `-f` not required either. Additionally removed the old `checking` prefix from testcase names. * tests: Move LDAP specific config into `test/config/ldap/` - No changes to any of these config files, just better isolation as not relevant to any other tests. - Section heading in `setup_file()` added to distinguish the remainder of the function is dedicated to the DMS container setup. - Comment providing some context about the `mv` to maintainers, this should be done after defaults are initialized but before starting up the container. * chore: Appease the lint gods * Apply suggestions from code review --- .../ldap}/docker-openldap/Dockerfile | 0 .../bootstrap/ldif/01_mail-tree.ldif | 0 .../bootstrap/ldif/02_user-email.ldif | 0 .../03_user-email-other-primary-domain.ldif | 0 .../ldif/04_user-email-different-uid.ldif | 0 .../bootstrap/schema/mmc/postfix-book.schema | 0 .../{ => ldap/overrides}/ldap-aliases.cf | 0 .../{ => ldap/overrides}/ldap-groups.cf | 0 .../config/{ => ldap/overrides}/ldap-users.cf | 0 test/tests/serial/mail_with_ldap.bats | 426 ++++++++++-------- 10 files changed, 247 insertions(+), 179 deletions(-) rename test/{ => config/ldap}/docker-openldap/Dockerfile (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/01_mail-tree.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/02_user-email.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/schema/mmc/postfix-book.schema (100%) rename test/config/{ => ldap/overrides}/ldap-aliases.cf (100%) rename test/config/{ => ldap/overrides}/ldap-groups.cf (100%) rename test/config/{ => ldap/overrides}/ldap-users.cf (100%) diff --git a/test/docker-openldap/Dockerfile b/test/config/ldap/docker-openldap/Dockerfile similarity index 100% rename from test/docker-openldap/Dockerfile rename to test/config/ldap/docker-openldap/Dockerfile diff --git a/test/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/01_mail-tree.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif diff --git a/test/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/02_user-email.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif diff --git a/test/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif diff --git a/test/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif diff --git a/test/docker-openldap/bootstrap/schema/mmc/postfix-book.schema b/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema similarity index 100% rename from test/docker-openldap/bootstrap/schema/mmc/postfix-book.schema rename to test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema diff --git a/test/config/ldap-aliases.cf b/test/config/ldap/overrides/ldap-aliases.cf similarity index 100% rename from test/config/ldap-aliases.cf rename to test/config/ldap/overrides/ldap-aliases.cf diff --git a/test/config/ldap-groups.cf b/test/config/ldap/overrides/ldap-groups.cf similarity index 100% rename from test/config/ldap-groups.cf rename to test/config/ldap/overrides/ldap-groups.cf diff --git a/test/config/ldap-users.cf b/test/config/ldap/overrides/ldap-users.cf similarity index 100% rename from test/config/ldap-users.cf rename to test/config/ldap/overrides/ldap-users.cf diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 75768bbb..57251aa1 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -1,245 +1,313 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +BATS_TEST_NAME_PREFIX='[LDAP] ' +CONTAINER1_NAME='dms-test_ldap' +CONTAINER2_NAME='dms-test_ldap_provider' function setup_file() { - pushd test/docker-openldap/ || return 1 - docker build -f Dockerfile -t ldap --no-cache . - popd || return 1 - - export DOMAIN='my-domain.com' + export DMS_TEST_NETWORK='test-network-ldap' + export DOMAIN='example.test' export FQDN_MAIL="mail.${DOMAIN}" export FQDN_LDAP="ldap.${DOMAIN}" + # LDAP is provisioned with two domains (via `.ldif` files) unrelated to the FQDN of DMS: export FQDN_LOCALHOST_A='localhost.localdomain' export FQDN_LOCALHOST_B='localhost.otherdomain' - export DMS_TEST_NETWORK='test-network-ldap' + # Link the test containers to separate network: # NOTE: If the network already exists, test will fail to start. docker network create "${DMS_TEST_NETWORK}" - docker run -d --name ldap_for_mail \ + # Setup local openldap service: + # NOTE: Building via Dockerfile is required? Image won't accept read-only if it needs to adjust permissions for bootstrap files. + # TODO: Upstream image is no longer maintained, may want to migrate? + docker build -t dms-openldap test/config/ldap/docker-openldap/ + + docker run -d --name "${CONTAINER2_NAME}" \ --env LDAP_DOMAIN="${FQDN_LOCALHOST_A}" \ - --network "${DMS_TEST_NETWORK}" \ - --network-alias 'ldap' \ --hostname "${FQDN_LDAP}" \ - --tty \ - ldap # Image name - - # _setup_ldap uses _replace_by_env_in_file with ENV vars like DOVECOT_TLS with a prefix (eg. DOVECOT_ or LDAP_) - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container .) - docker run -d --name mail_with_ldap \ - -v "${PRIVATE_CONFIG}:/tmp/docker-mailserver" \ - -v "$(pwd)/test/test-files:/tmp/docker-mailserver-test:ro" \ - -e DOVECOT_PASS_FILTER="(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))" \ - -e DOVECOT_TLS=no \ - -e DOVECOT_USER_FILTER="(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))" \ - -e ACCOUNT_PROVISIONER=LDAP \ - -e PFLOGSUMM_TRIGGER=logrotate \ - -e ENABLE_SASLAUTHD=1 \ - -e LDAP_BIND_DN=cn=admin,dc=localhost,dc=localdomain \ - -e LDAP_BIND_PW=admin \ - -e LDAP_QUERY_FILTER_ALIAS="(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))" \ - -e LDAP_QUERY_FILTER_DOMAIN="(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))" \ - -e LDAP_QUERY_FILTER_GROUP="(&(mailGroupMember=%s)(mailEnabled=TRUE))" \ - -e LDAP_QUERY_FILTER_SENDERS="(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(uniqueIdentifier=some.user.id))" \ - -e LDAP_QUERY_FILTER_USER="(&(mail=%s)(mailEnabled=TRUE))" \ - -e LDAP_START_TLS=no \ - -e LDAP_SEARCH_BASE=ou=people,dc=localhost,dc=localdomain \ - -e LDAP_SERVER_HOST=ldap \ - -e PERMIT_DOCKER=container \ - -e POSTMASTER_ADDRESS="postmaster@${FQDN_LOCALHOST_A}" \ - -e REPORT_RECIPIENT=1 \ - -e SASLAUTHD_MECHANISMS=ldap \ - -e SPOOF_PROTECTION=1 \ - -e SSL_TYPE='snakeoil' \ --network "${DMS_TEST_NETWORK}" \ - --hostname "${FQDN_MAIL}" \ - --tty \ - "${NAME}" # Image name + dms-openldap - wait_for_smtp_port_in_container mail_with_ldap + # + # Setup DMS container + # + + local ENV_LDAP_CONFIG=( + # Configure for LDAP account provisioner and alternative to Dovecot SASL: + --env ACCOUNT_PROVISIONER=LDAP + --env ENABLE_SASLAUTHD=1 + --env SASLAUTHD_MECHANISMS=ldap + + # ENV to configure LDAP configs for Dovecot + Postfix: + # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): + # Dovecot: + --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + --env DOVECOT_TLS=no + --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + # Postfix: + --env LDAP_BIND_DN='cn=admin,dc=localhost,dc=localdomain' + --env LDAP_BIND_PW='admin' + --env LDAP_QUERY_FILTER_ALIAS='(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))' + --env LDAP_QUERY_FILTER_DOMAIN='(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))' + --env LDAP_QUERY_FILTER_GROUP='(&(mailGroupMember=%s)(mailEnabled=TRUE))' + --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(uniqueIdentifier=some.user.id))' + --env LDAP_QUERY_FILTER_USER='(&(mail=%s)(mailEnabled=TRUE))' + --env LDAP_SEARCH_BASE='ou=people,dc=localhost,dc=localdomain' + --env LDAP_SERVER_HOST="${FQDN_LDAP}" + --env LDAP_START_TLS=no + ) + + # Extra ENV needed to support specific testcases: + local ENV_SUPPORT=( + --env PERMIT_DOCKER=container # Required for attempting SMTP auth on port 25 via nc + # Required for openssl commands to be successul: + # NOTE: snakeoil cert is created (for `docker-mailserver.invalid`) via Debian post-install script for Postfix package. + # TODO: Use proper TLS cert + --env SSL_TYPE='snakeoil' + + # TODO; All below are questionable value to LDAP tests? + --env POSTMASTER_ADDRESS="postmaster@${FQDN_LOCALHOST_A}" # TODO: Only required because LDAP accounts use unrelated domain part. FQDN_LOCALHOST_A / ldif files can be adjusted to FQDN_MAIL + --env PFLOGSUMM_TRIGGER=logrotate + --env REPORT_RECIPIENT=1 # TODO: Invalid value, should be a recipient address (if not default postmaster), remove? + --env SPOOF_PROTECTION=1 + ) + + local CUSTOM_SETUP_ARGUMENTS=( + --hostname "${FQDN_MAIL}" + --network "${DMS_TEST_NETWORK}" + + "${ENV_LDAP_CONFIG[@]}" + "${ENV_SUPPORT[@]}" + ) + + # Set default implicit container fallback for helpers: + export CONTAINER_NAME=${CONTAINER1_NAME} + + _init_with_defaults + # NOTE: `test/config/` has now been duplicated, can move test specific files to host-side `/tmp/docker-mailserver`: + mv "${TEST_TMP_CONFIG}/ldap/overrides/"*.cf "${TEST_TMP_CONFIG}/" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_smtp_port_in_container } function teardown_file() { - docker rm -f ldap_for_mail mail_with_ldap + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" docker network rm "${DMS_TEST_NETWORK}" } # postfix -@test "checking postfix: ldap lookup works correctly" { - run docker exec mail_with_ldap /bin/sh -c "postmap -q some.user@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-users.cf" - assert_success - assert_output "some.user@${FQDN_LOCALHOST_A}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q postmaster@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-aliases.cf" - assert_success - assert_output "some.user@${FQDN_LOCALHOST_A}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q employees@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-groups.cf" - assert_success - assert_output "some.user@${FQDN_LOCALHOST_A}" - - # Test of the user part of the domain is not the same as the uniqueIdentifier part in the ldap - run docker exec mail_with_ldap /bin/sh -c "postmap -q some.user.email@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-users.cf" - assert_success - assert_output "some.user.email@${FQDN_LOCALHOST_A}" +# NOTE: Each of the 3 user accounts tested below are defined in separate LDIF config files, +# Those are bundled into the locally built OpenLDAP Dockerfile. +@test "postfix: ldap lookup works correctly" { + _should_exist_in_ldap_tables "some.user@${FQDN_LOCALHOST_A}" # Test email receiving from a other domain then the primary domain of the mailserver - run docker exec mail_with_ldap /bin/sh -c "postmap -q some.other.user@${FQDN_LOCALHOST_B} ldap:/etc/postfix/ldap-users.cf" + _should_exist_in_ldap_tables "some.other.user@${FQDN_LOCALHOST_B}" + + # Should not require `uniqueIdentifier` to match the local part of `mail` (`.ldif` defined settings): + # REF: https://github.com/docker-mailserver/docker-mailserver/pull/642#issuecomment-313916384 + # NOTE: This account has no `mailAlias` or `mailGroupMember` defined in it's `.ldif`. + local MAIL_ACCOUNT="some.user.email@${FQDN_LOCALHOST_A}" + _run_in_container postmap -q "${MAIL_ACCOUNT}" ldap:/etc/postfix/ldap-users.cf assert_success - assert_output "some.other.user@${FQDN_LOCALHOST_B}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q postmaster@${FQDN_LOCALHOST_B} ldap:/etc/postfix/ldap-aliases.cf" - assert_success - assert_output "some.other.user@${FQDN_LOCALHOST_B}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q employees@${FQDN_LOCALHOST_B} ldap:/etc/postfix/ldap-groups.cf" - assert_success - assert_output "some.other.user@${FQDN_LOCALHOST_B}" + assert_output "${MAIL_ACCOUNT}" } -@test "checking postfix: ldap custom config files copied" { - run docker exec mail_with_ldap /bin/sh -c "grep '# Testconfig for ldap integration' /etc/postfix/ldap-users.cf" - assert_success +# Custom LDAP config files support: +# TODO: Compare to provided configs and if they're just including a test comment, +# could just copy the config and append without carrying a separate test config? +@test "postfix: ldap custom config files copied" { + local LDAP_CONFIGS_POSTFIX=( + /etc/postfix/ldap-users.cf + /etc/postfix/ldap-groups.cf + /etc/postfix/ldap-aliases.cf + ) - run docker exec mail_with_ldap /bin/sh -c "grep '# Testconfig for ldap integration' /etc/postfix/ldap-groups.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep '# Testconfig for ldap integration' /etc/postfix/ldap-aliases.cf" - assert_success + for LDAP_CONFIG in "${LDAP_CONFIGS_POSTFIX[@]}"; do + _run_in_container grep '# Testconfig for ldap integration' "${LDAP_CONFIG}" + assert_success + done } -@test "checking postfix: ldap config overwrites success" { - run docker exec mail_with_ldap /bin/sh -c "grep 'server_host = ldap' /etc/postfix/ldap-users.cf" - assert_success +@test "postfix: ldap config overwrites success" { + local LDAP_SETTINGS_POSTFIX=( + "server_host = ${FQDN_LDAP}" + 'start_tls = no' + 'search_base = ou=people,dc=localhost,dc=localdomain' + 'bind_dn = cn=admin,dc=localhost,dc=localdomain' + ) - run docker exec mail_with_ldap /bin/sh -c "grep 'start_tls = no' /etc/postfix/ldap-users.cf" - assert_success + for LDAP_SETTING in "${LDAP_SETTINGS_POSTFIX[@]}"; do + # "${LDAP_SETTING%=*}" is to match only the key portion of the var (helpful for assert_output error messages) + # NOTE: `start_tls = no` is a default setting, but the white-space differs when ENV `LDAP_START_TLS` is not set explicitly. + _run_in_container grep "${LDAP_SETTING%=*}" /etc/postfix/ldap-users.cf + assert_output "${LDAP_SETTING}" + assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'search_base = ou=people,dc=localhost,dc=localdomain' /etc/postfix/ldap-users.cf" - assert_success + _run_in_container grep "${LDAP_SETTING%=*}" /etc/postfix/ldap-groups.cf + assert_output "${LDAP_SETTING}" + assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'bind_dn = cn=admin,dc=localhost,dc=localdomain' /etc/postfix/ldap-users.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'server_host = ldap' /etc/postfix/ldap-groups.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'start_tls = no' /etc/postfix/ldap-groups.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'search_base = ou=people,dc=localhost,dc=localdomain' /etc/postfix/ldap-groups.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'bind_dn = cn=admin,dc=localhost,dc=localdomain' /etc/postfix/ldap-groups.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'server_host = ldap' /etc/postfix/ldap-aliases.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'start_tls = no' /etc/postfix/ldap-aliases.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'search_base = ou=people,dc=localhost,dc=localdomain' /etc/postfix/ldap-aliases.cf" - assert_success - - run docker exec mail_with_ldap /bin/sh -c "grep 'bind_dn = cn=admin,dc=localhost,dc=localdomain' /etc/postfix/ldap-aliases.cf" - assert_success + _run_in_container grep "${LDAP_SETTING%=*}" /etc/postfix/ldap-aliases.cf + assert_output "${LDAP_SETTING}" + assert_success + done } # dovecot -@test "checking dovecot: ldap imap connection and authentication works" { - run docker exec mail_with_ldap /bin/sh -c "nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-ldap-auth.txt" +@test "dovecot: ldap imap connection and authentication works" { + _run_in_container_bash 'nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-ldap-auth.txt' assert_success } -@test "checking dovecot: ldap mail delivery works" { - run docker exec mail_with_ldap /bin/sh -c "sendmail -f user@external.tld some.user@${FQDN_LOCALHOST_A} < /tmp/docker-mailserver-test/email-templates/test-email.txt" - sleep 10 - run docker exec mail_with_ldap /bin/sh -c "grep -R 'This is a test mail.' /var/mail/${FQDN_LOCALHOST_A}/some.user/new/ | wc -l" - assert_success - assert_output 1 +@test "dovecot: ldap mail delivery works" { + _should_successfully_deliver_mail_to "some.user@${FQDN_LOCALHOST_A}" "/var/mail/${FQDN_LOCALHOST_A}/some.user/new/" + + # Should support delivering to a local recipient with a different domain (and disjoint mail location): + # NOTE: Mail is delivered to location defined in `.ldif` (an account config setting, either `mailHomeDirectory` or `mailStorageDirectory`). + # `some.other.user` has been configured to use a mailbox domain different from it's address domain part, hence the difference here: + _should_successfully_deliver_mail_to "some.other.user@${FQDN_LOCALHOST_B}" "/var/mail/${FQDN_LOCALHOST_A}/some.other.user/new/" } -@test "checking dovecot: ldap mail delivery works for a different domain then the mailserver" { - run docker exec mail_with_ldap /bin/sh -c "sendmail -f user@external.tld some.other.user@${FQDN_LOCALHOST_B} < /tmp/docker-mailserver-test/email-templates/test-email.txt" - sleep 10 - run docker exec mail_with_ldap /bin/sh -c "ls -A /var/mail/${FQDN_LOCALHOST_A}/some.other.user/new | wc -l" - assert_success - assert_output 1 +@test "dovecot: ldap config overwrites success" { + local LDAP_SETTINGS_DOVECOT=( + "uris = ldap://${FQDN_LDAP}" + 'tls = no' + 'base = ou=people,dc=localhost,dc=localdomain' + 'dn = cn=admin,dc=localhost,dc=localdomain' + ) + + for LDAP_SETTING in "${LDAP_SETTINGS_DOVECOT[@]}"; do + _run_in_container grep "${LDAP_SETTING%=*}" /etc/dovecot/dovecot-ldap.conf.ext + assert_output "${LDAP_SETTING}" + assert_success + done } -@test "checking dovecot: ldap config overwrites success" { - run docker exec mail_with_ldap /bin/sh -c "grep 'uris = ldap://ldap' /etc/dovecot/dovecot-ldap.conf.ext" - assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'tls = no' /etc/dovecot/dovecot-ldap.conf.ext" - assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'base = ou=people,dc=localhost,dc=localdomain' /etc/dovecot/dovecot-ldap.conf.ext" - assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'dn = cn=admin,dc=localhost,dc=localdomain' /etc/dovecot/dovecot-ldap.conf.ext" +# Requires ENV `POSTMASTER_ADDRESS` +# NOTE: Not important to LDAP feature tests? +@test "dovecot: postmaster address" { + _run_in_container grep "postmaster_address = postmaster@${FQDN_LOCALHOST_A}" /etc/dovecot/conf.d/15-lda.conf assert_success } -@test "checking dovecot: postmaster address" { - run docker exec mail_with_ldap /bin/sh -c "grep 'postmaster_address = postmaster@${FQDN_LOCALHOST_A}' /etc/dovecot/conf.d/15-lda.conf" - assert_success -} - -@test "checking dovecot: quota plugin is disabled" { - run docker exec mail_with_ldap /bin/sh -c "grep '\$mail_plugins quota' /etc/dovecot/conf.d/10-mail.conf" - assert_failure - run docker exec mail_with_ldap /bin/sh -c "grep '\$mail_plugins imap_quota' /etc/dovecot/conf.d/20-imap.conf" - assert_failure - run docker exec mail_with_ldap ls /etc/dovecot/conf.d/90-quota.conf - assert_failure - run docker exec mail_with_ldap ls /etc/dovecot/conf.d/90-quota.conf.disab - assert_success -} - -@test "checking postfix: dovecot quota absent in postconf" { - run docker exec mail_with_ldap /bin/bash -c "postconf | grep 'check_policy_service inet:localhost:65265'" +# NOTE: `target/scripts/startup/setup.d/dovecot.sh` should prevent enabling the quotas feature when using LDAP: +@test "dovecot: quota plugin is disabled" { + # Dovecot configs have not enabled the quota plugins: + _run_in_container grep "\$mail_plugins quota" /etc/dovecot/conf.d/10-mail.conf assert_failure + _run_in_container grep "\$mail_plugins imap_quota" /etc/dovecot/conf.d/20-imap.conf + assert_failure + + # Dovecot Quota config only present with disabled extension: + _run_in_container_bash '[[ -f /etc/dovecot/conf.d/90-quota.conf ]]' + assert_failure + _run_in_container_bash '[[ -f /etc/dovecot/conf.d/90-quota.conf.disab ]]' + assert_success + + # Postfix quotas policy service not configured in `main.cf`: + _run_in_container postconf smtpd_recipient_restrictions + refute_output --partial 'check_policy_service inet:localhost:65265' } -@test "checking spoofing (with LDAP): rejects sender forging" { - wait_for_smtp_port_in_container_to_respond mail_with_ldap - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed.txt | grep 'Sender address rejected: not owned by user'" +@test "saslauthd: sasl ldap authentication works" { + _run_in_container testsaslauthd -u some.user -p secret assert_success } -# ATTENTION: these tests must come after "checking dovecot: ldap mail delivery works" since they will deliver an email which skews the count in said test, leading to failure -@test "checking spoofing: accepts sending as alias (with LDAP)" { - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-alias.txt | grep 'End data with'" +# Requires ENV `PFLOGSUMM_TRIGGER=logrotate` +@test "pflogsumm delivery" { + # Verify default sender is `mailserver-report` when ENV `PFLOGSUMM_SENDER` + `REPORT_SENDER` are unset: + # NOTE: Mail is sent from Postfix (configured hostname used as domain part) + _run_in_container grep "mailserver-report@${FQDN_MAIL}" /etc/logrotate.d/maillog + assert_success + + # When `LOGROTATE_INTERVAL` is unset, the default should be configured as `weekly`: + _run_in_container grep 'weekly' /etc/logrotate.d/maillog assert_success } -@test "checking spoofing: uses senders filter" { + +# ATTENTION: Remaining tests must come after "dovecot: ldap mail delivery works" since the below tests would affect the expected count (by delivering extra mail), +# Thus not friendly for running testcases in this file in parallel + +# Requires ENV `SPOOF_PROTECTION=1` for the expected assert_output +@test "spoofing (with LDAP): rejects sender forging" { + _wait_for_smtp_port_in_container_to_respond dms-test_ldap + + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed.txt' + assert_output --partial 'Sender address rejected: not owned by user' +} + +@test "spoofing (with LDAP): accepts sending as alias" { + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-alias.txt' + assert_output --partial 'End data with' +} + +@test "spoofing (with LDAP): uses senders filter" { # skip introduced with #3006, changing port 25 to 465 + # Template used has invalid AUTH: https://github.com/docker-mailserver/docker-mailserver/pull/3006#discussion_r1073321432 skip 'TODO: This test seems to have been broken from the start (?)' - - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt | grep 'Sender address rejected: not owned by user'" - assert_success + + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt' + assert_output --partial 'Sender address rejected: not owned by user' } -# saslauthd -@test "checking saslauthd: sasl ldap authentication works" { - run docker exec mail_with_ldap bash -c "testsaslauthd -u some.user -p secret" - assert_success -} +@test "saslauthd: ldap smtp authentication" { + # Requires ENV `PERMIT_DOCKER=container` + _send_email 'auth/sasl-ldap-smtp-auth' '-w 5 0.0.0.0 25' + assert_output --partial 'Error: authentication not enabled' -@test "checking saslauthd: ldap smtp authentication" { - run docker exec mail_with_ldap /bin/sh -c "nc -w 5 0.0.0.0 25 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt | grep 'Error: authentication not enabled'" - assert_success - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt | grep 'Authentication successful'" - assert_success - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt | grep 'Authentication successful'" - assert_success + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + assert_output --partial 'Authentication successful' + + _run_in_container_bash 'openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + assert_output --partial 'Authentication successful' } # -# Pflogsumm delivery check +# Test helper methods: # -@test "checking pflogsum delivery" { - # checking default sender is correctly set when env variable not defined - run docker exec mail_with_ldap grep "mailserver-report@${FQDN_MAIL}" /etc/logrotate.d/maillog - assert_success +function _should_exist_in_ldap_tables() { + local MAIL_ACCOUNT=${1:?Mail account is required} + local DOMAIN_PART="${MAIL_ACCOUNT#*@}" - # checking default logrotation setup - run docker exec mail_with_ldap grep "weekly" /etc/logrotate.d/maillog + # Each LDAP config file sets `query_filter` to lookup a key in LDAP (values defined in `.ldif` test files) + # `mail` (ldap-users), `mailAlias` (ldap-aliases), `mailGroupMember` (ldap-groups) + # `postmap` is queried with the mail account address, and the LDAP service should respond with + # `result_attribute` which is the LDAP `mail` value (should match what we'r'e quering `postmap` with) + + _run_in_container postmap -q "${MAIL_ACCOUNT}" ldap:/etc/postfix/ldap-users.cf assert_success + assert_output "${MAIL_ACCOUNT}" + + # Check which account has the `postmaster` virtual alias: + _run_in_container postmap -q "postmaster@${DOMAIN_PART}" ldap:/etc/postfix/ldap-aliases.cf + assert_success + assert_output "${MAIL_ACCOUNT}" + + _run_in_container postmap -q "employees@${DOMAIN_PART}" ldap:/etc/postfix/ldap-groups.cf + assert_success + assert_output "${MAIL_ACCOUNT}" +} + +# NOTE: `test-email.txt` is only used for these two LDAP tests with `sendmail` command. +# The file excludes sender/recipient addresses, thus not usable with `_send_email()` helper (`nc` command)? +# TODO: Could probably adapt? +function _should_successfully_deliver_mail_to() { + local SENDER_ADDRESS='user@external.tld' + local RECIPIENT_ADDRESS=${1:?Recipient address is required} + local MAIL_STORAGE_RECIPIENT=${2:?Recipient storage location is required} + local MAIL_TEMPLATE='/tmp/docker-mailserver-test/email-templates/test-email.txt' + + _run_in_container_bash "sendmail -f ${SENDER_ADDRESS} ${RECIPIENT_ADDRESS} < ${MAIL_TEMPLATE}" + _wait_for_empty_mail_queue_in_container + + _run_in_container grep -R 'This is a test mail.' "${MAIL_STORAGE_RECIPIENT}" + assert_success + _should_output_number_of_lines 1 + + # NOTE: Prevents compatibility for running testcases in parallel (for same container) when the count could become racey: + _count_files_in_directory_in_container "${MAIL_STORAGE_RECIPIENT}" 1 } From bb2038e8c65eaeaef20c61694fa68041de0a8d25 Mon Sep 17 00:00:00 2001 From: H4R0 Date: Mon, 21 Aug 2023 00:32:26 +0200 Subject: [PATCH 004/125] feat: Allow marking spam as read via a sieve filter (ENV `MARK_SPAM_AS_READ=1`) (#3489) * add MARK_SPAM_AS_READ environment variable * review changes Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> * update unit test --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 +++ docs/content/config/environment.md | 12 +++++++++ docs/content/config/security/rspamd.md | 1 + mailserver.env | 3 +++ target/scripts/start-mailserver.sh | 1 + .../scripts/startup/setup.d/security/misc.sh | 25 +++++++++++++++++++ target/scripts/startup/variables-stack.sh | 1 + .../set1/spam_virus/spam_junk_folder.bats | 23 +++++++++++++++++ 8 files changed, 70 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1332976..028bffe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Added + +- New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) + ## [v12.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v12.1.0) ### Added diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index b81cef12..bea02ecb 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -309,6 +309,18 @@ will be automatically moved to the Junk folder (with the help of a Sieve script) - 0 => Spam messages will be delivered in the mailbox. - **1** => Spam messages will be delivered in the `Junk` folder. +##### MARK_SPAM_AS_READ + +Enable to treat received spam as "read" (_avoids notification to MUA client of new mail_). + +Mail is received as spam when it has been marked with either header: + +1. `X-Spam: Yes` (_by Rspamd_) +2. `X-Spam-Flag: YES` (_by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) + +- **0** => disabled +- 1 => Spam messages will be marked as read + #### Rspamd ##### ENABLE_RSPAMD diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index d1d0987e..7876b998 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -27,6 +27,7 @@ The following environment variables are related to Rspamd: 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) 8. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. diff --git a/mailserver.env b/mailserver.env index cb040b9f..6f270af4 100644 --- a/mailserver.env +++ b/mailserver.env @@ -368,6 +368,9 @@ ENABLE_SPAMASSASSIN_KAM=0 # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required) MOVE_SPAM_TO_JUNK=1 +# spam messages wil be marked as read +MARK_SPAM_AS_READ=0 + # add spam info headers if at, or above that level: SA_TAG=2.0 diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index a002c6c3..e41ae71e 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -48,6 +48,7 @@ function _register_functions() { _register_setup_function '_setup_dovecot_dhparam' _register_setup_function '_setup_dovecot_quota' _register_setup_function '_setup_spam_to_junk' + _register_setup_function '_setup_spam_mark_as_read' fi case "${ACCOUNT_PROVISIONER}" in diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index b1fa4860..78c1e60a 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -271,3 +271,28 @@ EOF _log 'debug' 'Spam emails will not be moved to the Junk folder' fi } + +function _setup_spam_mark_as_read() { + if [[ ${MARK_SPAM_AS_READ} -eq 1 ]]; then + _log 'debug' 'Spam emails will be marked as read' + mkdir -p /usr/lib/dovecot/sieve-global/after/ + + # Header support: `X-Spam-Flag` (SpamAssassin), `X-Spam` (Rspamd) + cat >/usr/lib/dovecot/sieve-global/after/spam_mark_as_read.sieve << EOF +require ["mailbox","imap4flags"]; + +if anyof (header :contains "X-Spam-Flag" "YES", + header :contains "X-Spam" "Yes") { + setflag "\\\\Seen"; +} +EOF + sievec /usr/lib/dovecot/sieve-global/after/spam_mark_as_read.sieve + chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_mark_as_read.{sieve,svbin} + + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]]; then + _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MARK_SPAM_AS_READ=1' to work" + fi + else + _log 'debug' 'Spam emails will not be marked as read' + fi +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index d6c5453c..a0d61242 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -46,6 +46,7 @@ function __environment_variables_general_setup() { VARS[CLAMAV_MESSAGE_SIZE_LIMIT]="${CLAMAV_MESSAGE_SIZE_LIMIT:=25M}" VARS[FAIL2BAN_BLOCKTYPE]="${FAIL2BAN_BLOCKTYPE:=drop}" VARS[MOVE_SPAM_TO_JUNK]="${MOVE_SPAM_TO_JUNK:=1}" + VARS[MARK_SPAM_AS_READ]="${MARK_SPAM_AS_READ:=0}" VARS[POSTGREY_AUTO_WHITELIST_CLIENTS]="${POSTGREY_AUTO_WHITELIST_CLIENTS:=5}" VARS[POSTGREY_DELAY]="${POSTGREY_DELAY:=300}" VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 237218e8..94a9b9c4 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -8,6 +8,7 @@ BATS_TEST_NAME_PREFIX='[Spam - Amavis] ENV SPAMASSASSIN_SPAM_TO_INBOX ' CONTAINER1_NAME='dms-test_spam-amavis_bounced' CONTAINER2_NAME='dms-test_spam-amavis_env-move-spam-to-junk-0' CONTAINER3_NAME='dms-test_spam-amavis_env-move-spam-to-junk-1' +CONTAINER4_NAME='dms-test_spam-amavis_env-mark-spam-as-read-1' function teardown() { _default_teardown ; } @@ -69,6 +70,28 @@ function teardown() { _default_teardown ; } _should_receive_spam_at '/var/mail/localhost.localdomain/user1/.Junk/new/' } +# NOTE: Same as test for `CONTAINER3_NAME`, only differing by ENV `MARK_SPAM_AS_READ=1` + `_should_receive_spam_at` location +@test "(enabled + MARK_SPAM_AS_READ=1) should mark spam message as read" { + export CONTAINER_NAME=${CONTAINER4_NAME} + + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_SPAMASSASSIN=1 + --env SA_SPAM_SUBJECT="SPAM: " + --env SPAMASSASSIN_SPAM_TO_INBOX=1 + --env MARK_SPAM_AS_READ=1 + --env PERMIT_DOCKER=container + ) + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + + _should_send_spam_message + _should_be_received_by_amavis 'Passed SPAM {RelayedTaggedInbound,Quarantined}' + + # Should move delivered spam to Junk folder as read (`cur` instead of `new`) + _should_receive_spam_at '/var/mail/localhost.localdomain/user1/.Junk/cur/' +} + function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis From 0dc862156f7cd748628643c6a3e964d37f7dfb50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 10:30:10 +1200 Subject: [PATCH 005/125] chore(deps): Bump nwtgck/actions-netlify from 2.0 to 2.1 (#3495) Bumps [nwtgck/actions-netlify](https://github.com/nwtgck/actions-netlify) from 2.0 to 2.1. - [Release notes](https://github.com/nwtgck/actions-netlify/releases) - [Changelog](https://github.com/nwtgck/actions-netlify/blob/develop/CHANGELOG.md) - [Commits](https://github.com/nwtgck/actions-netlify/compare/v2.0...v2.1) --- updated-dependencies: - dependency-name: nwtgck/actions-netlify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-preview-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 18b0bab9..278cf60c 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -56,7 +56,7 @@ jobs: context: 'Deploy Preview (pull_request => workflow_run)' - name: 'Send preview build to Netlify' - uses: nwtgck/actions-netlify@v2.0 + uses: nwtgck/actions-netlify@v2.1 id: preview timeout-minutes: 1 env: From 758fd9c913cee012dd25b8215b8b0b16f12743ca Mon Sep 17 00:00:00 2001 From: HeySora Date: Tue, 22 Aug 2023 07:49:15 +0200 Subject: [PATCH 006/125] Docs: Drop mention of port 25 support for authenticated submission (#3496) * FAQ: Remove outdated port 25 for mail client use --- docs/content/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/faq.md b/docs/content/faq.md index 58a327b9..bb2fbe63 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -143,7 +143,7 @@ imap port: 143 or 993 with STARTTLS/SSL (recommended) imap path prefix: INBOX # SMTP -smtp port: 25 or 587/465 with STARTTLS/SSL (recommended) +smtp port: 587 or 465 with STARTTLS/SSL (recommended) username: password: ``` From cf9eb8278ad56d6a06a7e4370b9a0d7855ce9999 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 22 Aug 2023 10:03:41 +0200 Subject: [PATCH 007/125] scripts: add wrapper to update Postfix configuration safely (#3484) The new function can 1. update/append 2. update/prepend 3. initialize if non-existent options in `/etc/postfix/main.cf` in a safe and secure manner. When the container is improperly restarted, the option is not applied twice. --- Co-authored-by: Casper Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- target/scripts/helpers/aliases.sh | 7 +--- target/scripts/helpers/postfix.sh | 37 +++++++++++++++++++ target/scripts/helpers/relay.sh | 2 +- target/scripts/helpers/utils.sh | 9 +++++ .../scripts/startup/setup.d/dmarc_dkim_spf.sh | 9 ++--- .../startup/setup.d/security/rspamd.sh | 7 ++-- 6 files changed, 55 insertions(+), 16 deletions(-) diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 9690aae5..0890d994 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -30,12 +30,7 @@ function _handle_postfix_regexp_config() { _log 'trace' "Adding regexp alias file postfix-regexp.cf" cp -f /tmp/docker-mailserver/postfix-regexp.cf /etc/postfix/regexp - - if ! grep 'virtual_alias_maps.*pcre:/etc/postfix/regexp' /etc/postfix/main.cf; then - sed -i -E \ - 's|virtual_alias_maps(.*)|virtual_alias_maps\1 pcre:/etc/postfix/regexp|g' \ - /etc/postfix/main.cf - fi + _add_to_or_update_postfix_main 'virtual_alias_maps' 'pcre:/etc/postfix/regexp' fi } diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index 32897d0e..b376dce9 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -91,3 +91,40 @@ function _vhost_ldap_support() { # # /etc/aliases is handled by `alias.sh` and uses `postalias` to update the Postfix alias database. No need for `postmap`. # http://www.postfix.org/postalias.1.html + +# Add an key with an value to Postfix's main configuration file +# or update an existing key. An already existing key can be updated +# by either appending to the existing value (default) or by prepending. +# +# @param ${1} = key name in Postfix's main configuration file +# @param ${2} = new value (appended or prepended) +# @param ${3} = "append" (default) or "prepend" [OPTIONAL] +function _add_to_or_update_postfix_main() { + local KEY=${1:?Key name is required} + local NEW_VALUE=${2:?New value is required} + local ACTION=${3:-append} + + # If entry does not exist, add it - otherwise update with ACTION: + if ! grep -q -E "^${KEY}" /etc/postfix/main.cf; then + postconf "${KEY} = ${NEW_VALUE}" + else + KEY=$(_escape_for_sed "${KEY}") + NEW_VALUE=$(_escape_for_sed "${NEW_VALUE}") + local SED_STRING + + case "${ACTION}" in + ('append') + SED_STRING="/${NEW_VALUE}/! s|^(${KEY} *=.*)|\1 ${NEW_VALUE}|g" + ;; + ('prepend') + SED_STRING="/${NEW_VALUE}/! s|^(${KEY}) *= *(.*)|\1 = ${NEW_VALUE} \2|g" + ;; + (*) + _log 'error' "Action '${3}' in _add_to_or_update_postfix_main is unknown" + return 1 + ;; + esac + + sed -i -E "${SED_STRING}" /etc/postfix/main.cf + fi +} diff --git a/target/scripts/helpers/relay.sh b/target/scripts/helpers/relay.sh index c2713651..de29dddc 100644 --- a/target/scripts/helpers/relay.sh +++ b/target/scripts/helpers/relay.sh @@ -173,7 +173,7 @@ function _setup_relayhost() { _log 'debug' 'Setting up Postfix Relay Hosts' if [[ -n ${DEFAULT_RELAY_HOST} ]]; then - _log 'trace' "Setting default relay host ${DEFAULT_RELAY_HOST} to /etc/postfix/main.cf" + _log 'trace' "Setting default relay host ${DEFAULT_RELAY_HOST}" postconf "relayhost = ${DEFAULT_RELAY_HOST}" fi diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index 0c5a0321..02d29cc9 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -4,6 +4,15 @@ function _escape() { echo "${1//./\\.}" } +# Replaces a string so that it can be used inside +# `sed` safely. +# +# @param ${1} = string to escape +# @output = prints the escaped string +function _escape_for_sed() { + sed -E 's/[]\/$*.^[]/\\&/g' <<< "${1:?String to escape for sed is required}" +} + # Returns input after filtering out lines that are: # empty, white-space, comments (`#` as the first non-whitespace character) function _get_valid_lines_from_file() { diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index 85c63d96..c0d731f2 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -16,10 +16,9 @@ function _setup_opendkim() { _log 'trace' "Adding OpenDKIM to Postfix's milters" postconf 'dkim_milter = inet:localhost:8891' # shellcheck disable=SC2016 - sed -i -E \ - -e '/\$dkim_milter/! s|^(smtpd_milters =.*)|\1 \$dkim_milter|g' \ - -e '/\$dkim_milter/! s|^(non_smtpd_milters =.*)|\1 \$dkim_milter|g' \ - /etc/postfix/main.cf + _add_to_or_update_postfix_main 'smtpd_milters' '$dkim_milter' + # shellcheck disable=SC2016 + _add_to_or_update_postfix_main 'non_smtpd_milters' '$dkim_milter' # check if any keys are available if [[ -e /tmp/docker-mailserver/opendkim/KeyTable ]]; then @@ -64,7 +63,7 @@ function _setup_opendmarc() { postconf 'dmarc_milter = inet:localhost:8893' # Make sure to append the OpenDMARC milter _after_ the OpenDKIM milter! # shellcheck disable=SC2016 - sed -i -E '/\$dmarc_milter/! s|^(smtpd_milters =.*)|\1 \$dmarc_milter|g' /etc/postfix/main.cf + _add_to_or_update_postfix_main 'smtpd_milters' '$dmarc_milter' sed -i \ -e "s|^AuthservID.*$|AuthservID ${HOSTNAME}|g" \ diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 4ece646b..d2389a07 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -138,7 +138,7 @@ function __rspamd__setup_postfix() { postconf 'rspamd_milter = inet:localhost:11332' # shellcheck disable=SC2016 - sed -i -E 's|^(smtpd_milters =.*)|\1 \$rspamd_milter|g' /etc/postfix/main.cf + _add_to_or_update_postfix_main 'smtpd_milters' '$rspamd_milter' } # If ClamAV is enabled, we will integrate it into Rspamd. @@ -180,6 +180,7 @@ function __rspamd__setup_default_modules() { metric_exporter ) + local MODULE for MODULE in "${DISABLE_MODULES[@]}"; do __rspamd__helper__enable_disable_module "${MODULE}" 'false' done @@ -289,7 +290,7 @@ function __rspamd__handle_user_modules_adjustments() { # # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ # @param ${2} = module name as it should appear in the log - # @patam ${3} = option name in the module + # @param ${3} = option name in the module # @param ${4} = value of the option # # ## Note @@ -333,7 +334,6 @@ function __rspamd__handle_user_modules_adjustments() { while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do case "${COMMAND}" in - ('disable-module') __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' ;; @@ -367,7 +367,6 @@ function __rspamd__handle_user_modules_adjustments() { __rspamd__log 'warn' "Command '${COMMAND}' is invalid" continue ;; - esac done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") fi From 39ae101266af7511d85fbb5e9e4b19324390f5ce Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 22 Aug 2023 21:38:25 +1200 Subject: [PATCH 008/125] tests: Change OpenLDAP image to `bitnami/openldap` (#3494) **TL;DR:** - New image is actively maintained vs existing one that is over 5 years old. - Slight improvement to LDAP tree config via `.ldif` files. - No more `Dockerfile` required to build, we can just rely on `docker run`. `osixia/openldap` has not seen any activity since Feb 2021, while our `Dockerfile` was fixed to v1.1.6` (Feb 2018). Startup time for this new image is around 5 seconds? (_The LDAP test uses a standard 20 second timeout check to wait until the server is ready before continuing with starting the DMS image_). This commit migrates to `bitnami/openldap` which required modifying the `01_mail-tree.ldif` to also include adding the root object to start successfully. This image is actively maintained and one of the most popular OpenLDAP images on DockerHub. The user account `.ldif` files have minimal changes: - Lines moved around for better organization - Additional comments for context - Removal of inherited `objectClass` attributes (`person`, `top`) from the `orgnizationalPerson` class. Attribute `sn` changed to long form `surname` and values corrected with `givenName`. `changetype: add` was also not necessary. Additionally the image does not support the `.schema` format, they must be converted to `.ldif` which has been done for `postfix-book.schema`. See PR for more details. --- test/config/ldap/docker-openldap/Dockerfile | 5 -- .../bootstrap/ldif/01_mail-tree.ldif | 10 ++- .../bootstrap/ldif/02_user-email.ldif | 25 +++---- .../03_user-email-other-primary-domain.ldif | 26 ++++--- .../ldif/04_user-email-different-uid.ldif | 30 ++++---- .../bootstrap/schema/mmc/postfix-book.schema | 70 ------------------- .../bootstrap/schemas/postfix-book.ldif | 14 ++++ test/tests/serial/mail_with_ldap.bats | 17 +++-- 8 files changed, 69 insertions(+), 128 deletions(-) delete mode 100644 test/config/ldap/docker-openldap/Dockerfile delete mode 100644 test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema create mode 100644 test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif diff --git a/test/config/ldap/docker-openldap/Dockerfile b/test/config/ldap/docker-openldap/Dockerfile deleted file mode 100644 index 934c498f..00000000 --- a/test/config/ldap/docker-openldap/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM osixia/openldap:1.1.6 -LABEL maintainer="Dennis Stumm " - -COPY bootstrap /container/service/slapd/assets/config/bootstrap -RUN rm /container/service/slapd/assets/config/bootstrap/schema/mmc/mail.schema diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif index 940fef24..5e3d8b0a 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif @@ -1,5 +1,11 @@ +# The root object, all entries will branch off this one: +dn: dc=localhost,dc=localdomain +objectClass: dcObject +objectClass: organization +dc: localhost +o: DMS Test + +# User accounts will belong to this subtree: dn: ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalUnit -objectClass: top ou: people diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif index 993a4e70..cd656bf8 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif @@ -1,25 +1,22 @@ -# -------------------------------------------------------------------- -# Create mail accounts -# -------------------------------------------------------------------- -# Some User +# NOTE: A standard user account to test against dn: uniqueIdentifier=some.user,ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalPerson -objectClass: person -objectClass: top objectClass: PostfixBookMailAccount objectClass: extensibleObject cn: Some User -givenName: User +givenName: Some +surname: User +uniqueIdentifier: some.user +# Password is: secret +userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.user@localhost.localdomain +# postfix-book.schema: mailAlias: postmaster@localhost.localdomain mailGroupMember: employees@localhost.localdomain -mailEnabled: TRUE -mailGidNumber: 5000 mailHomeDirectory: /var/mail/localhost.localdomain/some.user/ -mailQuota: 10240 mailStorageDirectory: maildir:/var/mail/localhost.localdomain/some.user/ +# postfix-book.schema generic options: +mailEnabled: TRUE mailUidNumber: 5000 -sn: Some -uniqueIdentifier: some.user -userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx +mailGidNumber: 5000 +mailQuota: 10240 diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif index f949349c..d27cec7e 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif @@ -1,25 +1,23 @@ -# -------------------------------------------------------------------- -# Create mail accounts -# -------------------------------------------------------------------- -# Some User +# NOTE: This user differs via the domain-part of their mail address +# They also have their mail directory attributes using the primary domain, not their domain-part dn: uniqueIdentifier=some.other.user,ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalPerson -objectClass: person -objectClass: top objectClass: PostfixBookMailAccount objectClass: extensibleObject cn: Some Other User -givenName: Other User +givenName: Some +surname: Other User +uniqueIdentifier: some.other.user +# Password is: secret +userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.other.user@localhost.otherdomain +# postfix-book.schema: mailAlias: postmaster@localhost.otherdomain mailGroupMember: employees@localhost.otherdomain -mailEnabled: TRUE -mailGidNumber: 5000 mailHomeDirectory: /var/mail/localhost.localdomain/some.other.user/ -mailQuota: 10240 mailStorageDirectory: maildir:/var/mail/localhost.localdomain/some.other.user/ +# postfix-book.schema generic options: +mailEnabled: TRUE mailUidNumber: 5000 -sn: Some -uniqueIdentifier: some.other.user -userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx +mailGidNumber: 5000 +mailQuota: 10240 diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif index b991993f..be49fd9f 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif @@ -1,23 +1,21 @@ -# -------------------------------------------------------------------- -# Create mail accounts -# -------------------------------------------------------------------- -# Some User +# NOTE: This user differs by local-part of mail address not matching their uniqueIdentifier attribute +# They also do not have any alias or groups configured dn: uniqueIdentifier=some.user.id,ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalPerson -objectClass: person -objectClass: top objectClass: PostfixBookMailAccount objectClass: extensibleObject cn: Some User -givenName: User -mail: some.user.email@localhost.localdomain -mailEnabled: TRUE -mailGidNumber: 5000 -mailHomeDirectory: /var/mail/localhost.localdomain/some.user.id/ -mailQuota: 10240 -mailStorageDirectory: maildir:/var/mail/localhost.localdomain/some.user.id/ -mailUidNumber: 5000 -sn: Some +givenName: Some +surname: User uniqueIdentifier: some.user.id +# Password is: secret userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx +mail: some.user.email@localhost.localdomain +# postfix-book.schema: +mailHomeDirectory: /var/mail/localhost.localdomain/some.user.id/ +mailStorageDirectory: maildir:/var/mail/localhost.localdomain/some.user.id/ +# postfix-book.schema generic options: +mailEnabled: TRUE +mailUidNumber: 5000 +mailGidNumber: 5000 +mailQuota: 10240 diff --git a/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema b/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema deleted file mode 100644 index 9f0d7e53..00000000 --- a/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema +++ /dev/null @@ -1,70 +0,0 @@ -# $Id$ -# -# State of Mind -# Private Enterprise Number: 29426 -# -# OID prefix: 1.3.6.1.4.1.29426 -# -# Attributes: 1.3.6.1.4.1.29426.1.10.x -# - - -attributetype ( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' - DESC 'The absolute path to the mail user home directory' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' - DESC 'RFC822 Mailbox - mail alias' - EQUALITY caseIgnoreIA5Match - SUBSTR caseIgnoreIA5SubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' - DESC 'UID required to access the mailbox' - EQUALITY integerMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' - DESC 'GID required to access the mailbox' - EQUALITY integerMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' - DESC 'TRUE to enable, FALSE to disable account' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' - DESC 'Name of a mail distribution list' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' - DESC 'Mail quota limit in kilobytes' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' - DESC 'The absolute path to the mail users mailbox' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - - -# -# Objects: 1.3.6.1.4.1.29426.1.2.2.x -# - -objectclass ( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' - SUP top AUXILIARY - DESC 'Mail account used in Postfix Book' - MUST ( mail ) - MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember - $ mailUidNumber $ mailGidNumber $ mailEnabled - $ mailQuota $mailStorageDirectory ) ) - -objectclass ( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' - SUP top AUXILIARY - DESC 'Mail forward used in Postfix Book' - MUST ( mail $ mailAlias )) - diff --git a/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif b/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif new file mode 100644 index 00000000..9ea787d5 --- /dev/null +++ b/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif @@ -0,0 +1,14 @@ +dn: cn=postfix-book,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: postfix-book +olcAttributeTypes: {0}( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' DESC 'The absolute path to the mail user home directory' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +olcAttributeTypes: {1}( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' DESC 'RFC822 Mailbox - mail alias' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) +olcAttributeTypes: {2}( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' DESC 'UID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {3}( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' DESC 'GID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {4}( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' DESC 'TRUE to enable, FALSE to disable account' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +olcAttributeTypes: {5}( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' DESC 'Name of a mail distribution list' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {6}( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' DESC 'Mail quota limit in kilobytes' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {7}( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' DESC 'The absolute path to the mail users mailbox' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +# PostfixBook object classes: +olcObjectClasses: {0}( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' DESC 'Mail account used in Postfix Book' SUP top AUXILIARY MUST mail MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember $ mailUidNumber $ mailGidNumber $ mailEnabled $ mailQuota $ mailStorageDirectory ) ) +olcObjectClasses: {1}( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' DESC 'Mail forward used in Postfix Book' SUP top AUXILIARY MUST ( mail $ mailAlias ) ) diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 57251aa1..baeb61e2 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -19,15 +19,18 @@ function setup_file() { docker network create "${DMS_TEST_NETWORK}" # Setup local openldap service: - # NOTE: Building via Dockerfile is required? Image won't accept read-only if it needs to adjust permissions for bootstrap files. - # TODO: Upstream image is no longer maintained, may want to migrate? - docker build -t dms-openldap test/config/ldap/docker-openldap/ - - docker run -d --name "${CONTAINER2_NAME}" \ - --env LDAP_DOMAIN="${FQDN_LOCALHOST_A}" \ + docker run --rm -d --name "${CONTAINER2_NAME}" \ + --env LDAP_ADMIN_PASSWORD=admin \ + --env LDAP_ROOT='dc=localhost,dc=localdomain' \ + --env LDAP_PORT_NUMBER=389 \ + --env LDAP_SKIP_DEFAULT_TREE=yes \ + --volume './test/config/ldap/docker-openldap/bootstrap/ldif/:/ldifs/:ro' \ + --volume './test/config/ldap/docker-openldap/bootstrap/schemas/:/schemas/:ro' \ --hostname "${FQDN_LDAP}" \ --network "${DMS_TEST_NETWORK}" \ - dms-openldap + bitnami/openldap:latest + + _run_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER2_NAME} 2>&1 | grep 'LDAP setup finished'" # # Setup DMS container From af09db6648b2bbb13b7364f39b9cf76d482cdfb2 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 24 Aug 2023 02:56:24 +1200 Subject: [PATCH 009/125] ci: `question.yml` - Clarify that the issue tracker is not for personal support (#3498) * ci: Revise `question.yml` to better clarify the issue tracker is not for support queries Users have been making low effort reports (_bypassing the dedicated form_) through this alternative that is intended for addressing other concerns related to the project - not troubleshooting user problems. When a user does not want to put the effort in of a full bug report (_and following our debug docs tips that it refers them to_), they should be using the Github Discussions page which provides the same free-form input, but should not require attention of project devs (contributors / maintainers). --- The markdown rendered field above the "Description" input field didn't seem too relevant for this template. I've opted for a markdown comment (so it won't render if kept) into the input field with hopes that'll be more visible to the readers attention. * chore: Fix typo --- .github/ISSUE_TEMPLATE/question.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml index 7a743511..e94062e0 100644 --- a/.github/ISSUE_TEMPLATE/question.yml +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -1,18 +1,10 @@ name: Other -description: Miscellaneous questions and reports +description: Miscellaneous questions and reports for the project (not support) title: 'other: ' labels: - meta/help wanted body: - - type: markdown - attributes: - value: | - Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. - - Be as precise as possible, and if in doubt, it's best to add more information that too few. - - --- - type: dropdown id: subject attributes: @@ -21,8 +13,7 @@ body: - I would like to contribute to the project - I would like to configure a not documented mail server use case - I would like some feedback concerning a use case - - I have questions about TLS/SSL/STARTTLS/OpenSSL - - Other + - Something else that requires developers attention validations: required: true - type: textarea @@ -31,3 +22,9 @@ body: label: Description validations: required: true + value: | + From c8a0bfd36195281b629916d64edf0a2c1fbd89d1 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 25 Aug 2023 00:29:08 +1200 Subject: [PATCH 010/125] ci: Fix `question.yml` template - `value` should be an attribute (#3502) The recent change to this template was invalid, as `value` should have been nested under the `attributes` object. --- .github/ISSUE_TEMPLATE/question.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml index e94062e0..b58d0370 100644 --- a/.github/ISSUE_TEMPLATE/question.yml +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -18,13 +18,13 @@ body: required: true - type: textarea id: description - attributes: - label: Description validations: required: true - value: | - + attributes: + label: Description + value: | + From f89cbac21c26ac306ff649bfef96605b44dba86f Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 27 Aug 2023 23:44:53 +1200 Subject: [PATCH 011/125] tests: TLS cipher suites - Update `testssl.sh` tag to `3.2` (#3504) --- test/tests/parallel/set2/tls_cipherlists.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/parallel/set2/tls_cipherlists.bats b/test/tests/parallel/set2/tls_cipherlists.bats index d9a82eb3..2b9511b9 100644 --- a/test/tests/parallel/set2/tls_cipherlists.bats +++ b/test/tests/parallel/set2/tls_cipherlists.bats @@ -25,7 +25,7 @@ function setup_file() { # Pull `testssl.sh` image in advance to avoid it interfering with the `run` captured output. # Only interferes (potential test failure) with `assert_output` not `assert_success`? - docker pull drwetter/testssl.sh:3.1dev + docker pull drwetter/testssl.sh:3.2 # Only used in `_should_support_expected_cipherlists()` to set a storage location for `testssl.sh` JSON output: # `${BATS_TMPDIR}` maps to `/tmp`: https://bats-core.readthedocs.io/en/v1.8.2/writing-tests.html#special-variables @@ -166,7 +166,7 @@ function _collect_cipherlists() { --volume "${TLS_CONFIG_VOLUME}" \ --volume "${RESULTS_PATH}:/output" \ --workdir "/output" \ - drwetter/testssl.sh:3.1dev "${TESTSSL_CMD[@]}" + drwetter/testssl.sh:3.2 "${TESTSSL_CMD[@]}" assert_success } From 43a122fe18abec5d7607e9659a890780930bb1ce Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 27 Aug 2023 23:40:24 +0200 Subject: [PATCH 012/125] scripts: add wrapper to update Postfix configuration safely (follow up) (#3503) --- target/scripts/helpers/postfix.sh | 26 +++++++++++++++----------- target/scripts/helpers/utils.sh | 1 + 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index b376dce9..5fa4fa83 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -92,39 +92,43 @@ function _vhost_ldap_support() { # /etc/aliases is handled by `alias.sh` and uses `postalias` to update the Postfix alias database. No need for `postmap`. # http://www.postfix.org/postalias.1.html -# Add an key with an value to Postfix's main configuration file +# Add a key with a value to Postfix's main configuration file # or update an existing key. An already existing key can be updated # by either appending to the existing value (default) or by prepending. # # @param ${1} = key name in Postfix's main configuration file # @param ${2} = new value (appended or prepended) -# @param ${3} = "append" (default) or "prepend" [OPTIONAL] +# @param ${3} = action "append" (default) or "prepend" [OPTIONAL] function _add_to_or_update_postfix_main() { local KEY=${1:?Key name is required} local NEW_VALUE=${2:?New value is required} local ACTION=${3:-append} + local CURRENT_VALUE - # If entry does not exist, add it - otherwise update with ACTION: - if ! grep -q -E "^${KEY}" /etc/postfix/main.cf; then + # Get current value from /etc/postfix/main.cf + _adjust_mtime_for_postfix_maincf + CURRENT_VALUE=$(postconf -h "${KEY}" 2>/dev/null) + + # If key does not exist or value is empty, add it - otherwise update with ACTION: + if [[ -z ${CURRENT_VALUE} ]]; then postconf "${KEY} = ${NEW_VALUE}" else - KEY=$(_escape_for_sed "${KEY}") - NEW_VALUE=$(_escape_for_sed "${NEW_VALUE}") - local SED_STRING + # If $NEW_VALUE is already present --> nothing to do, skip. + if [[ " ${CURRENT_VALUE} " == *" ${NEW_VALUE} "* ]]; then + return 0 + fi case "${ACTION}" in ('append') - SED_STRING="/${NEW_VALUE}/! s|^(${KEY} *=.*)|\1 ${NEW_VALUE}|g" + postconf "${KEY} = ${CURRENT_VALUE} ${NEW_VALUE}" ;; ('prepend') - SED_STRING="/${NEW_VALUE}/! s|^(${KEY}) *= *(.*)|\1 = ${NEW_VALUE} \2|g" + postconf "${KEY} = ${NEW_VALUE} ${CURRENT_VALUE}" ;; (*) _log 'error' "Action '${3}' in _add_to_or_update_postfix_main is unknown" return 1 ;; esac - - sed -i -E "${SED_STRING}" /etc/postfix/main.cf fi } diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index 02d29cc9..c74fd7e8 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -4,6 +4,7 @@ function _escape() { echo "${1//./\\.}" } +# TODO: Not in use currently. Maybe in the future: https://github.com/docker-mailserver/docker-mailserver/pull/3484/files#r1299410851 # Replaces a string so that it can be used inside # `sed` safely. # From 855d9acb5306aa79c8c5e4164a222d388f1cef05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 19:33:05 +0200 Subject: [PATCH 013/125] chore(deps): Bump docker/setup-buildx-action from 2.9.1 to 2.10.0 (#3511) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 7dbe0517..a86b0ed6 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index ea837364..598c12a3 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 7c9f6f05..5b460cce 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 934c4d48..69e8cea3 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From e9f04cf8a7d31efbc8ae9a4a35603409f637f3b4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 09:40:02 +1200 Subject: [PATCH 014/125] chore: Change `setup config dkim` default key size to `2048` (`open-dkim`) (#3508) * chore: Adjust default DKIM size (`open-dkim`) from 4096-bit to 2048-bit 4096-bit is excessive in size for DKIM key. 2048-bit is plenty. * chore: Additional revisions to `open-dkim` command help output - The examples use `keysize 2048`, but as that's the new default it makes sense to change that. - Other help text was also revised. - Last example for domains did not need to demonstrate the other options. Changed example domains to more appropriate values. * docs: Revise DKIM docs Primarily for the change in default key size, but does revise some text to better communicate to the user. - While the referenced RFC advises 512-bit to 2048-bit key size, we now explicitly discourage `512-bit` as it's not secure. `1024-bit` is still likely safe for most, but `2048-bit` is a good default for those not rotating their keys. - Adjusted the domains example to match the new `setup config dkim domain` domains example. - Tip for changing default key size changed to "info" with added clarity of lowering security or increasing it (excessively). - Rspamd section is minor formatting changes, with the exception of clarifying the "main domain" for the mail accounts is assumed as the DMS FQDN with any subdomain (like `mail.`) stripped away. This is not great, but a legacy issue that needs to be addressed in future. - `docs-rspamd-override-d` ref removed, and usage replaced with equivalent ref `docs-rspamd-config-dropin`, while `docs-rspamd-config-declarative` ref was not in use and also removed. - Revised the `.txt` DNS formatting info section to better communicate with the reader. Additionally it had mixed usage of default `mail` and custom `dkim-rsa` selectors (_file content and output_). * docs: Sync DKIM commands help messages and update DKIM docs for LDAP - Adopt the help options format style from the `rspamd-dkim` into `open-dkim` command. And convert `./setup.sh` to `setup`. `selector` option has been implemented. for a while now. - Update `rspamd-dkim` examples help output to align with `open-dkim` command examples. - Give both DKIM command tools a consistent description. The two tools differ in support for the `domain` option (_implicit domain sourcing for default account provisioner, and support for multiple domains as input_). - DKIM docs for LDAP domain support revised to better communicate when explicit domain config is necessary. * tests: Adjust test-cases for `setup config dkim` change `rspamd_dkim.bats`: - Update assert for command help output. - Don't bother creating a DKIM key at 512-bit size. `setup_cli.bats`: - Update assert for command help output of the `setup config dkim` (OpenDKIM) command. * docs: Update DKIM section for large keys to newer RFC The linked discussion from 2021 does mention this updated RFC over the original. That removes outdated advice about `512-bit` key length support. The discussion link is still kept to reference a comment for the reader to better understand the security strength of 2048-bit RSA keys and why larger keys are not worthwhile, especially for DKIM. * docs: Extract out common DKIM generation command from content tabs Should be fine to be DRY here, not specific to `open-dkim` or `rspamd` generation/support. Previously rspamd lacked support of an equivalent command in DMS. * docs: DKIM refactoring - Shifted out the info admonition on key size advice out of the content tabs as it's now generic information. - Indented the 4096-bit warning into this, which is less of a concern as the default for our DKIM generation tools is consistently 2048-bit now. - Reworked the LDAP and Rspamd multi-domain advice. To avoid causing a bad diff, these sections haven't been moved/merged yet. * docs: Revise DKIM docs Advice for managing domains individually with LDAP and Rspamd extracted out of the content tabs. Default domain behaviour explained with extra info about OpenDKIM + FILE provisioner sourcing extra domains implicitly. --- .../config/best-practices/dkim_dmarc_spf.md | 131 ++++++++++-------- target/bin/open-dkim | 41 +++--- target/bin/rspamd-dkim | 31 ++--- .../parallel/set1/spam_virus/rspamd_dkim.bats | 4 +- .../parallel/set3/scripts/setup_cli.bats | 2 +- test/tests/serial/open_dkim.bats | 8 +- 6 files changed, 115 insertions(+), 102 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 8ab00f48..eda98600 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -35,14 +35,67 @@ DKIM requires a public/private key pair to enable **signing (_via private key_)* ### Generating Keys +You'll need to repeat this process if you add any new domains. + You should have: - At least one [email account setup][docs-accounts-add] - Attached a [volume for config][docs-volumes-config] to persist the generated files to local storage -!!! warning "RSA Key Sizes >= 4096 Bit" +!!! example "Creating DKIM Keys" - Keys of 4096 bits could be denied by some mail servers. According to [RFC 6376][rfc-6376], keys are [preferably between 512 and 2048 bits][github-issue-dkimlength]. + DKIM keys can be generated with good defaults by running: + + ```bash + docker exec -it setup config dkim + ``` + + If you need to generate your keys with different settings, check the `help` output for supported config options and examples: + + ```bash + docker exec -it setup config dkim help + ``` + + As described by the help output, you may need to use the `domain` option explicitly when you're using LDAP or Rspamd. + +??? info "Changing the key size" + + The keypair generated for using with DKIM presently defaults to RSA-2048. This is a good size but you can lower the security to `1024-bit`, or increase it to `4096-bit` (_discouraged as that is excessive_). + + To generate a key with different size (_for RSA 1024-bit_) run: + + ```sh + setup config dkim keysize 1024 + ``` + + !!! warning "RSA Key Sizes >= 4096 Bit" + + According to [RFC 8301][rfc-8301], keys are preferably between 1024 and 2048 bits. Keys of size 4096-bit or larger may not be compatible to all systems your mail is intended for. + + You [should not need a key length beyond 2048-bit][github-issue-dkimlength]. If 2048-bit does not meet your security needs, you may want to instead consider adopting key rotation or switching from RSA to ECC keys for DKIM. + +??? note "You may need to specify mail domains explicitly" + + Required when using LDAP and Rspamd. + + `setup config dkim` will generate DKIM keys for what is assumed as the primary mail domain (_derived from the FQDN assigned to DMS, minus any subdomain_). + + When the DMS FQDN is `mail.example.com` or `example.com`, by default this command will generate DKIM keys for `example.com` as the primary domain for your users mail accounts (eg: `hello@example.com`). + + The DKIM generation does not have support to query LDAP for additionanl mail domains it should know about. If the primary mail domain is not sufficient, then you must explicitly specify any extra domains via the `domain` option: + + ```sh + # ENABLE_OPENDKIM=1 (default): + setup config dkim domain 'example.com,another-example.com' + + # ENABLE_RSPAMD=1 + ENABLE_OPENDKIM=0: + setup config dkim domain example.com + setup config dkim domain another-example.com + ``` + + !!! info "OpenDKIM with `ACCOUNT_PROVISIONER=FILE`" + + When DMS uses this configuration, it will by default also detect mail domains (_from accounts added via `setup email add`_), generating additional DKIM keys. DKIM is currently supported by either OpenDKIM or Rspamd: @@ -50,68 +103,27 @@ DKIM is currently supported by either OpenDKIM or Rspamd: OpenDKIM is currently [enabled by default][docs-env-opendkim]. - The command `docker exec setup config dkim help` details supported config options, along with some examples. - - !!! example "Creating a DKIM key" - - Generate the DKIM files with: - - ```sh - docker exec -ti setup config dkim - ``` - - Your new DKIM key(s) and OpenDKIM config files have been added to `/tmp/docker-mailserver/opendkim/`. - - ??? note "LDAP accounts need to specify domains explicitly" - - The command is unable to infer the domains from LDAP user accounts, you must specify them: - - ```sh - setup config dkim domain 'example.com,example.io' - ``` - - ??? tip "Changing the key size" - - The private key presently defaults to RSA-4096. To create an RSA 2048-bit key run: - - ```sh - setup config dkim keysize 2048 - ``` + After running `setup config dkim`, your new DKIM key files (_and OpenDKIM config_) have been added to `/tmp/docker-mailserver/opendkim/`. !!! info "Restart required" After restarting DMS, outgoing mail will now be signed with your new DKIM key(s) :tada: - You'll need to repeat this process if you add any new domains. - === "Rspamd" - Opt-in via [`ENABLE_RSPAMD=1`][docs-env-rspamd] (_and disable the default OpenDKIM: `ENABLE_OPENDKIM=0`_). + Requires opt-in via [`ENABLE_RSPAMD=1`][docs-env-rspamd] (_and disable the default OpenDKIM: `ENABLE_OPENDKIM=0`_). Rspamd provides DKIM support through two separate modules: 1. [Verifying DKIM signatures from inbound mail][rspamd-docs-dkim-checks] is enabled by default. 2. [Signing outbound mail with your DKIM key][rspamd-docs-dkim-signing] needs additional setup (key + dns + config). - !!! example "Creating DKIM Keys" - - You can simply run - - ```bash - docker exec -ti setup config dkim help - ``` - - which provides you with an overview of what the script can do. Just running - - ```bash - docker exec -ti setup config dkim - ``` - - will execute the helper script with default parameters. - ??? warning "Using Multiple Domains" - Unlike the current script for OpenDKIM, the Rspamd script will **not** create keys for all domains DMS is managing, but only for the one it assumes to be the main domain (derived from DMS' domain name). Moreover, the default `dkim_signing.conf` configuration file that DMS ships will also only contain one domain. If you have multiple domains, you need to run the command `docker exec -ti setup config dkim domain ` multiple times to create all the keys for all domains, and then provide a custom `dkim_signing.conf` (for which an example is shown below). + If you have multiple domains, you need to: + + - Create a key wth `docker exec -it setup config dkim domain ` for each domain DMS should sign outgoing mail for. + - Provide a custom `dkim_signing.conf` (for which an example is shown below), as the default config only supports one domain. !!! info "About the Helper Script" @@ -121,7 +133,9 @@ DKIM is currently supported by either OpenDKIM or Rspamd: --- - In case you have not already provided a default DKIM signing configuration, the script will create one and write it to `/etc/rspamd/override.d/dkim_signing.conf`. If this file already exist, it will not be overwritten. When you're already using [the `rspamd/override.d/` directory][docs-rspamd-override-d], the file is created inside your volume and therefore persisted correctly. If you are not using `rspamd/override.d/`, you will need to persist the file yourself (otherwise it is lost on container restart). + In case you have not already provided a default DKIM signing configuration, the script will create one and write it to `/etc/rspamd/override.d/dkim_signing.conf`. If this file already exists, it will not be overwritten. + + When you're already using [the `rspamd/override.d/` directory][docs-rspamd-config-dropin], the file is created inside your volume and therefore persisted correctly. If you are not using `rspamd/override.d/`, you will need to persist the file yourself (otherwise it is lost on container restart). An example of what a default configuration file for DKIM signing looks like can be found by expanding the example below. @@ -190,8 +204,6 @@ DKIM is currently supported by either OpenDKIM or Rspamd: If there is a mismatch, a warning will be emitted to the Rspamd log `/var/log/supervisor/rspamd.log`. - [docs-rspamd-override-d]: ../security/rspamd.md#manually - ### DNS Record { #dkim-dns } When mail signed with your DKIM key is sent from your mail server, the receiver needs to check a DNS `TXT` record to verify the DKIM signature is trustworthy. @@ -221,11 +233,13 @@ When mail signed with your DKIM key is sent from your mail server, the receiver ??? info "`.txt` - Formatting the `TXT` record value correctly" - This file was generated for use within a [DNS zone file][dns::wikipedia-zonefile]. DNS `TXT` records values that are longer than 255 characters need to be split into multiple parts. This is why the public key has multiple parts wrapped within double-quotes between `(` and `)`. + This file was generated for use within a [DNS zone file][dns::wikipedia-zonefile]. The file name uses the DKIM selector it was generated with (default DKIM selector is `mail`, which creates `mail.txt`_). - A DNS web-interface may handle this internally instead, while [others may not, but expect the input as a single line][dns::webui-dkim]_). You'll need to manually format the value as described below. + For your DNS setup, DKIM support needs to create a `TXT` record to store the public key for mail clients to use. `TXT` records with values that are longer than 255 characters need to be split into multiple parts. This is why the generated `.txt` file (_containing your public key for use with DKIM_) has multiple value parts wrapped within double-quotes between `(` and `)`. - Your DNS record file (eg: `mail.txt`) should look similar to this: + A DNS web-interface may handle this separation internally instead, and [could expect the value provided all as a single line][dns::webui-dkim] instead of split. When that is required, you'll need to manually format the value as described below. + + Your generated DNS record file (`.txt`) should look similar to this: ```txt mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " @@ -243,7 +257,7 @@ When mail signed with your DKIM key is sent from your mail server, the receiver To test that your new DKIM record is correct, query it with the `dig` command. The `TXT` value response should be a single line split into multiple parts wrapped in double-quotes: ```console - $ dig +short TXT dkim-rsa._domainkey.example.com + $ dig +short TXT mail._domainkey.example.com "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQMMqhb1S52Rg7VFS3EC6JQIMxNDdiBmOKZvY5fiVtD3Z+yd9ZV+V8e4IARVoMXWcJWSR6xkloitzfrRtJRwOYvmrcgugOalkmM0V4Gy/2aXeamuiBuUc4esDQEI3egmtAsHcVY1XCoYfs+9VqoHEq3vdr3UQ8zP/l+FP5UfcaJFCK/ZllqcO2P1GjIDVSHLdPpRHbMP/tU1a9mNZ5QMZBJ/JuJK/s+2bp8gpxKn8rh1akSQjlynlV9NI+7J3CC7CUf3bGvoXIrb37C/lpJehS39" "KNtcGdaRufKauSfqx/7SxA0zyZC+r13f7ASbMaQFzm+/RRusTqozY/p/MsWx8QIDAQAB" ``` @@ -328,10 +342,9 @@ volumes: [docs-env-opendkim]: ../environment.md#enable_opendkim [docs-env-rspamd]: ../environment.md#enable_rspamd [docs-rspamd-config-dropin]: ../security/rspamd.md#manually -[docs-rspamd-config-declarative]: ../security/rspamd.md#with-the-help-of-a-custom-file [cloudflare-dkim-dmarc-spf]: https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/ -[rfc-6376]: https://tools.ietf.org/html/rfc6376 -[github-issue-dkimlength]: https://github.com/docker-mailserver/docker-mailserver/issues/1854 +[rfc-8301]: https://datatracker.ietf.org/doc/html/rfc8301#section-3.2 +[github-issue-dkimlength]: https://github.com/docker-mailserver/docker-mailserver/issues/1854#issuecomment-806280929 [rspamd-docs-dkim-checks]: https://www.rspamd.com/doc/modules/dkim.html [rspamd-docs-dkim-signing]: https://www.rspamd.com/doc/modules/dkim_signing.html [dns::example-webui]: https://www.vultr.com/docs/introduction-to-vultr-dns/ diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 50b18f6c..0036ff8d 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -8,7 +8,7 @@ if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 exit fi -KEYSIZE=4096 +KEYSIZE=2048 SELECTOR=mail DOMAINS= @@ -16,37 +16,40 @@ function __usage() { printf '%s' "${PURPLE}OPEN-DKIM${RED}(${YELLOW}8${RED}) ${ORANGE}NAME${RESET} - open-dkim - configure DomainKeys Identified Mail (DKIM) + open-dkim - Configure DKIM (DomainKeys Identified Mail) ${ORANGE}SYNOPSIS${RESET} - ./setup.sh config dkim [ OPTIONS${RED}...${RESET} ] + setup config dkim [ OPTIONS${RED}...${RESET} ] ${ORANGE}DESCRIPTION${RESET} - Configures DKIM keys. OPTIONS can be used to configure a more complex setup. - LDAP setups require these options. + Creates DKIM keys and configures them within DMS for OpenDKIM. + OPTIONS can be used when your requirements are not met by the defaults. + When not using 'ACCOUNT_PROVISIONER=FILE' (default), you may need to explicitly + use the 'domain' option to generate DKIM keys for your mail account domains. ${ORANGE}OPTIONS${RESET} ${BLUE}Generic Program Information${RESET} - help Print the usage information. + help Print the usage information. ${BLUE}Configuration adjustments${RESET} - keysize Set the size of the keys to be generated. Possible are 1024, 2048 and 4096 (default). - selector Set a manual selector (default is 'mail') for the key. (${LCYAN}ATTENTION${RESET}: NOT IMPLEMENTED YET!) - domain Provide the domain(s) for which keys are to be generated. + keysize Set the size of the keys to be generated. + Possible values: 1024, 2048 and 4096 + Default: 2048 + selector Set a manual selector for the key. + Default: mail + domain Provide the domain(s) for which to generate keys for. + Default: The FQDN assigned to DMS, excluding any subdomain. + 'ACCOUNT_PROVISIONER=FILE' also sources domains from mail accounts. ${ORANGE}EXAMPLES${RESET} - ${LWHITE}./setup.sh config dkim keysize 2048${RESET} - Creates keys of length 2048 bit in a default setup where domains are obtained from - your accounts. + ${LWHITE}setup config dkim keysize 4096${RESET} + Creates keys with their length increased to a size of 4096-bit. - ${LWHITE}./setup.sh config dkim keysize 2048 selector 2021-dkim${RESET} - Creates keys of length 2048 bit in a default setup where domains are obtained from - your accounts. The DKIM selector used is '2021-dkim'. + ${LWHITE}setup config dkim keysize 1024 selector 2023-dkim${RESET} + Creates 1024-bit sized keys, and changes the DKIM selector to '2023-dkim'. - ${LWHITE}./setup.sh config dkim keysize 2048 selector 2021-dkim domain 'whoami.com,whoareyou.org'${RESET} - Appropriate for an LDAP setup. Creates keys of length 2048 bit in a default setup - where domains are obtained from your accounts. The DKIM selector used is '2021-dkim'. - The domains for which DKIM keys are generated are 'whoami.com' and 'whoareyou.org'. + ${LWHITE}setup config dkim domain 'example.com,another-example.com'${RESET} + Only generates DKIM keys for the specified domains: 'example.com' and 'another-example.com'. ${ORANGE}EXIT STATUS${RESET} Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 7627a863..de7129de 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -16,14 +16,14 @@ function __usage() { echo -e "${PURPLE}RSPAMD-DKIM${RED}(${YELLOW}8${RED}) ${ORANGE}NAME${RESET} - rspamd-dkim - Configure DomainKeys Identified Mail (DKIM) via Rspamd + rspamd-dkim - Configure DKIM (DomainKeys Identified Mail) ${ORANGE}SYNOPSIS${RESET} setup config dkim [ OPTIONS${RED}...${RESET} ] ${ORANGE}DESCRIPTION${RESET} - This script aids in creating DKIM signing keys. The keys are created and managed by Rspamd. - OPTIONS can be used to configure a more complex setup. + Creates DKIM keys and configures them within DMS for Rspamd. + OPTIONS can be used when your requirements are not met by the defaults. ${ORANGE}OPTIONS${RESET} ${BLUE}Generic Program Information${RESET} @@ -32,30 +32,27 @@ ${ORANGE}OPTIONS${RESET} help Print the usage information. ${BLUE}Configuration adjustments${RESET} - keytype Set the type of key you want to use + keytype Set the type of key you want to use. Possible values: rsa, ed25519 Default: rsa - keysize Set the size of the keys to be generated + keysize Set the size of the keys to be generated. Possible values: 1024, 2048 and 4096 Default: 2048 Only applies when using keytype=rsa - selector Set a manual selector for the key + selector Set a manual selector for the key. Default: mail - domain Provide the domain for which keys are to be generated - Default: primary domain name of DMS + domain Provide the domain for which to generate keys for. + Default: The FQDN assigned to DMS, excluding any subdomain. ${ORANGE}EXAMPLES${RESET} - ${LWHITE}setup config dkim keysize 2048${RESET} - Creates keys of length 2048 bit in a default setup where domains are obtained from - your accounts. + ${LWHITE}setup config dkim keysize 4096${RESET} + Creates keys with their length increased to a size of 4096-bit. - ${LWHITE}setup config dkim keysize 512 selector 2023-dkim${RESET} - Creates keys of length 512 bit in a default setup where domains are obtained from - your accounts. The DKIM selector used is '2023-dkim'. + ${LWHITE}setup config dkim keysize 1024 selector 2023-dkim${RESET} + Creates 1024-bit sized keys, and changes the DKIM selector to '2023-dkim'. - ${LWHITE}setup config dkim keysize 1024 selector 2023-dkim domain whoami.com${RESET} - Creates keys of length 1024 bit in a default setup where domains are obtained from your accounts. - The DKIM selector used is '2023-dkim'. The domain for which DKIM keys are generated is whoami.com. + ${LWHITE}setup config dkim domain example.com${RESET} + Generate the DKIM key for a different domain (example.com). ${ORANGE}EXIT STATUS${RESET} Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 97c98b6a..cd880e52 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -49,7 +49,7 @@ function teardown_file() { _default_teardown ; } _run_in_container setup config dkim help __log_is_free_of_warnings_and_errors assert_output --partial 'Showing usage message now' - assert_output --partial 'rspamd-dkim - Configure DomainKeys Identified Mail (DKIM) via Rspamd' + assert_output --partial 'rspamd-dkim - Configure DKIM (DomainKeys Identified Mail)' } @test 'default signing config is created if it does not exist and not overwritten' { @@ -143,7 +143,7 @@ function teardown_file() { _default_teardown ; } } @test "argument 'keysize' is applied correctly for RSA keys" { - for KEYSIZE in 512 1024 2048 4096; do + for KEYSIZE in 1024 2048 4096; do __create_key 'rsa' 'mail' "${DOMAIN_NAME}" "${KEYSIZE}" assert_success __log_is_free_of_warnings_and_errors diff --git a/test/tests/parallel/set3/scripts/setup_cli.bats b/test/tests/parallel/set3/scripts/setup_cli.bats index c5d2a2d6..dca61358 100644 --- a/test/tests/parallel/set3/scripts/setup_cli.bats +++ b/test/tests/parallel/set3/scripts/setup_cli.bats @@ -237,7 +237,7 @@ function teardown_file() { _default_teardown ; } @test "config dkim (help correctly displayed)" { run ./setup.sh -c "${CONTAINER_NAME}" config dkim help assert_success - assert_line --index 3 --partial " open-dkim - configure DomainKeys Identified Mail (DKIM)" + assert_line --index 3 --partial "open-dkim - Configure DKIM (DomainKeys Identified Mail)" } # debug diff --git a/test/tests/serial/open_dkim.bats b/test/tests/serial/open_dkim.bats index a71cb3e7..b4950d1f 100644 --- a/test/tests/serial/open_dkim.bats +++ b/test/tests/serial/open_dkim.bats @@ -113,14 +113,14 @@ function teardown() { _default_teardown ; } __init_container_without_waiting '/tmp/docker-mailserver' # generate first key (with a custom selector) - __should_generate_dkim_key 4 '2048' 'domain1.tld' 'mailer' + __should_generate_dkim_key 4 '1024' 'domain1.tld' 'mailer' __assert_outputs_common_dkim_logs # generate two additional keys different to the previous one - __should_generate_dkim_key 2 '2048' 'domain2.tld,domain3.tld' + __should_generate_dkim_key 2 '1024' 'domain2.tld,domain3.tld' __assert_logged_dkim_creation 'domain2.tld' __assert_logged_dkim_creation 'domain3.tld' # generate an additional key whilst providing already existing domains - __should_generate_dkim_key 1 '2048' 'domain3.tld,domain4.tld' + __should_generate_dkim_key 1 '1024' 'domain3.tld,domain4.tld' __assert_logged_dkim_creation 'domain4.tld' __should_have_tables_trustedhosts_for_domain @@ -197,7 +197,7 @@ function __should_support_creating_key_of_size() { __assert_logged_dkim_creation 'localhost.localdomain' __assert_logged_dkim_creation 'otherdomain.tld' - __should_have_expected_files "${EXPECTED_KEYSIZE:-4096}" + __should_have_expected_files "${EXPECTED_KEYSIZE:-2048}" _run_in_container rm -r /tmp/docker-mailserver/opendkim } From 351ef2afa171070c97ac3a0281acfebf987d7304 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:16:08 +1200 Subject: [PATCH 015/125] tests: LDAP - Improvements to LDIF (#3506) - The `uniqueIdentifier` attribute is not appropriate and was relying on `objectClass: extensibleObject` as a workaround to allow it. A more appropriate attribute to use instead is `userID` (_short name: `uid`_). - Removing `extensibleObject` now requires switching the user accounts to use `inetOrgPerson` class (_which inherits from `organizationalPerson`_). which allows the attributes `givenName`, `userID` and `mail` (_also provided via the `PostfixBookMailAccount` class_). - The LDAP root object now uses `dc` attributes for `example.test` instead of `localhost.localdomain`. This has nothing to do with DMS or LDAP containers networking config, nor the users mail addresses. - Users are now grouped under the organizational unit of `users` instead of `people`. Purely a naming change out of preference, no functional difference. The LDAP test ENV has been updated to accommodate the above changes. An additional ENV override was required for SASLAuthd to switch an attribute set for `ldap_filter` in `/etc/saslauthd.conf` from the implicit default of `uniqueIdentifier` (_that we set during startup as an ENV default for fallback_) to the `userID` attribute. --- .../bootstrap/ldif/01_mail-tree.ldif | 15 +++++++---- .../bootstrap/ldif/02_user-email.ldif | 7 +++-- .../03_user-email-other-primary-domain.ldif | 7 +++-- .../ldif/04_user-email-different-uid.ldif | 7 +++-- test/tests/serial/mail_with_ldap.bats | 27 ++++++++++--------- 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif index 5e3d8b0a..a6587f89 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif @@ -1,11 +1,16 @@ -# The root object, all entries will branch off this one: -dn: dc=localhost,dc=localdomain +# The root object of the tree, all entries will branch off this one: +dn: dc=example,dc=test +# DN is formed from `example.test` DNS labels: +# NOTE: This is just a common convention (not dependent on hostname or any external config) objectClass: dcObject +# Must reference left most component: +dc: example +# It's required to use an `objectClass` that implements a "Structural Class": objectClass: organization -dc: localhost +# Value is purely descriptive, not important to tests: o: DMS Test # User accounts will belong to this subtree: -dn: ou=people,dc=localhost,dc=localdomain +dn: ou=users,dc=example,dc=test objectClass: organizationalUnit -ou: people +ou: users diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif index cd656bf8..67696a81 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif @@ -1,12 +1,11 @@ # NOTE: A standard user account to test against -dn: uniqueIdentifier=some.user,ou=people,dc=localhost,dc=localdomain -objectClass: organizationalPerson +dn: uid=some.user,ou=users,dc=example,dc=test +objectClass: inetOrgPerson objectClass: PostfixBookMailAccount -objectClass: extensibleObject cn: Some User givenName: Some surname: User -uniqueIdentifier: some.user +userID: some.user # Password is: secret userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.user@localhost.localdomain diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif index d27cec7e..66a84343 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif @@ -1,13 +1,12 @@ # NOTE: This user differs via the domain-part of their mail address # They also have their mail directory attributes using the primary domain, not their domain-part -dn: uniqueIdentifier=some.other.user,ou=people,dc=localhost,dc=localdomain -objectClass: organizationalPerson +dn: uid=some.other.user,ou=users,dc=example,dc=test +objectClass: inetOrgPerson objectClass: PostfixBookMailAccount -objectClass: extensibleObject cn: Some Other User givenName: Some surname: Other User -uniqueIdentifier: some.other.user +userID: some.other.user # Password is: secret userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.other.user@localhost.otherdomain diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif index be49fd9f..e405c7b2 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif @@ -1,13 +1,12 @@ # NOTE: This user differs by local-part of mail address not matching their uniqueIdentifier attribute # They also do not have any alias or groups configured -dn: uniqueIdentifier=some.user.id,ou=people,dc=localhost,dc=localdomain -objectClass: organizationalPerson +dn: uid=some.user.id,ou=users,dc=example,dc=test +objectClass: inetOrgPerson objectClass: PostfixBookMailAccount -objectClass: extensibleObject cn: Some User givenName: Some surname: User -uniqueIdentifier: some.user.id +userID: some.user.id # Password is: secret userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.user.email@localhost.localdomain diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index baeb61e2..03a9052b 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -21,7 +21,7 @@ function setup_file() { # Setup local openldap service: docker run --rm -d --name "${CONTAINER2_NAME}" \ --env LDAP_ADMIN_PASSWORD=admin \ - --env LDAP_ROOT='dc=localhost,dc=localdomain' \ + --env LDAP_ROOT='dc=example,dc=test' \ --env LDAP_PORT_NUMBER=389 \ --env LDAP_SKIP_DEFAULT_TREE=yes \ --volume './test/config/ldap/docker-openldap/bootstrap/ldif/:/ldifs/:ro' \ @@ -37,26 +37,29 @@ function setup_file() { # local ENV_LDAP_CONFIG=( - # Configure for LDAP account provisioner and alternative to Dovecot SASL: --env ACCOUNT_PROVISIONER=LDAP + + # Postfix SASL auth provider (SASLAuthd instead of default Dovecot provider): --env ENABLE_SASLAUTHD=1 --env SASLAUTHD_MECHANISMS=ldap + --env SASLAUTHD_LDAP_FILTER='(&(userID=%U)(mailEnabled=TRUE))' # ENV to configure LDAP configs for Dovecot + Postfix: # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): # Dovecot: - --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' --env DOVECOT_TLS=no - --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' + # Postfix: - --env LDAP_BIND_DN='cn=admin,dc=localhost,dc=localdomain' + --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' --env LDAP_BIND_PW='admin' --env LDAP_QUERY_FILTER_ALIAS='(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))' --env LDAP_QUERY_FILTER_DOMAIN='(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))' --env LDAP_QUERY_FILTER_GROUP='(&(mailGroupMember=%s)(mailEnabled=TRUE))' - --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(uniqueIdentifier=some.user.id))' + --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(userID=some.user.id))' --env LDAP_QUERY_FILTER_USER='(&(mail=%s)(mailEnabled=TRUE))' - --env LDAP_SEARCH_BASE='ou=people,dc=localhost,dc=localdomain' + --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' --env LDAP_SERVER_HOST="${FQDN_LDAP}" --env LDAP_START_TLS=no ) @@ -108,7 +111,7 @@ function teardown_file() { # Test email receiving from a other domain then the primary domain of the mailserver _should_exist_in_ldap_tables "some.other.user@${FQDN_LOCALHOST_B}" - # Should not require `uniqueIdentifier` to match the local part of `mail` (`.ldif` defined settings): + # Should not require `userID` / `uid` to match the local part of `mail` (`.ldif` defined settings): # REF: https://github.com/docker-mailserver/docker-mailserver/pull/642#issuecomment-313916384 # NOTE: This account has no `mailAlias` or `mailGroupMember` defined in it's `.ldif`. local MAIL_ACCOUNT="some.user.email@${FQDN_LOCALHOST_A}" @@ -137,8 +140,8 @@ function teardown_file() { local LDAP_SETTINGS_POSTFIX=( "server_host = ${FQDN_LDAP}" 'start_tls = no' - 'search_base = ou=people,dc=localhost,dc=localdomain' - 'bind_dn = cn=admin,dc=localhost,dc=localdomain' + 'search_base = ou=users,dc=example,dc=test' + 'bind_dn = cn=admin,dc=example,dc=test' ) for LDAP_SETTING in "${LDAP_SETTINGS_POSTFIX[@]}"; do @@ -177,8 +180,8 @@ function teardown_file() { local LDAP_SETTINGS_DOVECOT=( "uris = ldap://${FQDN_LDAP}" 'tls = no' - 'base = ou=people,dc=localhost,dc=localdomain' - 'dn = cn=admin,dc=localhost,dc=localdomain' + 'base = ou=users,dc=example,dc=test' + 'dn = cn=admin,dc=example,dc=test' ) for LDAP_SETTING in "${LDAP_SETTINGS_DOVECOT[@]}"; do From 9446fa9b9ad135001daae153ec95e8afcb1a2ca5 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:19:03 +1200 Subject: [PATCH 016/125] chore: Adapt `ENABLE_LDAP=1` to `ACCOUNT_PROVISIONER=LDAP` (#3507) - Deprecation startup script check is kept for `ENABLE_LDAP=1` but adjusted to emit an error instead. It can be dropped in a future release. Just a precaution for those who mistakenly update (_possibly via automation_) without checking the release notes, an error log is somewhat helpful, although it could alternatively panic? - Docs updated to remove the `ENABLE_LDAP=1` usage - ENV docs updated to reference a maintained LDAP image. - Changelog includes the breaking change, and slight revision to prior release mention of deprecation. --- CHANGELOG.md | 14 +++++++++----- docs/content/config/advanced/auth-ldap.md | 3 --- docs/content/config/environment.md | 4 +--- ...ard-only-mailserver-with-ldap-authentication.md | 1 - mailserver.env | 7 +------ target/scripts/startup/variables-stack.sh | 3 +-- 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 028bffe4..8b22fec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Breaking + +- The environment variable `ENABLE_LDAP=1` has been changed to `ACCOUNT_PROVISIONER=LDAP`. + ### Added - New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) @@ -210,7 +214,11 @@ Notable changes are: ### Summary -This release features a lot of small and medium-sized changes, many related to how the image is build and tested during CI. The build now requires Docker Buildkit as the ClamAV Signatures are added via `COPY --link ...` during build-time. Moreover, the build is now multi-stage. `ENABLE_LDAP` is now deprecated. +This release features a lot of small and medium-sized changes, many related to how the image is build and tested during CI. The build now multi-stage based and requires Docker Buildkit, as the ClamAV Signatures are added via `COPY --link ...` during build-time. + +### Deprecated + +- The environment variable `ENABLE_LDAP` is deprecated and will be removed in [13.0.0]. Use `ACCOUNT_PROVISIONER=LDAP` now. ### Added @@ -237,10 +245,6 @@ This release features a lot of small and medium-sized changes, many related to h - **build**: adjust build arguments - **build**: enhance build process -### Deprecated - -- The environment variable `ENABLE_LDAP` is deprecated and will be removed in [13.0.0]. Use `ACCOUNT_PROVISIONER=LDAP` now. - ### Removed - **configuration**: remove unnecessary configuration files diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index 44d3aec9..c275fd6c 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -34,7 +34,6 @@ Those variables contain the LDAP lookup filters for postfix, using `%s` as the p A really simple `LDAP_QUERY_FILTER` configuration, using only the _user filter_ and allowing only `admin@*` to spoof any sender addresses. ```yaml - - ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER - LDAP_START_TLS=yes - ACCOUNT_PROVISIONER=LDAP - LDAP_SERVER_HOST=ldap.example.org @@ -215,7 +214,6 @@ The changes on the configurations necessary to work with Active Directory (**onl - ENABLE_POSTGREY=1 # >>> Postfix LDAP Integration - - ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER - ACCOUNT_PROVISIONER=LDAP - LDAP_SERVER_HOST=ldap.example.org - LDAP_BIND_DN=cn=admin,ou=users,dc=example,dc=org @@ -284,7 +282,6 @@ The changes on the configurations necessary to work with Active Directory (**onl # <<< SASL Authentication # >>> Postfix Ldap Integration - - ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER - ACCOUNT_PROVISIONER=LDAP - LDAP_SERVER_HOST= - LDAP_SEARCH_BASE=dc=mydomain,dc=loc diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index bea02ecb..3e40bd89 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -49,7 +49,7 @@ User provisioning via OIDC is planned for the future, see [this tracking issue]( - OIDC => use OIDC authentication (**not yet implemented**) - FILE => use local files (this is used as the default) -A second container for the ldap service is necessary (e.g. [docker-openldap](https://github.com/osixia/docker-openldap)) +A second container for the ldap service is necessary (e.g. [`bitnami/openldap`](https://hub.docker.com/r/bitnami/openldap/)). ##### PERMIT_DOCKER @@ -584,9 +584,7 @@ Enable or disable `getmail`. #### LDAP -##### ENABLE_LDAP -Deprecated. See [`ACCOUNT_PROVISIONER`](#account_provisioner). ##### LDAP_START_TLS diff --git a/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md b/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md index dc930f13..85a5726a 100644 --- a/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md +++ b/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md @@ -30,7 +30,6 @@ We can create aliases with `./setup.sh`, like this: If you want to send emails from outside the mail server you have to authenticate somehow (with a username and password). One way of doing it is described in [this discussion][github-issue-1247]. However if there are many user accounts, it is better to use authentication with LDAP. The settings for this on `mailserver.env` are: ```env -ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER ACCOUNT_PROVISIONER=LDAP LDAP_START_TLS=yes LDAP_SERVER_HOST=ldap.example.org diff --git a/mailserver.env b/mailserver.env index 6f270af4..8e753006 100644 --- a/mailserver.env +++ b/mailserver.env @@ -405,12 +405,7 @@ GETMAIL_POLL=5 # --- LDAP Section ------------------------------ # ----------------------------------------------- -# A second container for the ldap service is necessary (i.e. https://github.com/osixia/docker-openldap) - -# with the :edge tag, use ACCOUNT_PROVISIONER=LDAP -# empty => LDAP authentication is disabled -# 1 => LDAP authentication is enabled -ENABLE_LDAP= +# A second container for the ldap service is necessary (i.e. https://hub.docker.com/r/bitnami/openldap/) # empty => no # yes => LDAP over TLS enabled for Postfix diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index a0d61242..cf099220 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -14,8 +14,7 @@ function _early_variables_setup() { # completely with a single version. function __environment_variables_backwards_compatibility() { if [[ ${ENABLE_LDAP:-0} -eq 1 ]]; then - _log 'warn' "'ENABLE_LDAP=1' is deprecated (and will be removed in v13.0.0) => use 'ACCOUNT_PROVISIONER=LDAP' instead" - ACCOUNT_PROVISIONER='LDAP' + _log 'error' "'ENABLE_LDAP=1' has been changed to 'ACCOUNT_PROVISIONER=LDAP' since DMS v13" fi # TODO this can be uncommented in a PR handling the HOSTNAME/DOMAINNAME issue From 19b72aead3e8015b3b855ed0678352ff596ab74c Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 23:33:39 +1200 Subject: [PATCH 017/125] docs: Update docs builder image (#3516) - Bump to release `9.2.x` - Image now has `MAJOR.MINOR` tag support to pull latest `PATCH` versions. --- .github/workflows/scripts/docs/build-docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index dd9ef3a5..aff66ab5 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -10,7 +10,7 @@ docker run \ --user "$(id -u):$(id -g)" \ --volume "${PWD}:/docs" \ --name "build-docs" \ - squidfunk/mkdocs-material:9.1.5 build --strict + squidfunk/mkdocs-material:9.2 build --strict # Remove unnecessary build artifacts: https://github.com/squidfunk/mkdocs-material/issues/2519 # site/ is the build output folder. From e025e4c6962d2f0f6147461f383853025e1a6986 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 23:52:06 +1200 Subject: [PATCH 018/125] tests: Revise LDAP config + setup (#3514) * chore: Use white-space in query filters to improve readability * tests: LDAP ENV query filters documented - These filters remain roughly the same as they were before. The conditions are the same, but restructured to make the complimentary constraints more separated from the actual target attribtues. - The DOMAIN query additionally includes the `mailAlias` from `PostfixBookMailAccount` class as well. - Heavy inline documentation breaking down how the filters work for any maintainer to reference. This will likely be migrated after revision into our user docs for LDAP. Some quirks have also been noted with advice for future changes. * tests: LDAP - Support test-case specific containers A bit more complicated than other test files due to the larger ENV config array that most containers may need to share. Example introduced with the test-case checking custom config file support. * tests: Adjust LDAP test configs - Paths for `.ldif` files used with volumes shortened - Postfix LDAP `.cf` files adjusted to conventions used in LDAP tests. --- .../ldif => openldap/ldifs}/01_mail-tree.ldif | 0 .../ldifs}/02_user-email.ldif | 0 .../03_user-email-other-primary-domain.ldif | 0 .../ldifs}/04_user-email-different-uid.ldif | 0 .../schemas/postfix-book.ldif | 0 test/config/ldap/overrides/ldap-aliases.cf | 6 +- test/config/ldap/overrides/ldap-groups.cf | 18 +-- test/config/ldap/overrides/ldap-users.cf | 6 +- test/tests/serial/mail_with_ldap.bats | 127 +++++++++++++++--- 9 files changed, 120 insertions(+), 37 deletions(-) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/01_mail-tree.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/02_user-email.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/03_user-email-other-primary-domain.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/04_user-email-different-uid.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap => openldap}/schemas/postfix-book.ldif (100%) diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/openldap/ldifs/01_mail-tree.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif rename to test/config/ldap/openldap/ldifs/01_mail-tree.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/openldap/ldifs/02_user-email.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif rename to test/config/ldap/openldap/ldifs/02_user-email.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/openldap/ldifs/03_user-email-other-primary-domain.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif rename to test/config/ldap/openldap/ldifs/03_user-email-other-primary-domain.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/openldap/ldifs/04_user-email-different-uid.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif rename to test/config/ldap/openldap/ldifs/04_user-email-different-uid.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif b/test/config/ldap/openldap/schemas/postfix-book.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif rename to test/config/ldap/openldap/schemas/postfix-book.ldif diff --git a/test/config/ldap/overrides/ldap-aliases.cf b/test/config/ldap/overrides/ldap-aliases.cf index a4579393..2436aa41 100644 --- a/test/config/ldap/overrides/ldap-aliases.cf +++ b/test/config/ldap/overrides/ldap-aliases.cf @@ -1,10 +1,10 @@ # Testconfig for ldap integration bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=test bind_pw = admin query_filter = (&(mailAlias=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=users,dc=example,dc=test +server_host = mail.example.test start_tls = no version = 3 diff --git a/test/config/ldap/overrides/ldap-groups.cf b/test/config/ldap/overrides/ldap-groups.cf index 6712db9e..b0b8da21 100644 --- a/test/config/ldap/overrides/ldap-groups.cf +++ b/test/config/ldap/overrides/ldap-groups.cf @@ -1,10 +1,10 @@ # Testconfig for ldap integration -bind = yes -bind_dn = cn=admin,dc=domain,dc=com -bind_pw = admin -query_filter = (&(mailGroupMember=%s)(mailEnabled=TRUE)) -result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com -start_tls = no -version = 3 +bind = yes +bind_dn = cn=admin,dc=example,dc=test +bind_pw = admin +query_filter = (&(mailGroupMember=%s)(mailEnabled=TRUE)) +result_attribute = mail +search_base = ou=users,dc=example,dc=test +server_host = mail.example.test +start_tls = no +version = 3 diff --git a/test/config/ldap/overrides/ldap-users.cf b/test/config/ldap/overrides/ldap-users.cf index 92cd2ed5..b32db9fd 100644 --- a/test/config/ldap/overrides/ldap-users.cf +++ b/test/config/ldap/overrides/ldap-users.cf @@ -1,10 +1,10 @@ # Testconfig for ldap integration bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=test bind_pw = admin query_filter = (&(mail=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=users,dc=example,dc=test +server_host = mail.example.test start_tls = no version = 3 diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 03a9052b..c3c60b92 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -4,12 +4,14 @@ load "${REPOSITORY_ROOT}/test/helper/common" BATS_TEST_NAME_PREFIX='[LDAP] ' CONTAINER1_NAME='dms-test_ldap' CONTAINER2_NAME='dms-test_ldap_provider' +# Single test-case specific containers: +CONTAINER3_NAME='dms-test_ldap_custom-config' function setup_file() { export DMS_TEST_NETWORK='test-network-ldap' - export DOMAIN='example.test' - export FQDN_MAIL="mail.${DOMAIN}" - export FQDN_LDAP="ldap.${DOMAIN}" + export DMS_DOMAIN='example.test' + export FQDN_MAIL="mail.${DMS_DOMAIN}" + export FQDN_LDAP="ldap.${DMS_DOMAIN}" # LDAP is provisioned with two domains (via `.ldif` files) unrelated to the FQDN of DMS: export FQDN_LOCALHOST_A='localhost.localdomain' export FQDN_LOCALHOST_B='localhost.otherdomain' @@ -24,8 +26,8 @@ function setup_file() { --env LDAP_ROOT='dc=example,dc=test' \ --env LDAP_PORT_NUMBER=389 \ --env LDAP_SKIP_DEFAULT_TREE=yes \ - --volume './test/config/ldap/docker-openldap/bootstrap/ldif/:/ldifs/:ro' \ - --volume './test/config/ldap/docker-openldap/bootstrap/schemas/:/schemas/:ro' \ + --volume "${REPOSITORY_ROOT}/test/config/ldap/openldap/ldifs/:/ldifs/:ro" \ + --volume "${REPOSITORY_ROOT}/test/config/ldap/openldap/schemas/:/schemas/:ro" \ --hostname "${FQDN_LDAP}" \ --network "${DMS_TEST_NETWORK}" \ bitnami/openldap:latest @@ -36,35 +38,86 @@ function setup_file() { # Setup DMS container # + # LDAP filter queries explained. + # NOTE: All LDAP configs for Postfix (with the exception of `ldap-senders.cf`), return the `mail` attribute value of matched results. + # This is through the config key `result_attribute`, which the ENV substitution feature can only replace across all configs, not selectively like `query_filter`. + # NOTE: The queries below rely specifically upon attributes and classes defined by the schema `postfix-book.ldif`. These are not compatible with all LDAP setups. + + # `mailAlias`` is supported by both classes provided from the schema `postfix-book.ldif`, but `mailEnabled` is only available to `PostfixBookMailAccount` class: + local QUERY_ALIAS='(&(mailAlias=%s) (| (objectClass=PostfixBookMailForward) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ))' + + # Postfix does domain lookups with the domain of the recipient to check if DMS manages the mail domain. + # For this lookup `%s` only represents the domain, not a full email address. Hence the match pattern using a wildcard prefix `*@`. + # For a breakdown, see QUERY_SENDERS comment. + # NOTE: Although `result_attribute = mail` will return each accounts full email address, Postfix will only compare to domain-part. + local QUERY_DOMAIN='(| (& (|(mail=*@%s) (mailAlias=*@%s) (mailGroupMember=*@%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=*@%s)(objectClass=PostfixBookMailForward)) )' + + # Simple queries for a single attribute that additionally requires `mailEnabled=TRUE` from the `PostfixBookMailAccount` class: + # NOTE: `mail` attribute is not unique to `PostfixBookMailAccount`. The `mailEnabled` attribute is to further control valid mail accounts. + # TODO: For tests, since `mailEnabled` is not relevant (always configured as TRUE currently), + # a simpler query like `mail=%s` or `mailGroupMember=%s` would be sufficient. The additional constraints could be covered in our docs instead. + local QUERY_GROUP='(&(mailGroupMember=%s) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) )' + local QUERY_USER='(&(mail=%s) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) )' + + # Given the sender address `%s` from Postfix, query LDAP for accounts that meet the search filter, + # the `result_attribute` is `mail` + `uid` (`userID`) attributes for login names that are authorized to use that sender address. + # One of the result values returned must match the login name of the user trying to send mail with that sender address. + # + # Filter: Search for accounts that meet any of the following conditions: + # - Has any attribute (`mail`, `mailAlias`, `mailGroupMember`) with a value of `%s`, AND is of class `PostfixBookMailAccount` with `mailEnabled=TRUE` attribute set. + # - Has attribute `mailAlias` with value of `%s` AND is of class PostfixBookMailForward + # - Has the attribute `userID` with value `some.user.id` + local QUERY_SENDERS='(| (& (|(mail=%s) (mailAlias=%s) (mailGroupMember=%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=%s)(objectClass=PostfixBookMailForward)) (userID=some.user.id))' + + # NOTE: The remaining queries below have been left as they were instead of rewritten, `mailEnabled` has no associated class requirement, nor does the class requirement ensure `mailEnabled=TRUE`. + + # When using SASLAuthd for login auth (only valid to Postfix in DMS), + # Postfix will pass on the login username and SASLAuthd will use it's backend to do a lookup. + # The current default is `uniqueIdentifier=%u`, but `%u` is inaccurate currently with LDAP backend + # as SASLAuthd will ignore the domain-part received (if any) and forward only the local-part as the query. + # TODO: Fix this by having supervisor start the service with `-r` option like it does `rimap` backend. + # NOTE: `%u` (full login with realm/domain-part) is presently equivalent to `%U` (local-part) only, + # As the `userID` is not a mail address, we ensure any domain-part is ignored, as a login name is not + # required to match the mail accounts actual `mail` attribute (nor the local-part), they are distinct. + # TODO: Docs should better cover this difference, as it does confuse some users of DMS (and past contributors to our tests..). + local SASLAUTHD_QUERY='(&(userID=%U)(mailEnabled=TRUE))' + + # Dovecot is configured to lookup a user account by their login name (`userID` in this case, but it could be any attribute like `mail`). + # Dovecot syntax token `%n` is the local-part of the full email address supplied as the login name. There must be a unique match on `userID` (which there will be as each account is configured via LDIF to use it in their DN) + # NOTE: We already have a constraint on the LDAP tree to search (`LDAP_SEARCH_BASE`), if all objects in that subtree use `PostfixBookMailAccount` class then there is no benefit in the extra constraint. + # TODO: For tests, that additional constraint is meaningless. We can detail it in our docs instead and just use `userID=%n`. + local DOVECOT_QUERY_PASS='(&(userID=%n)(objectClass=PostfixBookMailAccount))' + local DOVECOT_QUERY_USER='(&(userID=%n)(objectClass=PostfixBookMailAccount))' + local ENV_LDAP_CONFIG=( --env ACCOUNT_PROVISIONER=LDAP # Postfix SASL auth provider (SASLAuthd instead of default Dovecot provider): --env ENABLE_SASLAUTHD=1 --env SASLAUTHD_MECHANISMS=ldap - --env SASLAUTHD_LDAP_FILTER='(&(userID=%U)(mailEnabled=TRUE))' + --env SASLAUTHD_LDAP_FILTER="${SASLAUTHD_QUERY}" # ENV to configure LDAP configs for Dovecot + Postfix: # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): # Dovecot: - --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' + --env DOVECOT_PASS_FILTER="${DOVECOT_QUERY_PASS}" --env DOVECOT_TLS=no - --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' + --env DOVECOT_USER_FILTER="${DOVECOT_QUERY_USER}" # Postfix: --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' --env LDAP_BIND_PW='admin' - --env LDAP_QUERY_FILTER_ALIAS='(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))' - --env LDAP_QUERY_FILTER_DOMAIN='(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))' - --env LDAP_QUERY_FILTER_GROUP='(&(mailGroupMember=%s)(mailEnabled=TRUE))' - --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(userID=some.user.id))' - --env LDAP_QUERY_FILTER_USER='(&(mail=%s)(mailEnabled=TRUE))' + --env LDAP_QUERY_FILTER_ALIAS="${QUERY_ALIAS}" + --env LDAP_QUERY_FILTER_DOMAIN="${QUERY_DOMAIN}" + --env LDAP_QUERY_FILTER_GROUP="${QUERY_GROUP}" + --env LDAP_QUERY_FILTER_SENDERS="${QUERY_SENDERS}" + --env LDAP_QUERY_FILTER_USER="${QUERY_USER}" --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' --env LDAP_SERVER_HOST="${FQDN_LDAP}" --env LDAP_START_TLS=no ) - # Extra ENV needed to support specific testcases: + # Extra ENV needed to support specific test-cases: local ENV_SUPPORT=( --env PERMIT_DOCKER=container # Required for attempting SMTP auth on port 25 via nc # Required for openssl commands to be successul: @@ -79,22 +132,40 @@ function setup_file() { --env SPOOF_PROTECTION=1 ) + export CONTAINER_NAME=${CONTAINER1_NAME} local CUSTOM_SETUP_ARGUMENTS=( - --hostname "${FQDN_MAIL}" - --network "${DMS_TEST_NETWORK}" - "${ENV_LDAP_CONFIG[@]}" "${ENV_SUPPORT[@]}" + + --hostname "${FQDN_MAIL}" + --network "${DMS_TEST_NETWORK}" ) - # Set default implicit container fallback for helpers: - export CONTAINER_NAME=${CONTAINER1_NAME} + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_smtp_port_in_container + # Single test-case containers below cannot be defined in a test-case when expanding arrays + # defined in `setup()`. Those arrays would need to be hoisted up to the top of the file vars. + # Alternatively for ENV overrides, separate `.env` files could be used. Better options + # are available once switching to `compose.yaml` in tests. + + export CONTAINER_NAME=${CONTAINER3_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + "${ENV_LDAP_CONFIG[@]}" + + # `hostname` should be unique when connecting containers via network: + --hostname "custom-config.${DMS_DOMAIN}" + --network "${DMS_TEST_NETWORK}" + ) _init_with_defaults # NOTE: `test/config/` has now been duplicated, can move test specific files to host-side `/tmp/docker-mailserver`: mv "${TEST_TMP_CONFIG}/ldap/overrides/"*.cf "${TEST_TMP_CONFIG}/" _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - _wait_for_smtp_port_in_container + + + # Set default implicit container fallback for helpers: + export CONTAINER_NAME=${CONTAINER1_NAME} } function teardown_file() { @@ -102,6 +173,17 @@ function teardown_file() { docker network rm "${DMS_TEST_NETWORK}" } +# Could optionally call `_default_teardown` in test-cases that have specific containers. +# This will otherwise handle it implicitly which is helpful when the test-case hits a failure, +# As failure will bail early missing teardown, which then prevents network cleanup. This way is safer: +function teardown() { + if [[ ${CONTAINER_NAME} != "${CONTAINER1_NAME}" ]] \ + && [[ ${CONTAINER_NAME} != "${CONTAINER2_NAME}" ]] + then + _default_teardown + fi +} + # postfix # NOTE: Each of the 3 user accounts tested below are defined in separate LDIF config files, # Those are bundled into the locally built OpenLDAP Dockerfile. @@ -121,9 +203,10 @@ function teardown_file() { } # Custom LDAP config files support: -# TODO: Compare to provided configs and if they're just including a test comment, -# could just copy the config and append without carrying a separate test config? @test "postfix: ldap custom config files copied" { + # Use the test-case specific container from `setup()` (change only applies to test-case): + export CONTAINER_NAME=${CONTAINER3_NAME} + local LDAP_CONFIGS_POSTFIX=( /etc/postfix/ldap-users.cf /etc/postfix/ldap-groups.cf From 3d5f6aeec40ac1c719becdb031fe306f33bcfcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Pl=C3=B6tz?= Date: Tue, 29 Aug 2023 23:40:54 +0200 Subject: [PATCH 019/125] docs: Add documentation for iOS mail push support (#3513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add documentation for iOS mail push support --------- Signed-off-by: René Plötz --- .../use-cases/ios-mail-push-support.md | 232 ++++++++++++++++++ docs/mkdocs.yml | 1 + 2 files changed, 233 insertions(+) create mode 100644 docs/content/examples/use-cases/ios-mail-push-support.md diff --git a/docs/content/examples/use-cases/ios-mail-push-support.md b/docs/content/examples/use-cases/ios-mail-push-support.md new file mode 100644 index 00000000..d0eacd18 --- /dev/null +++ b/docs/content/examples/use-cases/ios-mail-push-support.md @@ -0,0 +1,232 @@ +--- +title: 'Advanced | iOS Mail Push Support' +--- + +## Introduction + +iOS Mail currently does not support the IMAP idle extension. Therefore users can only either check manually or configure intervals for fetching mails in their mail account preferences when using the default configuration. + +To support mail push Dovecot needs to advertise the `XAPPLEPUSHSERVICE` IMAP extension as well as sending the actual push notifications to the Apple Push Notification service (APNs) which will forward them to the device. + +This can be done with two components: + +- A Dovecot plugin (`dovecot-xaps-plugin`) which is triggered whenever a mail is created or moved from/to a mail folder. +- A daemon service (`dovecot-xaps-daemon`) that manages both the device registrations as well as sending notifications to the APNs. + +## Prerequisites + +- An Apple developer account to create the required Apple Push Notification service certificate. +- Knowledge creating Docker images, using the command-line, and creating shell scripts. + +## Limitations + +- You need to maintain a custom `docker-mailserver` image. +- Push support is limited to the INBOX folder. Changes to other folders will not be pushed to the device regardless of the configuration settings. +- You currently cannot use the same account UUID on multiple devices. This means that if you use the same backup on multiple devices (e.g. old phone / new phone) only one of them will get the notification. Use different backups or recreate the mail account. + +## Privacy concerns + +- The service does not send any part of the actual message to Apple. +- The information sent contains the device UUID to notify and the (on-device) account UUID which was generated by the iOS mail application when creating the account. +- Upon receiving the notification, the iOS mail application will connect to the IMAP server given by the provided account UUID and fetch the mail to notify the user. +- Apple therefore does not know the mail address for which the mail was received, only that a specific account on a specific device should be notified that a new mail or that a mail was moved to the INBOX folder. + +## Installation + +Both components will be built using Docker and included into a custom `docker-mailserver` image. Afterwards the required configuration is added to `docker-data/dms/config`. The registration data is stored in `/var/mail-state/lib-xapsd`. + +1. Create a Dockerfile to build a `docker-mailserver` image that includes the [`dovecot-xaps-plugin`](https://github.com/freswa/dovecot-xaps-plugin) as well as the [`dovecot-xaps-daemon`](https://github.com/freswa/dovecot-xaps-daemon). This is required to ensure that the Dovecot plugin is built against the same Dovecot version. The `:edge` tag is used here, but you might want to use a released version instead. + + ```dockerfile + FROM mailserver/docker-mailserver:edge AS dovecot-plugin-xaps + WORKDIR /tmp/dovecot-xaps-plugin + RUN <= 1.20 causes this issue: https://github.com/freswa/dovecot-xaps-daemon/issues/24#issuecomment-1483876081 + # Note that the underlying issue are non-standard-compliant Apple http servers which might get fixed at some point + FROM golang:1.19-alpine AS dovecot-xaps-daemon + ENV GOPROXY=https://proxy.golang.org,direct + ENV CGO_ENABLED=0 + WORKDIR /go/dovecot-xaps-daemon + RUN < Date: Fri, 1 Sep 2023 13:54:38 +0200 Subject: [PATCH 020/125] tests: add tests for helper function `_add_to_or_update_postfix_main()` (#3505) --- .../set3/scripts/postconf-helper.bats | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 test/tests/parallel/set3/scripts/postconf-helper.bats diff --git a/test/tests/parallel/set3/scripts/postconf-helper.bats b/test/tests/parallel/set3/scripts/postconf-helper.bats new file mode 100644 index 00000000..d77d50ac --- /dev/null +++ b/test/tests/parallel/set3/scripts/postconf-helper.bats @@ -0,0 +1,48 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +BATS_TEST_NAME_PREFIX='[Scripts] (helper functions) (postfix - _add_to_or_update_postfix_main) ' +CONTAINER_NAME='dms-test_postconf-helper' +# Various tests for the helper function `_add_to_or_update_postfix_main()` +function setup_file() { + _init_with_defaults + _common_container_setup + # Begin tests without 'relayhost' defined in 'main.cf' + _run_in_container postconf -X relayhost + assert_success +} +function teardown_file() { _default_teardown ; } +# Add or modify in Postfix config `main.cf` a parameter key with the provided value. +# When the key already exists, the new value is appended (default), or prepended (explicitly requested). +# NOTE: This test-case helper is hard-coded for testing with the 'relayhost' parameter. +# +# @param ${1} = new value (appended or prepended) +# @param ${2} = action "append" (default) or "prepend" [OPTIONAL] +function _modify_postfix_main_config() { + _run_in_container_bash "source /usr/local/bin/helpers/{postfix,utils}.sh && _add_to_or_update_postfix_main relayhost '${1}' '${2}'" + _run_in_container grep '^relayhost' '/etc/postfix/main.cf' +} +@test "check if initial value is empty" { + _run_in_container postconf -h 'relayhost' + assert_output '' +} +@test "add single value" { + _modify_postfix_main_config 'single-value-test' + assert_output 'relayhost = single-value-test' +} +@test "prepend value" { + _modify_postfix_main_config 'prepend-test' 'prepend' + assert_output 'relayhost = prepend-test single-value-test' +} +@test "append value (explicit)" { + _modify_postfix_main_config 'append-test-explicit' 'append' + assert_output 'relayhost = prepend-test single-value-test append-test-explicit' +} +@test "append value (implicit)" { + _modify_postfix_main_config 'append-test-implicit' + assert_output 'relayhost = prepend-test single-value-test append-test-explicit append-test-implicit' +} +@test "try to append already existing value" { + _modify_postfix_main_config 'append-test-implicit' + assert_output 'relayhost = prepend-test single-value-test append-test-explicit append-test-implicit' +} From ed84dca1471fa6d599f1672c874ab3585a2b0074 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 2 Sep 2023 22:07:02 +1200 Subject: [PATCH 021/125] chore: LDAP config improvements (#3522) * chore: Drop management of `SASLAUTHD_*` ENV - `variables-stack.sh` does not need to manage all these extra ENV or store them. They're not used anywhere else. - `saslauthd.sh` is the only consumer of these ENV which are effectively direct key/value mappings, with some defaults provided / inherited. Instead of trying to conditionally support key/value pairs when ENV is set, we could instead use `sed` to delete lines with empty values. * chore: Drop fallbacks + update configs to match docs - Drop deprecated support: - `DOVECOT_HOSTS` is an ENV deprecated since v10. - Fallback for missing URI scheme introduced for Dovecot and SASLAuthd in v10. - Adding error log message when no LDAP URI scheme is detected for the supported ENV (when set). - Docs updated for ENV to reflect the mandatory requirement. `mailserver.env` partially synced equivalent sections. - Provided base LDAP configs (for overriding) likewise updated from `domain.com` to `example.com`. - LDAP test updated for required `ldap://` URI scheme. Common ENV shared across LDAP configs hoisted out of the Postfix group. * chore: Remove unset lines in generated `saslauthd.conf` --- docs/content/config/environment.md | 11 ++-- mailserver.env | 8 +-- target/dovecot/dovecot-ldap.conf.ext | 6 +-- target/postfix/ldap-aliases.cf | 6 +-- target/postfix/ldap-domains.cf | 6 +-- target/postfix/ldap-groups.cf | 6 +-- target/postfix/ldap-senders.cf | 6 +-- target/postfix/ldap-users.cf | 6 +-- target/scripts/startup/setup.d/ldap.sh | 8 +-- target/scripts/startup/setup.d/saslauthd.sh | 39 +++++++------- target/scripts/startup/variables-stack.sh | 56 ++++----------------- test/tests/serial/mail_with_ldap.bats | 19 ++++--- 12 files changed, 69 insertions(+), 108 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 3e40bd89..e5aba889 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -594,8 +594,8 @@ Enable or disable `getmail`. ##### LDAP_SERVER_HOST - **empty** => mail.example.com -- => Specify the dns-name/ip-address where the ldap-server is listening, or an URI like `ldaps://mail.example.com` -- NOTE: If you going to use DMS in combination with `compose.yaml` you can set the service name here +- => Specify the `` / `` where the LDAP server is reachable via a URI like: `ldaps://mail.example.com`. +- Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). ##### LDAP_SEARCH_BASE @@ -669,9 +669,8 @@ The following variables overwrite the default values for ```/etc/dovecot/dovecot ##### DOVECOT_URIS - **empty** => same as `LDAP_SERVER_HOST` -- => Specify a space separated list of LDAP uris. -- Note: If the protocol is missing, `ldap://` will be used. -- Note: This deprecates `DOVECOT_HOSTS` (as it didn't allow to use LDAPS), which is currently still supported for backwards compatibility. +- => Specify a space separated list of LDAP URIs. +- Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). ##### DOVECOT_LDAP_VERSION @@ -764,7 +763,7 @@ Note: This postgrey setting needs `ENABLE_POSTGREY=1` ##### SASLAUTHD_LDAP_SERVER - **empty** => same as `LDAP_SERVER_HOST` -- Note: since version 10.0.0, you can specify a protocol here (like ldaps://); this deprecates SASLAUTHD_LDAP_SSL. +- Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). ##### SASLAUTHD_LDAP_START_TLS diff --git a/mailserver.env b/mailserver.env index 8e753006..632f6edf 100644 --- a/mailserver.env +++ b/mailserver.env @@ -411,9 +411,9 @@ GETMAIL_POLL=5 # yes => LDAP over TLS enabled for Postfix LDAP_START_TLS= -# If you going to use the mailserver in combination with Docker Compose you can set the service name here -# empty => mail.domain.com -# Specify the dns-name/ip-address where the ldap-server +# empty => mail.example.com +# Specify the `` / `` where the LDAP server is reachable via a URI like: `ldaps://mail.example.com`. +# Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). LDAP_SERVER_HOST= # empty => ou=people,dc=domain,dc=com @@ -500,7 +500,7 @@ SASLAUTHD_MECHANISMS= SASLAUTHD_MECH_OPTIONS= # empty => Use value of LDAP_SERVER_HOST -# Note: since version 10.0.0, you can specify a protocol here (like ldaps://); this deprecates SASLAUTHD_LDAP_SSL. +# Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). SASLAUTHD_LDAP_SERVER= # empty => Use value of LDAP_BIND_DN diff --git a/target/dovecot/dovecot-ldap.conf.ext b/target/dovecot/dovecot-ldap.conf.ext index edf04b9e..66311106 100644 --- a/target/dovecot/dovecot-ldap.conf.ext +++ b/target/dovecot/dovecot-ldap.conf.ext @@ -1,8 +1,8 @@ -base = ou=people,dc=domain,dc=com +base = ou=people,dc=example,dc=com default_pass_scheme = SSHA -dn = cn=admin,dc=domain,dc=com +dn = cn=admin,dc=example,dc=com dnpass = admin -uris = ldap://mail.domain.com +uris = ldap://mail.example.com tls = no ldap_version = 3 pass_attrs = uniqueIdentifier=user,userPassword=password diff --git a/target/postfix/ldap-aliases.cf b/target/postfix/ldap-aliases.cf index 73bfe722..5c8d443a 100644 --- a/target/postfix/ldap-aliases.cf +++ b/target/postfix/ldap-aliases.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(mailAlias=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-domains.cf b/target/postfix/ldap-domains.cf index 5edd2441..c118ebf6 100644 --- a/target/postfix/ldap-domains.cf +++ b/target/postfix/ldap-domains.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(|(mail=*@%s)(mailalias=*@%s))(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-groups.cf b/target/postfix/ldap-groups.cf index 914e31a1..dc7fa14f 100644 --- a/target/postfix/ldap-groups.cf +++ b/target/postfix/ldap-groups.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(mailGroupMember=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-senders.cf b/target/postfix/ldap-senders.cf index 5f1ed6f5..88a6cd87 100644 --- a/target/postfix/ldap-senders.cf +++ b/target/postfix/ldap-senders.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (mail=%s) result_attribute = mail, uid -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-users.cf b/target/postfix/ldap-users.cf index a7b29cb6..943801f4 100644 --- a/target/postfix/ldap-users.cf +++ b/target/postfix/ldap-users.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(mail=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/scripts/startup/setup.d/ldap.sh b/target/scripts/startup/setup.d/ldap.sh index ceaca4eb..1451ec32 100644 --- a/target/scripts/startup/setup.d/ldap.sh +++ b/target/scripts/startup/setup.d/ldap.sh @@ -38,13 +38,7 @@ function _setup_ldap() { DOVECOT_LDAP_MAPPING['DOVECOT_BASE']="${DOVECOT_BASE:="${LDAP_SEARCH_BASE}"}" DOVECOT_LDAP_MAPPING['DOVECOT_DN']="${DOVECOT_DN:="${LDAP_BIND_DN}"}" DOVECOT_LDAP_MAPPING['DOVECOT_DNPASS']="${DOVECOT_DNPASS:="${LDAP_BIND_PW}"}" - DOVECOT_LDAP_MAPPING['DOVECOT_URIS']="${DOVECOT_URIS:="${DOVECOT_HOSTS:="${LDAP_SERVER_HOST}"}"}" - - # Add protocol to DOVECOT_URIS so that we can use dovecot's "uris" option: - # https://doc.dovecot.org/configuration_manual/authentication/ldap/ - if [[ ${DOVECOT_LDAP_MAPPING["DOVECOT_URIS"]} != *'://'* ]]; then - DOVECOT_LDAP_MAPPING['DOVECOT_URIS']="ldap://${DOVECOT_LDAP_MAPPING["DOVECOT_URIS"]}" - fi + DOVECOT_LDAP_MAPPING['DOVECOT_URIS']="${DOVECOT_URIS:="${LDAP_SERVER_HOST}"}" # Default DOVECOT_PASS_FILTER to the same value as DOVECOT_USER_FILTER DOVECOT_LDAP_MAPPING['DOVECOT_PASS_FILTER']="${DOVECOT_PASS_FILTER:="${DOVECOT_USER_FILTER}"}" diff --git a/target/scripts/startup/setup.d/saslauthd.sh b/target/scripts/startup/setup.d/saslauthd.sh index 12f00726..eb33a243 100644 --- a/target/scripts/startup/setup.d/saslauthd.sh +++ b/target/scripts/startup/setup.d/saslauthd.sh @@ -1,29 +1,29 @@ #!/bin/bash - function _setup_saslauthd() { _log 'debug' 'Setting up SASLAUTHD' - if [[ ! -f /etc/saslauthd.conf ]]; then + # NOTE: It's unlikely this file would already exist, + # Unlike Dovecot/Postfix LDAP support, this file has no ENV replacement + # nor does it copy from the DMS config volume to this internal location. + if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]] \ + && [[ ! -f /etc/saslauthd.conf ]]; then _log 'trace' 'Creating /etc/saslauthd.conf' - cat > /etc/saslauthd.conf << EOF -ldap_servers: ${SASLAUTHD_LDAP_SERVER} - -ldap_auth_method: ${SASLAUTHD_LDAP_AUTH_METHOD} -ldap_bind_dn: ${SASLAUTHD_LDAP_BIND_DN} -ldap_bind_pw: ${SASLAUTHD_LDAP_PASSWORD} - -ldap_search_base: ${SASLAUTHD_LDAP_SEARCH_BASE} -ldap_filter: ${SASLAUTHD_LDAP_FILTER} - -ldap_start_tls: ${SASLAUTHD_LDAP_START_TLS} -ldap_tls_check_peer: ${SASLAUTHD_LDAP_TLS_CHECK_PEER} - -${SASLAUTHD_LDAP_TLS_CACERT_FILE} -${SASLAUTHD_LDAP_TLS_CACERT_DIR} -${SASLAUTHD_LDAP_PASSWORD_ATTR} -${SASLAUTHD_LDAP_MECH} + # Create a config based on ENV + sed '/^.*: $/d'> /etc/saslauthd.conf << EOF +ldap_servers: ${SASLAUTHD_LDAP_SERVER:=${LDAP_SERVER_HOST}} +ldap_auth_method: ${SASLAUTHD_LDAP_AUTH_METHOD:=bind} +ldap_bind_dn: ${SASLAUTHD_LDAP_BIND_DN:=${LDAP_BIND_DN}} +ldap_bind_pw: ${SASLAUTHD_LDAP_PASSWORD:=${LDAP_BIND_PW}} +ldap_search_base: ${SASLAUTHD_LDAP_SEARCH_BASE:=${LDAP_SEARCH_BASE}} +ldap_filter: ${SASLAUTHD_LDAP_FILTER:=(&(uniqueIdentifier=%u)(mailEnabled=TRUE))} +ldap_start_tls: ${SASLAUTHD_LDAP_START_TLS:=no} +ldap_tls_check_peer: ${SASLAUTHD_LDAP_TLS_CHECK_PEER:=no} +ldap_tls_cacert_file: ${SASLAUTHD_LDAP_TLS_CACERT_FILE} +ldap_tls_cacert_dir: ${SASLAUTHD_LDAP_TLS_CACERT_DIR} +ldap_password_attr: ${SASLAUTHD_LDAP_PASSWORD_ATTR} +ldap_mech: ${SASLAUTHD_LDAP_MECH} ldap_referrals: yes log_level: 10 EOF @@ -42,4 +42,3 @@ EOF gpasswd -a postfix sasl >/dev/null } - diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index cf099220..b18a61dc 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -17,6 +17,14 @@ function __environment_variables_backwards_compatibility() { _log 'error' "'ENABLE_LDAP=1' has been changed to 'ACCOUNT_PROVISIONER=LDAP' since DMS v13" fi + # Dovecot and SASLAuthd have applied an 'ldap://' fallback for compatibility since v10 (June 2021) + # This was silently applied, but users should be explicit: + if [[ ${LDAP_SERVER_HOST:-'://'} != *'://'* ]] \ + || [[ ${DOVECOT_URIS:-'://'} != *'://'* ]] \ + || [[ ${SASLAUTHD_LDAP_SERVER:-'://'} != *'://'* ]]; then + _log 'error' "The ENV for which LDAP host to connect to must include the URI scheme ('ldap://', 'ldaps://', 'ldapi://')" + fi + # TODO this can be uncommented in a PR handling the HOSTNAME/DOMAINNAME issue # TODO see check_for_changes.sh and dns.sh # if [[ -n ${OVERRIDE_HOSTNAME:-} ]] @@ -141,6 +149,7 @@ function __environment_variables_general_setup() { } # This function handles environment variables related to LDAP. +# NOTE: SASLAuthd and Dovecot LDAP support inherit these common ENV. function _environment_variables_ldap() { _log 'debug' 'Setting LDAP-related environment variables now' @@ -152,55 +161,12 @@ function _environment_variables_ldap() { } # This function handles environment variables related to SASLAUTHD -# and, if activated, variables related to SASLAUTHD and LDAP. +# LDAP specific ENV handled in: `startup/setup.d/saslauthd.sh:_setup_saslauthd()` function _environment_variables_saslauthd() { _log 'debug' 'Setting SASLAUTHD-related environment variables now' + # Only used by the supervisor service command (upstream default: `/etc/default/saslauthd`) VARS[SASLAUTHD_MECHANISMS]="${SASLAUTHD_MECHANISMS:=pam}" - - # SASL ENV for configuring an LDAP specific - # `saslauthd.conf` via `setup-stack.sh:_setup_sasulauthd()` - if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]]; then - _log 'trace' 'Setting SASLSAUTH-LDAP variables nnow' - - VARS[SASLAUTHD_LDAP_AUTH_METHOD]="${SASLAUTHD_LDAP_AUTH_METHOD:=bind}" - VARS[SASLAUTHD_LDAP_BIND_DN]="${SASLAUTHD_LDAP_BIND_DN:=${LDAP_BIND_DN}}" - VARS[SASLAUTHD_LDAP_FILTER]="${SASLAUTHD_LDAP_FILTER:=(&(uniqueIdentifier=%u)(mailEnabled=TRUE))}" - VARS[SASLAUTHD_LDAP_PASSWORD]="${SASLAUTHD_LDAP_PASSWORD:=${LDAP_BIND_PW}}" - VARS[SASLAUTHD_LDAP_SEARCH_BASE]="${SASLAUTHD_LDAP_SEARCH_BASE:=${LDAP_SEARCH_BASE}}" - VARS[SASLAUTHD_LDAP_SERVER]="${SASLAUTHD_LDAP_SERVER:=${LDAP_SERVER_HOST}}" - [[ ${SASLAUTHD_LDAP_SERVER} != *'://'* ]] && SASLAUTHD_LDAP_SERVER="ldap://${SASLAUTHD_LDAP_SERVER}" - VARS[SASLAUTHD_LDAP_START_TLS]="${SASLAUTHD_LDAP_START_TLS:=no}" - VARS[SASLAUTHD_LDAP_TLS_CHECK_PEER]="${SASLAUTHD_LDAP_TLS_CHECK_PEER:=no}" - - if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_FILE} ]]; then - SASLAUTHD_LDAP_TLS_CACERT_FILE='' - else - SASLAUTHD_LDAP_TLS_CACERT_FILE="ldap_tls_cacert_file: ${SASLAUTHD_LDAP_TLS_CACERT_FILE}" - fi - VARS[SASLAUTHD_LDAP_TLS_CACERT_FILE]="${SASLAUTHD_LDAP_TLS_CACERT_FILE}" - - if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_DIR} ]]; then - SASLAUTHD_LDAP_TLS_CACERT_DIR='' - else - SASLAUTHD_LDAP_TLS_CACERT_DIR="ldap_tls_cacert_dir: ${SASLAUTHD_LDAP_TLS_CACERT_DIR}" - fi - VARS[SASLAUTHD_LDAP_TLS_CACERT_DIR]="${SASLAUTHD_LDAP_TLS_CACERT_DIR}" - - if [[ -z ${SASLAUTHD_LDAP_PASSWORD_ATTR} ]]; then - SASLAUTHD_LDAP_PASSWORD_ATTR='' - else - SASLAUTHD_LDAP_PASSWORD_ATTR="ldap_password_attr: ${SASLAUTHD_LDAP_PASSWORD_ATTR}" - fi - VARS[SASLAUTHD_LDAP_PASSWORD_ATTR]="${SASLAUTHD_LDAP_PASSWORD_ATTR}" - - if [[ -z ${SASLAUTHD_LDAP_MECH} ]]; then - SASLAUTHD_LDAP_MECH='' - else - SASLAUTHD_LDAP_MECH="ldap_mech: ${SASLAUTHD_LDAP_MECH}" - fi - VARS[SASLAUTHD_LDAP_MECH]="${SASLAUTHD_LDAP_MECH}" - fi } # This function Writes the contents of the `VARS` map (associative array) diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index c3c60b92..1b73643e 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -92,29 +92,32 @@ function setup_file() { local ENV_LDAP_CONFIG=( --env ACCOUNT_PROVISIONER=LDAP + # Common LDAP ENV: + # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): + --env LDAP_SERVER_HOST="ldap://${FQDN_LDAP}" + --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' + --env LDAP_START_TLS=no + # Credentials needed for read access to LDAP_SEARCH_BASE: + --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' + --env LDAP_BIND_PW='admin' + # Postfix SASL auth provider (SASLAuthd instead of default Dovecot provider): --env ENABLE_SASLAUTHD=1 --env SASLAUTHD_MECHANISMS=ldap --env SASLAUTHD_LDAP_FILTER="${SASLAUTHD_QUERY}" # ENV to configure LDAP configs for Dovecot + Postfix: - # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): # Dovecot: --env DOVECOT_PASS_FILTER="${DOVECOT_QUERY_PASS}" - --env DOVECOT_TLS=no --env DOVECOT_USER_FILTER="${DOVECOT_QUERY_USER}" + --env DOVECOT_TLS=no # Postfix: - --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' - --env LDAP_BIND_PW='admin' --env LDAP_QUERY_FILTER_ALIAS="${QUERY_ALIAS}" --env LDAP_QUERY_FILTER_DOMAIN="${QUERY_DOMAIN}" --env LDAP_QUERY_FILTER_GROUP="${QUERY_GROUP}" --env LDAP_QUERY_FILTER_SENDERS="${QUERY_SENDERS}" --env LDAP_QUERY_FILTER_USER="${QUERY_USER}" - --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' - --env LDAP_SERVER_HOST="${FQDN_LDAP}" - --env LDAP_START_TLS=no ) # Extra ENV needed to support specific test-cases: @@ -221,7 +224,7 @@ function teardown() { @test "postfix: ldap config overwrites success" { local LDAP_SETTINGS_POSTFIX=( - "server_host = ${FQDN_LDAP}" + "server_host = ldap://${FQDN_LDAP}" 'start_tls = no' 'search_base = ou=users,dc=example,dc=test' 'bind_dn = cn=admin,dc=example,dc=test' From c5420530b7e21d4c0f77dbb910f9afd051f031f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:07:38 +0200 Subject: [PATCH 022/125] chore(deps): Bump actions/checkout from 3 to 4 (#3525) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/contributors.yml | 2 +- .github/workflows/docs-preview-prepare.yml | 2 +- .github/workflows/docs-production-deploy.yml | 8 ++++---- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- .github/workflows/linting.yml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 6e51495a..00b581c4 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: 'Checkout New Branch and Push It' run: | diff --git a/.github/workflows/docs-preview-prepare.yml b/.github/workflows/docs-preview-prepare.yml index a509ac77..befd4008 100644 --- a/.github/workflows/docs-preview-prepare.yml +++ b/.github/workflows/docs-preview-prepare.yml @@ -29,7 +29,7 @@ jobs: NETLIFY_SITE_PREFIX: pullrequest-${{ github.event.pull_request.number }} NETLIFY_SITE_NAME: dms-doc-previews steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: 'Build with mkdocs-material via Docker' working-directory: docs diff --git a/.github/workflows/docs-production-deploy.yml b/.github/workflows/docs-production-deploy.yml index 8c8faa23..cb8bdbed 100644 --- a/.github/workflows/docs-production-deploy.yml +++ b/.github/workflows/docs-production-deploy.yml @@ -28,7 +28,7 @@ jobs: name: 'Deploy Docs' runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: 'Check if deploy is for a `v.` tag version instead of `edge`' if: startsWith(github.ref, 'refs/tags/') @@ -79,10 +79,10 @@ jobs: needs: deploy steps: - name: 'Checkout the tagged commit (shallow clone)' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: 'Checkout the docs deployment branch to a subdirectory' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: gh-pages path: gh-pages @@ -115,7 +115,7 @@ jobs: needs: add-version-to-docs steps: - name: 'Checkout the docs deployment branch' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: gh-pages diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index a86b0ed6..d1e837a9 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -28,7 +28,7 @@ jobs: build-cache-key: ${{ steps.derive-image-cache-key.outputs.digest }} steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 598c12a3..b3a64ef0 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 5b460cce..bc2adb6c 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Required to retrieve bats (core + extras): submodules: recursive diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 69e8cea3..b5ed5e9d 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Get the cached build layers from the build job: # This should always be a cache-hit, thus `restore-keys` fallback is not used. diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index e9339a67..bdde2d0f 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Hadolint run: make hadolint From 20241691b832a1d4115f4fd19cdd05751e558c8f Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 7 Sep 2023 18:35:08 -0400 Subject: [PATCH 023/125] docs: Fix IPv6 example for Compose (#3531) The subnet must be specified as part of `ipam.configs`. This was unfortunately slightly incorrect due to a mistake in the official Docker docs being propagated, which has since been fixed upstream. Refer to the official Compose Spec for more details: * https://docs.docker.com/compose/compose-file/06-networks/#ipam --- docs/content/config/advanced/ipv6.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 4a45d168..69ed069b 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -92,7 +92,9 @@ Next, configure a network with an IPv6 subnet for your container with any of the networks: dms-ipv6: enable_ipv6: true - subnet: fd00:cafe:face:feed::/64 + ipam: + config: + - subnet: fd00:cafe:face:feed::/64 ``` ??? tip "Override the implicit `default` network" From ad8b618b461236fe504cdbb0d0ddf0bff0283840 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 9 Sep 2023 09:35:57 +1200 Subject: [PATCH 024/125] fix: Ensure files are committed with `eol=lf` via `.gitattributes` (#3527) * chore: Use `.yml` extension Both of these files support the `.yml` extension. Normalize on that. * fix: Add `.gitattributes` to ensure `LF` line-endings are committed Avoids accidentally committing files with `CRLF` when they're created on Windows. Or worse, if some editors don't detect `LF` and would introduce mixed line-endings with `CRLF`. Shouldn't be a problem in practice as we already have a linting check to catch this via CI during PRs. This file is complimentary, in that it should automate that concern away. --- .gitattributes | 160 ++++++++++++++++++ ...ature_request.yaml => feature_request.yml} | 0 .../linting/{.hadolint.yaml => .hadolint.yml} | 0 test/linting/lint.sh | 2 +- 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 .gitattributes rename .github/ISSUE_TEMPLATE/{feature_request.yaml => feature_request.yml} (100%) rename test/linting/{.hadolint.yaml => .hadolint.yml} (100%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d3dba13d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,160 @@ +# Normalize line endings of all non-binary files to LF upon check-in (`git add` / `git commit`): +* text=auto + +################################################# +### General ################################### +################################################# + +## GENERIC +### CI + docs/mkdocs.yml +*.yml text +### Documentation (Project, Tests, Docs site) +*.md text +### TLS certs (test/test-files/) + DHE params (target/shared/) +*.pem text +*.pem.sha512sum text + +################################################# +### Project ################################### +################################################# + +## BUILD: +.dockerignore text +Dockerfile text +Makefile +VERSION + +## EXAMPLE (RUNTIME): +*.env text +*.yaml text + +## PROJECT +.all-contributorsrc text export-ignore +.editorconfig text export-ignore +.gitattributes text export-ignore +.gitignore text export-ignore +.gitkeep text export-ignore +.gitmodules text export-ignore +LICENSE text + +## SOURCE CODE +*.sh text eol=lf +### acme.json extractor (target/bin/) +*.py text eol=lf +### Only contain scripts (glob for extensionless) +target/bin/** text eol=lf + +################################################# +### Config #################################### +################################################# + +## CONFIG +### Contains all text files (glob for extensionless) +target/amavis/** text +target/fetchmail/** text +target/getmail/** text +target/opendkim/** text +target/opendmarc/** text +target/postgrey/** text +target/postsrsd/** text +### Generic target/ + test/config/ +*.cf text +*.conf text +### Dovecot +*.ext text +*.sieve text +### Dovecot + Rspamd +*.inc text +### Fail2Ban + Postgrey (test/config/) +*.local text +### Postfix +*.pcre text + +################################################# +### Tests ##################################### +################################################# + +## BATS +*.bash text +*.bats text + +## CONFIG (test/config/) +### OpenLDAP image +*.ldif text +### OpenDKIM +*.private text +KeyTable text +SigningTable text +TrustedHosts text +### Postgrey +whitelist_recipients text + +## MISC +### test/config/ + test/test-files/ +*.txt text +### test/linting/ (.ecrc.json) + test/test-files/ (*.acme.json): +*.json text + +################################################# +### Documentation Website ##################### +################################################# + +## DOCUMENTATION +### docs/content/assets/ +*.css text +*.png binary +*.svg text -diff +*.woff binary +### docs/overrides/ +*.html text +*.ico binary +*.webp binary + +################################################# +### Info # ##################################### +################################################# + +### WHAT IS THIS FILE? +# `.gitattributes` - Pattern-based overrides (Project specific) +# Documentation: https://git-scm.com/docs/gitattributes +# +# Travels with the project and can override the defaults from `.gitconfig`. +# This helps to enforce consistent line endings (CRLF / LF) where needed via +# patterns (_when the git client supports `.gitattributes`_). + +# `.gitconfig` - Global Git defaults (Dev environment) +# Documentation: https://git-scm.com/docs/git-config +# +# Git settings `core.autocrlf` and `core.eol` can vary across dev environments. +# Those defaults can introduce subtle bugs due to incompatible line endings. + + +### WHY SHOULD I CARE? +# The desired result is to ensure the repo contains normalized LF line endings, +# notably avoiding unhelpful noise in diffs or issues incurred from mixed line +# endings. Storing as LF ensures no surprises for line endings during checkout. +# Additionally for checkout to the local working directory, line endings can be +# forced to CRLF or LF per file where appropriate, which ensures the files have +# compatible line endings where software expects a specific kind. +# +# Examples: +# Diffs with nothing visual changed. Line endings appear invisible. +# Tests that compare text from two sources where only line endings differ fail. +# /bin/sh with a shebang fails to run a binary at the given path due to a CRLF. + + +### ATTRIBUTES +# `text` normalizes the line endings of a file to LF upon commit (CRLF -> LF). +# `text=auto` sets `text` if Git doesn't consider the file as binary data. + +# `eol` sets an explicit line ending to write files to the working directory. +# `core.eol` is used for any files not explicitly set with an `eol` attr value. +# `core.eol` uses the native line endings for your platform by default. +# `core.autocrlf` (if set to `true` or `input`) overrides the `core.eol` value. + +# `binary` is an alias for `-text -diff`. The file won't be normalized (-text). +# `-diff` indicates to avoid creating a diff. Useful when diffs are unlikely +# to be meaningful, such as generated content (SVG, Source Maps, Lockfiles). + +# `export-ignore` excludes matched files and directories during `git archive`, +# which services like Github use to create releases with archived source files. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.yaml rename to .github/ISSUE_TEMPLATE/feature_request.yml diff --git a/test/linting/.hadolint.yaml b/test/linting/.hadolint.yml similarity index 100% rename from test/linting/.hadolint.yaml rename to test/linting/.hadolint.yml diff --git a/test/linting/lint.sh b/test/linting/lint.sh index c5718447..88103ddd 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -35,7 +35,7 @@ function _hadolint() { --volume "${REPOSITORY_ROOT}:/ci:ro" \ --workdir "/ci" \ --name dms-test_hadolint \ - "hadolint/hadolint:v${HADOLINT_VERSION}-alpine" hadolint --config "/ci/test/linting/.hadolint.yaml" Dockerfile + "hadolint/hadolint:v${HADOLINT_VERSION}-alpine" hadolint --config "/ci/test/linting/.hadolint.yml" Dockerfile then _log 'info' 'Hadolint succeeded' else From fc3229f8d205f5d5a7e74a8fbe95f4005940db38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:58:36 +0200 Subject: [PATCH 025/125] chore(deps): Bump docker/build-push-action from 4.1.1 to 4.2.1 (#3533) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index d1e837a9..23c710a6 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index b3a64ef0..34985553 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index bc2adb6c..48d2cbac 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b5ed5e9d..74667431 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . tags: mailserver-testing:ci From 8329fa19cc8b38a3776b1f4a5f68f9bf7fb6f5c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:50:55 +1200 Subject: [PATCH 026/125] chore(deps): Bump myrotvorets/set-commit-status-action (#3534) Bumps [myrotvorets/set-commit-status-action](https://github.com/myrotvorets/set-commit-status-action) from 1.1.7 to 2.0.0. - [Release notes](https://github.com/myrotvorets/set-commit-status-action/releases) - [Commits](https://github.com/myrotvorets/set-commit-status-action/compare/v1.1.7...v2.0.0) --- updated-dependencies: - dependency-name: myrotvorets/set-commit-status-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/contributors.yml | 2 +- .github/workflows/docs-preview-deploy.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 00b581c4..9028a60b 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -73,7 +73,7 @@ jobs: # workflow, which is required due to branch protection, is not important for this type # of PR, so we skip it and pretend it was successful. - name: 'Set Status for Linting Actions to Success (Skipped)' - uses: myrotvorets/set-commit-status-action@v1.1.7 + uses: myrotvorets/set-commit-status-action@v2.0.0 continue-on-error: true with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 278cf60c..cac2ac64 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -46,7 +46,7 @@ jobs: # but presently does not work correctly via split workflow. It is useful in a split workflow as the 1st stage # no longer indicates if the entire workflow/deployment was successful. - name: 'Commit Status: Set Workflow Status as Pending' - uses: myrotvorets/set-commit-status-action@v1.1.7 + uses: myrotvorets/set-commit-status-action@v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} status: pending @@ -106,7 +106,7 @@ jobs: Built with commit: ${{ env.PR_HEADSHA }} - name: 'Commit Status: Update deployment status' - uses: myrotvorets/set-commit-status-action@v1.1.7 + uses: myrotvorets/set-commit-status-action@v2.0.0 # Always run this step regardless of job failing early: if: ${{ always() }} env: From 86edaf9a8a3ad8d5b2bba3d9a6ad94b675ca7d1a Mon Sep 17 00:00:00 2001 From: Lucas Bartholemy Date: Wed, 13 Sep 2023 10:42:52 +0200 Subject: [PATCH 027/125] fix: DKIM key generation broken when Rspamd & OpenDKIM are enabled (#3535) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/bin/open-dkim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 0036ff8d..11b98975 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -4,8 +4,12 @@ source /usr/local/bin/helpers/index.sh if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 ]]; then - /usr/local/bin/rspamd-dkim "${@}" - exit + if [[ $(_get_dms_env_value 'ENABLE_OPENDKIM') -eq 1 ]]; then + log 'error' "You enabled Rspamd and OpenDKIM - OpenDKIM will be implicitly used for DKIM keys" + else + /usr/local/bin/rspamd-dkim "${@}" + exit + fi fi KEYSIZE=2048 From 62f4544dd2ebe1ff68f3e59bb0e9bca7a6905ed4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:53:39 +1200 Subject: [PATCH 028/125] chore(deps): Bump docker/setup-buildx-action from 2.10.0 to 3.0.0 (#3540) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2.10.0 to 3.0.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2.10.0...v3.0.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 23c710a6..bc2f62ad 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 34985553..218c6779 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 48d2cbac..19994a40 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 74667431..d2a2709e 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From c425cdddc56880637c8b1d4d193d75fdf8556d17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 02:55:20 +0000 Subject: [PATCH 029/125] chore(deps): Bump docker/build-push-action from 4.2.1 to 5.0.0 (#3541) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4.2.1 to 5.0.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4.2.1...v5.0.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index bc2f62ad..dc46b681 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 218c6779..b1830c66 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 19994a40..3c6e1288 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index d2a2709e..ff8c8c1a 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . tags: mailserver-testing:ci From af65189a82672f986a026c92039dbb68a0922273 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 02:57:36 +0000 Subject: [PATCH 030/125] chore(deps): Bump docker/setup-qemu-action from 2.2.0 to 3.0.0 (#3542) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2.2.0 to 3.0.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2.2.0...v3.0.0) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index dc46b681..c7dc6e3f 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -74,7 +74,7 @@ jobs: cache-buildx- - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v2.2.0 + uses: docker/setup-qemu-action@v3.0.0 with: platforms: arm64 diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index b1830c66..4e32d190 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -35,7 +35,7 @@ jobs: type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v2.2.0 + uses: docker/setup-qemu-action@v3.0.0 with: platforms: arm64 From 285266a6aab88a210efbc02e2b67b068ee7d75b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 02:59:39 +0000 Subject: [PATCH 031/125] chore(deps): Bump docker/metadata-action from 4.6.0 to 5.0.0 (#3544) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 4.6.0 to 5.0.0. - [Release notes](https://github.com/docker/metadata-action/releases) - [Upgrade guide](https://github.com/docker/metadata-action/blob/master/UPGRADE.md) - [Commits](https://github.com/docker/metadata-action/compare/v4.6.0...v5.0.0) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 4e32d190..1b84a6da 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v4.6.0 + uses: docker/metadata-action@v5.0.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 539a7bc3bb92d237f2f3b190b6a33bacfa71e2bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 03:01:59 +0000 Subject: [PATCH 032/125] chore(deps): Bump docker/login-action from 2 to 3 (#3543) Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 1b84a6da..5d68cd73 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -54,13 +54,13 @@ jobs: cache-buildx- - name: 'Login to DockerHub' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: 'Login to GitHub Container Registry' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} From 8c0cfa0836bcf20e78a2d5d1ae02270004ee0aed Mon Sep 17 00:00:00 2001 From: jpduyx Date: Tue, 19 Sep 2023 07:21:33 +0300 Subject: [PATCH 033/125] docs: Revise `update-and-cleanup.md` (#3539) * Update update-and-cleanup.md spotify dockergc is UNMAINTAINED, they advice to consider using the `docker system prune` command instead. "This repository has been archived by the owner on Feb 2, 2021. It is now read-only." https://github.com/spotify/docker-gc * Revise `update-and-cleanup.md` Merges the image update + cleanup sections. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .../maintenance/update-and-cleanup.md | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/content/config/advanced/maintenance/update-and-cleanup.md b/docs/content/config/advanced/maintenance/update-and-cleanup.md index 3e4301f9..b972c4a5 100644 --- a/docs/content/config/advanced/maintenance/update-and-cleanup.md +++ b/docs/content/config/advanced/maintenance/update-and-cleanup.md @@ -2,40 +2,41 @@ title: 'Maintenance | Update and Cleanup' --- -## Automatic Update +[`containrrr/watchtower`][watchtower-dockerhub] is a service that monitors Docker images for updates, automatically applying them to running containers. -Docker images are handy but it can become a hassle to keep them updated. Also when a repository is automated you want to get these images when they get out. +!!! example "Automatic image updates + cleanup" -One could setup a complex action/hook-based workflow using probes, but there is a nice, easy to use docker image that solves this issue and could prove useful: [`watchtower`](https://hub.docker.com/r/containrrr/watchtower). + ```yaml title="compose.yaml" + services: + watchtower: + image: containrrr/watchtower:latest + # Automatic cleanup (removes older image pulls from wasting disk space): + environment: + - WATCHTOWER_CLEANUP=true + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ``` -A Docker Compose example: +!!! tip "Updating only specific containers" -```yaml -services: - watchtower: - restart: always - image: containrrr/watchtower:latest - volumes: - - /var/run/docker.sock:/var/run/docker.sock -``` + The default `watchtower` service will check every 24 hours for any new image updates to pull, **not only the images** defined within your `compose.yaml`. -For more details, see the [manual](https://containrrr.github.io/watchtower/) + The images to update can be restricted with a custom command that provides a list of containers names and other config options. Configuration is detailed in the [`watchtower` docs][watchtower-docs]. -## Automatic Cleanup +!!! info "Manual cleanup" -When you are pulling new images in automatically, it would be nice to have them cleaned up as well. There is also a docker image for this: [`spotify/docker-gc`](https://hub.docker.com/r/spotify/docker-gc/). + `watchtower` also supports running on-demand with `docker run` or `compose.yaml` via the `--run-once` option. + + You can also directly invoke cleanup of Docker storage with: -A Docker Compose example: + - [`docker image prune --all`][docker-docs-prune-image] + - [`docker system prune --all`][docker-docs-prune-system] (_also removes unused containers, networks, build cache_). + - Avoid the `--all` option to only remove ["dangling" content][docker-prune-dangling] (_eg: Orphaned images_). -```yaml -services: - docker-gc: - restart: always - image: spotify/docker-gc:latest - volumes: - - /var/run/docker.sock:/var/run/docker.sock -``` +[watchtower-dockerhub]: https://hub.docker.com/r/containrrr/watchtower +[watchtower-cleanup]: https://containrrr.github.io/watchtower/arguments/#cleanup +[watchtower-docs]: https://containrrr.dev/watchtower/ -For more details, see the [manual](https://github.com/spotify/docker-gc/blob/master/README.md) - -Or you can just use the [`--cleanup`](https://containrrr.github.io/watchtower/arguments/#cleanup) option provided by `containrrr/watchtower`. +[docker-docs-prune-image]: https://docs.docker.com/engine/reference/commandline/image_prune/ +[docker-docs-prune-system]: https://docs.docker.com/engine/reference/commandline/system_prune/ +[docker-prune-dangling]: https://stackoverflow.com/questions/45142528/what-is-a-dangling-image-and-what-is-an-unused-image/60756668#60756668 From a9d6e329cd20b8e3cc68f31a12726dc56b1793e8 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 29 Sep 2023 01:37:15 +1300 Subject: [PATCH 034/125] tests(fix): `process_check_restart.bats` - Run `pgrep` within the actual container (#3553) This was missed during original review. On a linux host, processes running within a container have been visible via commands like `pgrep`. This is does not appear to be the case with WSL2 + Docker Desktop (Windows), resulting in test failure. The command should have been run from within the container regardless. --- .../set3/container_configuration/process_check_restart.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 2c932bd2..b559d21d 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -120,9 +120,9 @@ ENV_PROCESS_LIST=( # By this point the fetchmail processes have been verified to exist and restart, # For FETCHMAIL_PARALLEL=1 coverage, match full commandline for COUNTER values: - pgrep --full 'fetchmail-1.rc' + _run_in_container pgrep --full 'fetchmail-1.rc' assert_success - pgrep --full 'fetchmail-2.rc' + _run_in_container pgrep --full 'fetchmail-2.rc' assert_success _should_stop_cleanly From 89cb6d85b90f6070d53c15e455ebee916c899c7a Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 29 Sep 2023 10:17:57 +1300 Subject: [PATCH 035/125] tests(fix): `lmtp_ip.bats` improve partial failure output (#3552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of exit status of `124` (_signifies timeout_), it should fail with `1` (failure) like the others. Handled via using `_run_in_container_bash()` (_`timeout` failure `124` does not propagate and is treated as `1` instead_). In this case we are waiting on the status of the mail being sent, the pattern provided to `grep` is too specific and results in a timeout. Instead since we only expect the one log entry, match any status and assert the expected pattern afterwards. This provides a more helpful failure output that informs us that mail was at least processed by Postfix, but the sent status is not what we expected. ### Before ``` ✗ [ENV] (POSTFIX_DAGENT) delivers mail to existing account [60327] (from function `assert_success' in file test/test_helper/bats-assert/src/assert_success.bash, line 42, in test file test/tests/parallel/set3/mta/lmtp_ip.bats, line 47) `assert_success' failed -- command failed -- status : 124 output : -- ``` ### After ``` ✗ [ENV] (POSTFIX_DAGENT) delivers mail to existing account [1425] (from function `assert_output' in file test/test_helper/bats-assert/src/assert_output.bash, line 178, in test file test/tests/parallel/set3/mta/lmtp_ip.bats, line 48) `assert_output --regexp "${MATCH_LOG_LINE}=sent .* Saved)"' failed -- regular expression does not match output -- regexp : postfix/lmtp.* status=sent .* Saved) output : Sep 28 04:12:52 mail postfix/lmtp[721]: 23701B575: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.08, delays=0.07/0/0.01/0, dsn=4.2.0, status=deferred (host 127.0.0.1[127.0.0.1] said: 451 4.2.0 Internal error occurred. Refer to server log for more information. [2023-09-28 04:12:52] (in reply to end of DATA command)) -- ``` The expected pattern is logged as `assert_success` confirms a valid match for the log line of interest was found, and we have the mismatched value to debug the failure against. --- test/tests/parallel/set3/mta/lmtp_ip.bats | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/tests/parallel/set3/mta/lmtp_ip.bats b/test/tests/parallel/set3/mta/lmtp_ip.bats index 861f7f60..8d35c062 100644 --- a/test/tests/parallel/set3/mta/lmtp_ip.bats +++ b/test/tests/parallel/set3/mta/lmtp_ip.bats @@ -42,7 +42,9 @@ function teardown_file() { _default_teardown ; } # Verify delivery was successful, log line should look similar to: # postfix/lmtp[1274]: 0EA424ABE7D9: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.13, delays=0.07/0.01/0.01/0.05, dsn=2.0.0, status=sent (250 2.0.0 ixPpB+Zvv2P7BAAAUi6ngw Saved) - local MATCH_LOG_LINE='postfix/lmtp.* status=sent .* Saved)' - run timeout 60 docker exec "${CONTAINER_NAME}" bash -c "tail -F /var/log/mail/mail.log | grep --max-count 1 '${MATCH_LOG_LINE}'" + local MATCH_LOG_LINE='postfix/lmtp.* status' + _run_in_container_bash "timeout 60 tail -F /var/log/mail/mail.log | grep --max-count 1 '${MATCH_LOG_LINE}'" assert_success + # Assertion of full pattern here (instead of via grep) is a bit more helpful for debugging partial failures: + assert_output --regexp "${MATCH_LOG_LINE}=sent .* Saved)" } From bd96c1161e9f9b32bb0b3aaec4b930980a56945f Mon Sep 17 00:00:00 2001 From: Vincent Ducamps Date: Sat, 30 Sep 2023 13:20:03 +0200 Subject: [PATCH 036/125] feat: Allow changing the Dovecot vmail UID/GID via ENV (#3550) Some deployment scenarios are not compatible with `5000:5000` static vmail user with `/var/mail`. This feature allows adjusting the defaults to a UID / GID that is compatible. Signed-off-by: vincent Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/environment.md | 12 ++++ mailserver.env | 6 ++ target/scripts/helpers/accounts.sh | 4 +- target/scripts/helpers/utils.sh | 4 +- target/scripts/start-mailserver.sh | 1 + target/scripts/startup/setup.d/vmail-id.sh | 12 ++++ target/scripts/startup/variables-stack.sh | 2 + test/tests/serial/vmail-id.bats | 75 ++++++++++++++++++++++ 8 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 target/scripts/startup/setup.d/vmail-id.sh create mode 100644 test/tests/serial/vmail-id.bats diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index e5aba889..678ec965 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -33,6 +33,18 @@ Here you can adjust the [log-level for Supervisor](http://supervisord.org/loggin The log-level will show everything in its class and above. +##### DMS_VMAIL_UID + +Default: 5000 + +The User ID assigned to the static vmail user for `/var/mail` (_Mail storage managed by Dovecot_). + +##### DMS_VMAIL_GID + +Default: 5000 + +The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage managed by Dovecot_). + ##### ONE_DIR - 0 => state in default directories. diff --git a/mailserver.env b/mailserver.env index 632f6edf..1a57ceca 100644 --- a/mailserver.env +++ b/mailserver.env @@ -34,6 +34,12 @@ SUPERVISOR_LOGLEVEL= # 1 => consolidate all states into a single directory (`/var/mail-state`) to allow persistence using docker volumes ONE_DIR=1 +# Support for deployment where these defaults are not compatible (eg: some NAS appliances): +# /var/mail vmail User ID (default: 5000) +DMS_VMAIL_UID= +# /var/mail vmail Group ID (default: 5000) +DMS_VMAIL_GID= + # **empty** => use FILE # LDAP => use LDAP authentication # OIDC => use OIDC authentication (not yet implemented) diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 82b626c1..7499a2ec 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -70,7 +70,7 @@ function _create_accounts() { # Dovecot's userdb has the following format # user:password:uid:gid:(gecos):home:(shell):extra_fields - DOVECOT_USERDB_LINE="${LOGIN}:${PASS}:5000:5000::/var/mail/${DOMAIN}/${USER}/home::${USER_ATTRIBUTES}" + DOVECOT_USERDB_LINE="${LOGIN}:${PASS}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${DOMAIN}/${USER}/home::${USER_ATTRIBUTES}" if grep -qF "${DOVECOT_USERDB_LINE}" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Login '${LOGIN}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else @@ -141,7 +141,7 @@ function _create_dovecot_alias_dummy_accounts() { fi fi - DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:5000:5000::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}::${REAL_ACC[2]:-}" + DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}::${REAL_ACC[2]:-}" if grep -qi "^${ALIAS}:" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Alias '${ALIAS}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index c74fd7e8..e44f0aff 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -39,9 +39,9 @@ function _get_dms_env_value() { # /var/mail folders (used during startup and change detection handling). function _chown_var_mail_if_necessary() { # fix permissions, but skip this if 3 levels deep the user id is already set - if find /var/mail -maxdepth 3 -a \( \! -user 5000 -o \! -group 5000 \) | read -r; then + if find /var/mail -maxdepth 3 -a \( \! -user "${DMS_VMAIL_UID}" -o \! -group "${DMS_VMAIL_GID}" \) | read -r; then _log 'trace' 'Fixing /var/mail permissions' - chown -R 5000:5000 /var/mail || return 1 + chown -R "${DMS_VMAIL_UID}:${DMS_VMAIL_GID}" /var/mail || return 1 fi } diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index e41ae71e..f0f385f3 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -39,6 +39,7 @@ function _register_functions() { # ? >> Setup + _register_setup_function '_setup_vmail_id' _register_setup_function '_setup_logs_general' _register_setup_function '_setup_timezone' diff --git a/target/scripts/startup/setup.d/vmail-id.sh b/target/scripts/startup/setup.d/vmail-id.sh new file mode 100644 index 00000000..2b692169 --- /dev/null +++ b/target/scripts/startup/setup.d/vmail-id.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +function _setup_vmail_id() { + if [[ "${DMS_VMAIL_UID}" != "5000" ]]; then + _log 'debug' "Setting 'docker' UID to ${DMS_VMAIL_UID}" + usermod --uid "${DMS_VMAIL_UID}" docker + fi + if [[ "${DMS_VMAIL_GID}" != "5000" ]]; then + _log 'debug' "Setting 'docker' GID to ${DMS_VMAIL_GID}" + groupmod --gid "${DMS_VMAIL_GID}" docker + fi +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index b18a61dc..3b575f50 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -46,6 +46,8 @@ function __environment_variables_general_setup() { VARS[POSTMASTER_ADDRESS]="${POSTMASTER_ADDRESS:=postmaster@${DOMAINNAME}}" VARS[REPORT_RECIPIENT]="${REPORT_RECIPIENT:=${POSTMASTER_ADDRESS}}" VARS[REPORT_SENDER]="${REPORT_SENDER:=mailserver-report@${HOSTNAME}}" + VARS[DMS_VMAIL_UID]="${DMS_VMAIL_UID:=5000}" + VARS[DMS_VMAIL_GID]="${DMS_VMAIL_GID:=5000}" _log 'trace' 'Setting anti-spam & anti-virus environment variables' diff --git a/test/tests/serial/vmail-id.bats b/test/tests/serial/vmail-id.bats new file mode 100644 index 00000000..b44670b2 --- /dev/null +++ b/test/tests/serial/vmail-id.bats @@ -0,0 +1,75 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +BATS_TEST_NAME_PREFIX='[ENV] (DMS_VMAIL_UID + DMS_VMAIL_GID) ' +CONTAINER_NAME='dms-test_env-change-vmail-id' + +function setup_file() { + _init_with_defaults + + local CUSTOM_SETUP_ARGUMENTS=( + --env PERMIT_DOCKER=container + --env DMS_VMAIL_UID=9042 + --env DMS_VMAIL_GID=9042 + ) + + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_smtp_port_in_container +} + +function teardown_file() { _default_teardown ; } + +@test 'should successfully deliver mail' { + _send_email 'email-templates/existing-user1' + _wait_for_empty_mail_queue_in_container + + # Should be successfully sent (received) by Postfix: + _run_in_container grep 'to=' /var/log/mail/mail.log + assert_success + assert_output --partial 'status=sent' + _should_output_number_of_lines 1 + + # Verify successful delivery via Dovecot to `/var/mail` account by searching for the subject: + _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep -R \ + 'Subject: Test Message existing-user1.txt' \ + '/var/mail/localhost.localdomain/user1/new/' + assert_success + _should_output_number_of_lines 1 +} + +# TODO: Migrate to test/helper/common.bash +# This test case is shared with tests.bats, but provides context on errors + some minor edits +# TODO: Could improve in future with keywords from https://github.com/docker-mailserver/docker-mailserver/pull/3550#issuecomment-1738509088 +# Potentially via a helper that allows an optional fixed number of errors to be present if they were intentional +@test '/var/log/mail/mail.log is error free' { + # Postfix: https://serverfault.com/questions/934703/postfix-451-4-3-0-temporary-lookup-failure + _run_in_container grep 'non-null host address bits in' /var/log/mail/mail.log + assert_failure + + # Postfix delivery failure: https://github.com/docker-mailserver/docker-mailserver/issues/230 + _run_in_container grep 'mail system configuration error' /var/log/mail/mail.log + assert_failure + + # Unknown error source: https://github.com/docker-mailserver/docker-mailserver/pull/85 + _run_in_container grep -i ': error:' /var/log/mail/mail.log + assert_failure + + # Unknown error source: https://github.com/docker-mailserver/docker-mailserver/pull/320 + _run_in_container grep -i 'not writable' /var/log/mail/mail.log + assert_failure + _run_in_container grep -i 'permission denied' /var/log/mail/mail.log + assert_failure + + # Amavis: https://forum.howtoforge.com/threads/postfix-smtp-error-caused-by-clamav-cant-connect-to-a-unix-socket-var-run-clamav-clamd-ctl.81002/ + _run_in_container grep -i '(!)connect' /var/log/mail/mail.log + assert_failure + + # Postfix: https://github.com/docker-mailserver/docker-mailserver/pull/2597 + _run_in_container grep -i 'using backwards-compatible default setting' /var/log/mail/mail.log + assert_failure + + # Postgrey: https://github.com/docker-mailserver/docker-mailserver/pull/612#discussion_r117635774 + _run_in_container grep -i 'connect to 127.0.0.1:10023: Connection refused' /var/log/mail/mail.log + assert_failure +} + From aae42fae9bd0190c7b4326e3855d8492ecd61dfa Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 4 Oct 2023 23:53:32 +1300 Subject: [PATCH 037/125] ci(fix): Normalize for `.gitattributes` + improve `eclint` coverage (#3566) --- .editorconfig | 15 +- .github/pull_request_template.md | 6 +- CHANGELOG.md | 7 +- Dockerfile | 2 - .../advanced/dovecot-master-accounts.md | 2 +- docs/content/config/advanced/ipv6.md | 2 +- docs/content/config/advanced/mail-sieve.md | 12 +- docs/content/config/security/ssl.md | 35 ++-- docs/content/contributing/tests.md | 40 ++--- docs/content/faq.md | 5 +- docs/content/introduction.md | 36 ++-- target/amavis/conf.d/60-dms_default_config | 2 +- target/bin/open-dkim | 4 +- target/postgrey/postgrey.init | 154 ------------------ .../keys/localhost.localdomain/mail.txt | 2 +- .../keys/otherdomain.tld/mail.txt | 2 +- test/config/fetchmail/fetchmail.cf | 16 +- .../ldap/openldap/schemas/postfix-book.ldif | 28 ++-- test/config/override-configs/postfix-main.cf | 2 +- .../config/override-configs/postfix-master.cf | 2 +- test/config/postfix-regexp.cf | 3 +- test/config/postfix-virtual.cf | 2 +- test/config/templates/dovecot-masters.cf | 2 +- test/config/templates/postfix-accounts.cf | 8 +- test/linting/.ecrc.json | 26 +-- .../parallel/set1/spam_virus/amavis.bats | 2 +- test/tests/serial/mail_with_ldap.bats | 2 +- 27 files changed, 120 insertions(+), 299 deletions(-) delete mode 100644 target/postgrey/postgrey.init diff --git a/.editorconfig b/.editorconfig index fd68e2f2..f5f1e317 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,6 +8,7 @@ root = true [*] charset = utf-8 end_of_line = lf +indent_size = 2 indent_style = space insert_final_newline = true trim_trailing_whitespace = true @@ -16,21 +17,9 @@ trim_trailing_whitespace = true # --- Specific ---------------------------------- # ----------------------------------------------- -[*.{yaml,yml,sh,bats}] -indent_size = 2 - -[Makefile] +[{Makefile,.gitmodules}] indent_style = tab indent_size = 4 [*.md] trim_trailing_whitespace = false - -# ----------------------------------------------- -# --- Git Submodules ---------------------------- -# ----------------------------------------------- - -[{test/bats/**,test/test_helper/**}] -indent_style = none -indent_size = none -end_of_line = none diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index bcdf248a..26dc801e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,9 @@ # Description - + Fixes # diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b22fec5..8ce7b6fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -311,8 +311,8 @@ In this release the relay-host support saw [significant internal refactoring](ht 1. **Many** minor improvements were made (cleanup & refactoring). Please refer to the section below to get an overview over all improvements. Moreover, there was a lot of cleanup in the scripts and in the tests. The documentation was adjusted accordingly. 2. New environment variables were added: - 1. [`CLAMAV_MESSAGE_SIZE_LIMIT`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#clamav_message_size_limit) - 2. [`TZ`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#tz) + 1. [`CLAMAV_MESSAGE_SIZE_LIMIT`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#clamav_message_size_limit) + 2. [`TZ`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#tz) 3. SpamAssassin KAM was added with [`ENABLE_SPAMASSASSIN_KAM`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#enable_spamassassin_kam). 4. The `fail2ban` command was reworked and can now ban IP addresses as well. 5. There were a few small fixes, especially when it comes to bugs in scripts and service restart loops (no functionality changes, only fixes of existing functionality). When building an image from the Dockerfile - Installation of Postfix on modern Linux distributions should now always succeed. @@ -368,8 +368,7 @@ In this release the relay-host support saw [significant internal refactoring](ht ### Critical Changes -1. This release fixes a critical issue for LDAP users, installing a needed package on Debian 11 - on build-time. Moreover, a race-condition was eliminated ([#2341](https://github.com/docker-mailserver/docker-mailserver/pull/2341)). +1. This release fixes a critical issue for LDAP users, installing a needed package on Debian 11 on build-time. Moreover, a race-condition was eliminated ([#2341](https://github.com/docker-mailserver/docker-mailserver/pull/2341)). 2. A resource leak in `check-for-changes.sh` was fixed ([#2401](https://github.com/docker-mailserver/docker-mailserver/pull/2401)) ### Other Minor Changes diff --git a/Dockerfile b/Dockerfile index b5c5ba39..5e12689d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,9 +134,7 @@ EOF COPY target/postsrsd/postsrsd /etc/default/postsrsd COPY target/postgrey/postgrey /etc/default/postgrey -COPY target/postgrey/postgrey.init /etc/init.d/postgrey RUN <*` -Password: `` \ No newline at end of file +Password: `` diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 69ed069b..f035523a 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -132,7 +132,7 @@ Next, configure a network with an IPv6 subnet for your container with any of the !!! warning "This approach is discouraged" - The [`bridge` network is considered legacy][docker-docs-network-bridge-legacy]. + The [`bridge` network is considered legacy][docker-docs-network-bridge-legacy]. Add these two extra IPv6 settings to your daemon config. They only apply to the [default `bridge` docker network][docker-docs-ipv6-create-default] aka `docker0` (_which containers are attached to by default when using `docker run`_). diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index d050bf9f..fe540efb 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -69,12 +69,12 @@ It is possible to sort subaddresses such as `user+mailing-lists@example.com` int require ["envelope", "fileinto", "mailbox", "subaddress", "variables"]; if envelope :detail :matches "to" "*" { - set :lower :upperfirst "tag" "${1}"; - if mailboxexists "INBOX.${1}" { - fileinto "INBOX.${1}"; - } else { - fileinto :create "INBOX.${tag}"; - } + set :lower :upperfirst "tag" "${1}"; + if mailboxexists "INBOX.${1}" { + fileinto "INBOX.${1}"; + } else { + fileinto :create "INBOX.${tag}"; + } } ``` diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 4618fda1..c0c615cc 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -161,8 +161,9 @@ Obtain a Cloudflare API token: dns_cloudflare_api_token = YOUR_CLOUDFLARE_TOKEN_HERE ``` - - As this is sensitive data, you should restrict access to it with `chmod 600` and `chown 0:0`. - - Store the file in a folder if you like, such as `docker-data/certbot/secrets/`. + - As this is sensitive data, you should restrict access to it with `chmod 600` and `chown 0:0`. + - Store the file in a folder if you like, such as `docker-data/certbot/secrets/`. + 5. Your `compose.yaml` should include the following: ```yaml @@ -594,7 +595,7 @@ This setup only comes with one caveat: The domain has to be configured on anothe container_name: mailserver hostname: mail.example.com volumes: - - ./docker-data/traefik/acme.json:/etc/letsencrypt/acme.json:ro + - ./docker-data/traefik/acme.json:/etc/letsencrypt/acme.json:ro environment: SSL_TYPE: letsencrypt SSL_DOMAIN: mail.example.com @@ -605,26 +606,26 @@ This setup only comes with one caveat: The domain has to be configured on anothe image: docker.io/traefik:latest #v2.5 container_name: docker-traefik ports: - - "80:80" - - "443:443" + - "80:80" + - "443:443" command: - - --providers.docker - - --entrypoints.http.address=:80 - - --entrypoints.http.http.redirections.entryPoint.to=https - - --entrypoints.http.http.redirections.entryPoint.scheme=https - - --entrypoints.https.address=:443 - - --entrypoints.https.http.tls.certResolver=letsencrypt - - --certificatesresolvers.letsencrypt.acme.email=admin@example.com - - --certificatesresolvers.letsencrypt.acme.storage=/acme.json - - --certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=http + - --providers.docker + - --entrypoints.http.address=:80 + - --entrypoints.http.http.redirections.entryPoint.to=https + - --entrypoints.http.http.redirections.entryPoint.scheme=https + - --entrypoints.https.address=:443 + - --entrypoints.https.http.tls.certResolver=letsencrypt + - --certificatesresolvers.letsencrypt.acme.email=admin@example.com + - --certificatesresolvers.letsencrypt.acme.storage=/acme.json + - --certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=http volumes: - - ./docker-data/traefik/acme.json:/acme.json - - /var/run/docker.sock:/var/run/docker.sock:ro + - ./docker-data/traefik/acme.json:/acme.json + - /var/run/docker.sock:/var/run/docker.sock:ro whoami: image: docker.io/traefik/whoami:latest labels: - - "traefik.http.routers.whoami.rule=Host(`mail.example.com`)" + - "traefik.http.routers.whoami.rule=Host(`mail.example.com`)" ``` ### Self-Signed Certificates diff --git a/docs/content/contributing/tests.md b/docs/content/contributing/tests.md index ba63bf74..6f649529 100644 --- a/docs/content/contributing/tests.md +++ b/docs/content/contributing/tests.md @@ -85,10 +85,10 @@ In this example, you've made a change to the Rspamd feature support (_or adjuste ```console $ make clean generate-accounts test/rspamd rspamd.bats - ✓ [Rspamd] Postfix's main.cf was adjusted [12] - ✓ [Rspamd] normal mail passes fine [44] - ✓ [Rspamd] detects and rejects spam [122] - ✓ [Rspamd] detects and rejects virus [189] + ✓ [Rspamd] Postfix's main.cf was adjusted [12] + ✓ [Rspamd] normal mail passes fine [44] + ✓ [Rspamd] detects and rejects spam [122] + ✓ [Rspamd] detects and rejects virus [189] ``` As your feature work progresses your change for Rspamd also affects ClamAV. As your change now spans more than just the Rspamd test file, you could run multiple test files serially: @@ -96,16 +96,17 @@ As your feature work progresses your change for Rspamd also affects ClamAV. As y ```console $ make clean generate-accounts test/rspamd,clamav rspamd.bats - ✓ [Rspamd] Postfix's main.cf was adjusted [12] - ✓ [Rspamd] normal mail passes fine [44] - ✓ [Rspamd] detects and rejects spam [122] - ✓ [Rspamd] detects and rejects virus [189] + ✓ [Rspamd] Postfix's main.cf was adjusted [12] + ✓ [Rspamd] normal mail passes fine [44] + ✓ [Rspamd] detects and rejects spam [122] + ✓ [Rspamd] detects and rejects virus [189] + clamav.bats - ✓ [ClamAV] log files exist at /var/log/mail directory [68] - ✓ [ClamAV] should be identified by Amavis [67] - ✓ [ClamAV] freshclam cron is enabled [76] - ✓ [ClamAV] env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly [63] - ✓ [ClamAV] rejects virus [60] + ✓ [ClamAV] log files exist at /var/log/mail directory [68] + ✓ [ClamAV] should be identified by Amavis [67] + ✓ [ClamAV] freshclam cron is enabled [76] + ✓ [ClamAV] env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly [63] + ✓ [ClamAV] rejects virus [60] ``` You're almost finished with your change before submitting it as a PR. It's a good idea to run the full parallel set those individual tests belong to (_especially if you've modified any tests_): @@ -113,13 +114,15 @@ You're almost finished with your change before submitting it as a PR. It's a goo ```console $ make clean generate-accounts tests/parallel/set1 default_relay_host.bats - ✓ [Relay] (ENV) 'DEFAULT_RELAY_HOST' should configure 'main.cf:relayhost' [88] + ✓ [Relay] (ENV) 'DEFAULT_RELAY_HOST' should configure 'main.cf:relayhost' [88] + spam_virus/amavis.bats - ✓ [Amavis] SpamAssassin integration should be active [1165] + ✓ [Amavis] SpamAssassin integration should be active [1165] + spam_virus/clamav.bats - ✓ [ClamAV] log files exist at /var/log/mail directory [73] - ✓ [ClamAV] should be identified by Amavis [67] - ✓ [ClamAV] freshclam cron is enabled [76] + ✓ [ClamAV] log files exist at /var/log/mail directory [73] + ✓ [ClamAV] should be identified by Amavis [67] + ✓ [ClamAV] freshclam cron is enabled [76] ... ``` @@ -127,7 +130,6 @@ Even better, before opening a PR run the full test suite: ```console $ make clean tests -... ``` [BATS]: https://github.com/bats-core/bats-core diff --git a/docs/content/faq.md b/docs/content/faq.md index bb2fbe63..4da64b60 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -511,8 +511,9 @@ require ["comparator-i;ascii-numeric","relational","fileinto"]; if header :contains "X-Spam-Flag" "YES" { fileinto "Junk"; } elsif allof ( - not header :matches "x-spam-score" "-*", - header :value "ge" :comparator "i;ascii-numeric" "x-spam-score" "3.75" ) { + not header :matches "x-spam-score" "-*", + header :value "ge" :comparator "i;ascii-numeric" "x-spam-score" "3.75" +) { fileinto "Junk"; } ``` diff --git a/docs/content/introduction.md b/docs/content/introduction.md index bbfd6ef4..3be9ed9d 100644 --- a/docs/content/introduction.md +++ b/docs/content/introduction.md @@ -43,10 +43,10 @@ Here's where DMS's toolchain fits within the delivery chain: ```txt docker-mailserver is here: - ┏━━━━━━━┓ -Sending an email: MUA ---> MTA ---> (MTA relays) ---> ┫ MTA ╮ ┃ -Fetching an email: MUA <------------------------------ ┫ MDA ╯ ┃ - ┗━━━━━━━┛ + ┏━━━━━━━┓ +Sending an email: MUA ---> MTA ---> (MTA relays) ---> ┫ MTA ╮ ┃ +Fetching an email: MUA <------------------------------ ┫ MDA ╯ ┃ + ┗━━━━━━━┛ ``` ??? example "An Example" @@ -86,18 +86,18 @@ When it comes to the specifics of email exchange, we have to look at protocols a The following picture gives a visualization of the interplay of all components and their [respective ports][docs-understandports]: ```txt - ┏━━━━━━━━━━ Submission ━━━━━━━━━━━━┓┏━━━━━━━━━━━━━ Transfer/Relay ━━━━━━━━━━━┓ + ┏━━━━━━━━━━ Submission ━━━━━━━━━━━━━┓┏━━━━━━━━━━━━━ Transfer/Relay ━━━━━━━━━━━┓ - ┌─────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ -MUA ----- STARTTLS ------> ┤(587) MTA ╮ (25)├ <-- cleartext ---> ┊ Third-party MTA ┊ - ----- implicit TLS --> ┤(465) │ | └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ----- cleartext -----> ┤(25) │ | - |┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄| -MUA <---- STARTTLS ------- ┤(143) MDA ╯ | - <---- implicit TLS --- ┤(993) | - └─────────────────────┘ + ┌─────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ +MUA ----- STARTTLS -------> ┤(587) MTA ╮ (25)├ <-- cleartext ---> ┊ Third-party MTA ┊ + ----- implicit TLS ---> ┤(465) │ | └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ + ----- cleartext ------> ┤(25) │ | + |┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄| +MUA <---- STARTTLS -------- ┤(143) MDA ╯ | + <---- implicit TLS ---- ┤(993) | + └─────────────────────┘ - ┗━━━━━━━━━━ Retrieval ━━━━━━━━━━━━━┛ + ┗━━━━━━━━━━ Retrieval ━━━━━━━━━━━━━━┛ ``` If you're new to email infrastructure, both that table and the schema may be confusing. @@ -124,7 +124,7 @@ My MTA will thus have to support two kinds of Submission: - Inbound Submission (third-party email has been submitted & relayed, then is accepted "inside" by the MTA) ```txt - ┏━━━━ Outbound Submission ━━━━┓ + ┏━━━ Outbound Submission ━━━┓ ┌────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ Me ---------------> ┤ ├ -----------------> ┊ ┊ @@ -132,7 +132,7 @@ Me ---------------> ┤ ├ -----------------> ┊ │ ├ <----------------- ┊ ┊ └────────────────────┘ └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ + ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ ``` #### Outbound Submission @@ -168,7 +168,7 @@ Granted it's still very difficult enforcing encryption between MTAs (Transfer/Re Overall, DMS's default configuration for SMTP looks like this: ```txt - ┏━━━━ Outbound Submission ━━━━┓ + ┏━━━ Outbound Submission ━━━┓ ┌────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ Me -- cleartext --> ┤(25) (25)├ --- cleartext ---> ┊ ┊ @@ -177,7 +177,7 @@ Me -- STARTTLS ---> ┤(587) │ ┊ │ (25)├ <---cleartext ---- ┊ ┊ └────────────────────┘ └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ + ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ ``` ### Retrieval - IMAP diff --git a/target/amavis/conf.d/60-dms_default_config b/target/amavis/conf.d/60-dms_default_config index e5b71e2f..5a22b3ad 100644 --- a/target/amavis/conf.d/60-dms_default_config +++ b/target/amavis/conf.d/60-dms_default_config @@ -1,7 +1,7 @@ use strict; @local_domains_maps = ( - read_hash('/etc/postfix/vhost') + read_hash('/etc/postfix/vhost') ); 1; # ensure a defined return diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 11b98975..86fbfb81 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -42,8 +42,8 @@ ${ORANGE}OPTIONS${RESET} selector Set a manual selector for the key. Default: mail domain Provide the domain(s) for which to generate keys for. - Default: The FQDN assigned to DMS, excluding any subdomain. - 'ACCOUNT_PROVISIONER=FILE' also sources domains from mail accounts. + Default: The FQDN assigned to DMS, excluding any subdomain. + 'ACCOUNT_PROVISIONER=FILE' also sources domains from mail accounts. ${ORANGE}EXAMPLES${RESET} ${LWHITE}setup config dkim keysize 4096${RESET} diff --git a/target/postgrey/postgrey.init b/target/postgrey/postgrey.init deleted file mode 100644 index 0ac7564b..00000000 --- a/target/postgrey/postgrey.init +++ /dev/null @@ -1,154 +0,0 @@ -#! /bin/sh - -# postgrey start/stop the postgrey greylisting deamon for postfix -# (priority should be smaller than that of postfix) -# -# Author: (c)2004-2006 Adrian von Bidder -# Based on Debian sarge's 'skeleton' example -# Distribute and/or modify at will. -# -# Version: $Id: postgrey.init 1436 2006-12-07 07:15:03Z avbidder $ -# altered by Georg Lauterbach as aendeavor 2020-11.05 14:02:00Z - -### BEGIN INIT INFO -# Provides: postgrey -# Required-Start: $syslog $local_fs $remote_fs -# Required-Stop: $syslog $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start/stop the postgrey daemon -### END INIT INFO - -set -e - -PATH='/sbin:/bin:/usr/sbin:/usr/bin' -DAEMON='/usr/sbin/postgrey' -DAEMON_NAME='postgrey' -DESC='postfix greylisting daemon' -DAEMON_USER='postgrey' - -PIDFILE="/var/run/${DAEMON_NAME}/${DAEMON_NAME}.pid" -SCRIPTNAME="/etc/init.d/${DAEMON_NAME}" - -# gracefully exit if the package has been removed. -[ -x "${DAEMON}" ] || exit 0 - -# shellcheck source=/dev/null -. /lib/lsb/init-functions - -# Read config file if it is present. -# shellcheck source=/dev/null -[ -r "/etc/default/${DAEMON_NAME}" ] && . "/etc/default/${DAEMON_NAME}" - -POSTGREY_OPTS="--pidfile=${PIDFILE} --daemonize ${POSTGREY_OPTS}" - -if [ -z "${POSTGREY_TEXT}" ]; then - POSTGREY_TEXT_OPT="" -else - POSTGREY_TEXT_OPT="--greylist-text=${POSTGREY_TEXT}" -fi - -ret=0 - -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile \ - "${PIDFILE}" --exec "${DAEMON}" --test >/dev/null || return 1 - - start-stop-daemon --start --quiet --pidfile \ - "${PIDFILE}" --exec "${DAEMON}" -- "${POSTGREY_OPTS}" \ - "${POSTGREY_TEXT_OPT}" || return 2 -} - -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --user "${DAEMON_USER}" --stop --quiet \ - --retry=TERM/30/KILL/5 --pidfile "${PIDFILE}" - - RETVAL="$?" - [ "${RETVAL}" -eq 2 ] && return 2 - - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --user "${DAEMON_USER}" --stop --quiet \ - --oknodo --retry=0/30/KILL/5 --exec "${DAEMON}" - [ "$?" -eq 2 ] && return 2 - - # Many daemons don't delete their pidfiles when they exit. - rm -f "${PIDFILE}" - return "${RETVAL}" -} - -do_reload() -{ - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - start-stop-daemon --stop --signal 1 --quiet --pidfile "${PIDFILE}" - return 0 -} - -case "${1}" in - start ) - [ "${VERBOSE}" != no ] && log_daemon_msg "Starting ${DESC}" "${DAEMON_NAME}" - do_start - - case "${?}" in - 0|1) [ "${VERBOSE}" != no ] && log_end_msg 0 ;; - 2) [ "${VERBOSE}" != no ] && log_end_msg 1 ;; - esac - ;; - - stop ) - [ "${VERBOSE}" != no ] && log_daemon_msg "Stopping ${DESC}" "${DAEMON_NAME}" - do_stop - - case "${?}" in - 0|1) [ "${VERBOSE}" != no ] && log_end_msg 0 ;; - 2) [ "${VERBOSE}" != no ] && log_end_msg 1 ;; - esac - ;; - - reload|force-reload) - [ "${VERBOSE}" != no ] && log_daemon_msg "Reloading ${DESC}" "${DAEMON_NAME}" - do_reload - - case "${?}" in - 0|1) [ "${VERBOSE}" != no ] && log_end_msg 0 ;; - 2) [ "${VERBOSE}" != no ] && log_end_msg 1 ;; - esac - ;; - - restart ) - do_stop - do_start - ;; - - status ) - status_of_proc -p "${PIDFILE}" "${DAEMON}" "${DAEMON_NAME}" 2>/dev/null - ret=${?} - ;; - - * ) - echo "Usage: ${SCRIPTNAME} {start|stop|restart|reload|force-reload|status}" >&2 - exit 1 - ;; -esac - -exit ${ret} - diff --git a/test/config/example-opendkim/keys/localhost.localdomain/mail.txt b/test/config/example-opendkim/keys/localhost.localdomain/mail.txt index ccc08dc0..e9c8cd1a 100644 --- a/test/config/example-opendkim/keys/localhost.localdomain/mail.txt +++ b/test/config/example-opendkim/keys/localhost.localdomain/mail.txt @@ -1,2 +1,2 @@ mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " - "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCzUJyyhq+TeT1wlIth5Z0yr7Ohd62n4rL5X3vRJO4EDyOEicJ73cjuaU4JLTYhbqmbNalOyXE9btS9I55Gv3RyomVBD1JpVTKdjVBUQug2L/ggw2dtt1FAn99svQWMs1XxmxiTR+sCEVkgKMmLSkCJuDCIfY/Bc9nlcng9+juB8wIDAQAB" ) ; ----- DKIM key mail for localhost.localdomain + "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCzUJyyhq+TeT1wlIth5Z0yr7Ohd62n4rL5X3vRJO4EDyOEicJ73cjuaU4JLTYhbqmbNalOyXE9btS9I55Gv3RyomVBD1JpVTKdjVBUQug2L/ggw2dtt1FAn99svQWMs1XxmxiTR+sCEVkgKMmLSkCJuDCIfY/Bc9nlcng9+juB8wIDAQAB" ) ; ----- DKIM key mail for localhost.localdomain diff --git a/test/config/example-opendkim/keys/otherdomain.tld/mail.txt b/test/config/example-opendkim/keys/otherdomain.tld/mail.txt index d132a31c..9d1079f4 100644 --- a/test/config/example-opendkim/keys/otherdomain.tld/mail.txt +++ b/test/config/example-opendkim/keys/otherdomain.tld/mail.txt @@ -1,2 +1,2 @@ mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " - "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCurRsOh4NyTOqDnpPlPLGlQDuoQl32Gdkfzw7BBRKDcelIZBmQf0uhXKSZVKe5Q596w/3ESJ9WOlB03SISnHy8lq/ZJ1+vhSZQfHvp0cHQl4BgNzktRCARdPY+5nVerF8aUSsT3bG2O+2r09AY4okLCVfkiwg6Nz2Eo7j4Z7mqNwIDAQAB" ) ; ----- DKIM key mail for otherdomain.tld + "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCurRsOh4NyTOqDnpPlPLGlQDuoQl32Gdkfzw7BBRKDcelIZBmQf0uhXKSZVKe5Q596w/3ESJ9WOlB03SISnHy8lq/ZJ1+vhSZQfHvp0cHQl4BgNzktRCARdPY+5nVerF8aUSsT3bG2O+2r09AY4okLCVfkiwg6Nz2Eo7j4Z7mqNwIDAQAB" ) ; ----- DKIM key mail for otherdomain.tld diff --git a/test/config/fetchmail/fetchmail.cf b/test/config/fetchmail/fetchmail.cf index aead698c..11168505 100644 --- a/test/config/fetchmail/fetchmail.cf +++ b/test/config/fetchmail/fetchmail.cf @@ -1,11 +1,11 @@ poll pop3.third-party.test. with proto POP3 - user 'remote_username' there with - password 'secret' - is 'local_username' here - options keep ssl + user 'remote_username' there with + password 'secret' + is 'local_username' here + options keep ssl poll imap.remote-service.test. with proto IMAP - user 'user3' there with - password 'secret' - is 'user3@example.test' here - options keep ssl + user 'user3' there with + password 'secret' + is 'user3@example.test' here + options keep ssl diff --git a/test/config/ldap/openldap/schemas/postfix-book.ldif b/test/config/ldap/openldap/schemas/postfix-book.ldif index 9ea787d5..543dc61a 100644 --- a/test/config/ldap/openldap/schemas/postfix-book.ldif +++ b/test/config/ldap/openldap/schemas/postfix-book.ldif @@ -1,14 +1,14 @@ -dn: cn=postfix-book,cn=schema,cn=config -objectClass: olcSchemaConfig -cn: postfix-book -olcAttributeTypes: {0}( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' DESC 'The absolute path to the mail user home directory' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) -olcAttributeTypes: {1}( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' DESC 'RFC822 Mailbox - mail alias' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) -olcAttributeTypes: {2}( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' DESC 'UID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) -olcAttributeTypes: {3}( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' DESC 'GID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) -olcAttributeTypes: {4}( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' DESC 'TRUE to enable, FALSE to disable account' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) -olcAttributeTypes: {5}( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' DESC 'Name of a mail distribution list' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) -olcAttributeTypes: {6}( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' DESC 'Mail quota limit in kilobytes' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) -olcAttributeTypes: {7}( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' DESC 'The absolute path to the mail users mailbox' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) -# PostfixBook object classes: -olcObjectClasses: {0}( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' DESC 'Mail account used in Postfix Book' SUP top AUXILIARY MUST mail MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember $ mailUidNumber $ mailGidNumber $ mailEnabled $ mailQuota $ mailStorageDirectory ) ) -olcObjectClasses: {1}( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' DESC 'Mail forward used in Postfix Book' SUP top AUXILIARY MUST ( mail $ mailAlias ) ) +dn: cn=postfix-book,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: postfix-book +olcAttributeTypes: {0}( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' DESC 'The absolute path to the mail user home directory' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +olcAttributeTypes: {1}( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' DESC 'RFC822 Mailbox - mail alias' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) +olcAttributeTypes: {2}( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' DESC 'UID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {3}( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' DESC 'GID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {4}( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' DESC 'TRUE to enable, FALSE to disable account' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +olcAttributeTypes: {5}( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' DESC 'Name of a mail distribution list' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {6}( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' DESC 'Mail quota limit in kilobytes' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {7}( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' DESC 'The absolute path to the mail users mailbox' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +# PostfixBook object classes: +olcObjectClasses: {0}( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' DESC 'Mail account used in Postfix Book' SUP top AUXILIARY MUST mail MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember $ mailUidNumber $ mailGidNumber $ mailEnabled $ mailQuota $ mailStorageDirectory ) ) +olcObjectClasses: {1}( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' DESC 'Mail forward used in Postfix Book' SUP top AUXILIARY MUST ( mail $ mailAlias ) ) diff --git a/test/config/override-configs/postfix-main.cf b/test/config/override-configs/postfix-main.cf index 77c2d787..ce07bd56 100644 --- a/test/config/override-configs/postfix-main.cf +++ b/test/config/override-configs/postfix-main.cf @@ -1,4 +1,4 @@ max_idle = 600s # this is a comment - # this is also a comment + # this is also a comment readme_directory = /tmp diff --git a/test/config/override-configs/postfix-master.cf b/test/config/override-configs/postfix-master.cf index 4ebc7d21..516fea81 100644 --- a/test/config/override-configs/postfix-master.cf +++ b/test/config/override-configs/postfix-master.cf @@ -1,3 +1,3 @@ submission/inet/smtpd_sasl_security_options=noanonymous # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O diff --git a/test/config/postfix-regexp.cf b/test/config/postfix-regexp.cf index 6bbb6fa7..01d448a8 100644 --- a/test/config/postfix-regexp.cf +++ b/test/config/postfix-regexp.cf @@ -1,6 +1,5 @@ /^test[0-9][0-9]*@localhost.localdomain/ user1@localhost.localdomain # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O /^bounce.*@.*/ external1@otherdomain.tld /^postmaster@/ user1@localhost.localdomain - diff --git a/test/config/postfix-virtual.cf b/test/config/postfix-virtual.cf index 67a58cff..4dec6bbb 100644 --- a/test/config/postfix-virtual.cf +++ b/test/config/postfix-virtual.cf @@ -1,5 +1,5 @@ alias1@localhost.localdomain user1@localhost.localdomain # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O alias2@localhost.localdomain external1@otherdomain.tld @localdomain2.com user1@localhost.localdomain diff --git a/test/config/templates/dovecot-masters.cf b/test/config/templates/dovecot-masters.cf index e519ec75..8d3f8977 100644 --- a/test/config/templates/dovecot-masters.cf +++ b/test/config/templates/dovecot-masters.cf @@ -1 +1 @@ -masterusername|{SHA512-CRYPT}$6$IOybywiyl1nuDno0$gRW625qH7ThmbRaByNVpuAGgDOkMd7tc3yuVmwVRuk7IXgiN8KDwcqtMcU0LyvS5RGAskbplavjPpCmFjbKEt1 +masterusername|{SHA512-CRYPT}$6$IOybywiyl1nuDno0$gRW625qH7ThmbRaByNVpuAGgDOkMd7tc3yuVmwVRuk7IXgiN8KDwcqtMcU0LyvS5RGAskbplavjPpCmFjbKEt1 diff --git a/test/config/templates/postfix-accounts.cf b/test/config/templates/postfix-accounts.cf index 03f6ef0b..9d538ad3 100644 --- a/test/config/templates/postfix-accounts.cf +++ b/test/config/templates/postfix-accounts.cf @@ -1,5 +1,5 @@ -user1@localhost.localdomain|{SHA512-CRYPT}$6$DBEbjh4I9P7aROk8$XosqE.YI2Z4bUkWD1/bedrSNpw79nsO60yiAKk04jARhPVX5VD/SaVM5HWFDQyzftESVDjbVdhzn/d4TJxFwg0 -user2@otherdomain.tld|{SHA512-CRYPT}$6$PQRkR3RRzpYP4WET$NKLJk3PkwTRRSxryqFhQloBR7qSAYjoQH/IbD1ZQKX2UJJ3jmdbOMQPfMRGXBZv3JGhDUPmAiWzoJL6/NJN5d/ -user3@localhost.localdomain|{SHA512-CRYPT}$6$lZwv0IoijHyEjDtM$vGsAS7KM5O5Q1NdWjard1LbJyGiHcqHhKAXBKDIMudjB/CuVvOvXKVy2yKeeRvKxVtkCdYac738VQPL.kpSVB.|userdb_mail=mbox:~/mail:INBOX=~/inbox +user1@localhost.localdomain|{SHA512-CRYPT}$6$DBEbjh4I9P7aROk8$XosqE.YI2Z4bUkWD1/bedrSNpw79nsO60yiAKk04jARhPVX5VD/SaVM5HWFDQyzftESVDjbVdhzn/d4TJxFwg0 +user2@otherdomain.tld|{SHA512-CRYPT}$6$PQRkR3RRzpYP4WET$NKLJk3PkwTRRSxryqFhQloBR7qSAYjoQH/IbD1ZQKX2UJJ3jmdbOMQPfMRGXBZv3JGhDUPmAiWzoJL6/NJN5d/ +user3@localhost.localdomain|{SHA512-CRYPT}$6$lZwv0IoijHyEjDtM$vGsAS7KM5O5Q1NdWjard1LbJyGiHcqHhKAXBKDIMudjB/CuVvOvXKVy2yKeeRvKxVtkCdYac738VQPL.kpSVB.|userdb_mail=mbox:~/mail:INBOX=~/inbox # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O diff --git a/test/linting/.ecrc.json b/test/linting/.ecrc.json index d9abd2f6..ba92bf77 100644 --- a/test/linting/.ecrc.json +++ b/test/linting/.ecrc.json @@ -1,25 +1,9 @@ { - "Verbose": false, - "Debug": false, "IgnoreDefaults": false, - "SpacesAftertabs": true, - "NoColor": false, "Exclude": [ - "^test/", - "\\.git.*", - "\\.cf$", - "\\.conf$", - "\\.init$", - "\\.md$" - ], - "AllowedContentTypes": [], - "PassedFiles": [], - "Disable": { - "EndOfLine": false, - "Indentation": false, - "InsertFinalNewline": false, - "TrimTrailingWhitespace": false, - "IndentSize": false, - "MaxLineLength": false - } + "^test/bats/", + "^test/test_helper/bats-(assert|support)", + "^test/test-files/", + "\\.git/" + ] } diff --git a/test/tests/parallel/set1/spam_virus/amavis.bats b/test/tests/parallel/set1/spam_virus/amavis.bats index 5ab8cd9d..bc920234 100644 --- a/test/tests/parallel/set1/spam_virus/amavis.bats +++ b/test/tests/parallel/set1/spam_virus/amavis.bats @@ -19,7 +19,7 @@ function setup_file() { CONTAINER_NAME=${CONTAINER2_NAME} _init_with_defaults - local CUSTOM_SETUP_ARGUMENTS=( + local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=0 --env ENABLE_SPAMASSASSIN=0 ) diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 1b73643e..b7b3884b 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -50,7 +50,7 @@ function setup_file() { # For this lookup `%s` only represents the domain, not a full email address. Hence the match pattern using a wildcard prefix `*@`. # For a breakdown, see QUERY_SENDERS comment. # NOTE: Although `result_attribute = mail` will return each accounts full email address, Postfix will only compare to domain-part. - local QUERY_DOMAIN='(| (& (|(mail=*@%s) (mailAlias=*@%s) (mailGroupMember=*@%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=*@%s)(objectClass=PostfixBookMailForward)) )' + local QUERY_DOMAIN='(| (& (|(mail=*@%s) (mailAlias=*@%s) (mailGroupMember=*@%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=*@%s)(objectClass=PostfixBookMailForward)) )' # Simple queries for a single attribute that additionally requires `mailEnabled=TRUE` from the `PostfixBookMailAccount` class: # NOTE: `mail` attribute is not unique to `PostfixBookMailAccount`. The `mailEnabled` attribute is to further control valid mail accounts. From 82c38f242648290ab109de8e3994ea7564120deb Mon Sep 17 00:00:00 2001 From: Vincent Ducamps Date: Sun, 8 Oct 2023 00:25:57 +0200 Subject: [PATCH 038/125] docs: TLS - Include `passthrough=true` on implicit ports for Traefik example (#3568) --- docs/content/examples/tutorials/mailserver-behind-proxy.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 501d1209..7bdef5d6 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -72,14 +72,15 @@ Feel free to add your configuration if you achieved the same goal using differen - "traefik.tcp.services.smtp.loadbalancer.server.port=25" - "traefik.tcp.services.smtp.loadbalancer.proxyProtocol.version=1" - "traefik.tcp.routers.smtp-ssl.rule=HostSNI(`*`)" - - "traefik.tcp.routers.smtp-ssl.tls=false" - "traefik.tcp.routers.smtp-ssl.entrypoints=smtp-ssl" + - "traefik.tcp.routers.smtp-ssl.tls.passthrough=true" - "traefik.tcp.routers.smtp-ssl.service=smtp-ssl" - "traefik.tcp.services.smtp-ssl.loadbalancer.server.port=465" - "traefik.tcp.services.smtp-ssl.loadbalancer.proxyProtocol.version=1" - "traefik.tcp.routers.imap-ssl.rule=HostSNI(`*`)" - "traefik.tcp.routers.imap-ssl.entrypoints=imap-ssl" - "traefik.tcp.routers.imap-ssl.service=imap-ssl" + - "traefik.tcp.routers.imap-ssl.tls.passthrough=true" - "traefik.tcp.services.imap-ssl.loadbalancer.server.port=10993" - "traefik.tcp.services.imap-ssl.loadbalancer.proxyProtocol.version=2" - "traefik.tcp.routers.sieve.rule=HostSNI(`*`)" From 894978ddd751f0d09e9f68e1a3de13926e17d432 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 14 Oct 2023 17:14:10 +0200 Subject: [PATCH 039/125] refactor: `logrotate` setup + rspamd log path + tests log helper fallback path (#3576) * simplify `_setup_logrotate` * adjust Rspamd's log file and improve it's management * add information to docs about Rspamd log * update log query helper to allow another file location * bail in case `LOGROTATE_INTERVAL` is invalid --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/security/rspamd.md | 4 ++ target/scripts/startup/setup.d/log.sh | 39 +++++++------------ .../startup/setup.d/security/rspamd.sh | 15 +++++++ target/supervisor/conf.d/supervisor-app.conf | 4 +- test/helper/common.bash | 5 ++- 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 7876b998..94be2db9 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -69,6 +69,10 @@ DMS does not supply custom values for DNS servers to Rspamd. If you need to use This setting is enabled to not allow spam to proceed just because DNS requests did not succeed. It could deny legitimate e-mails to pass though too in case your DNS setup is incorrect or not functioning properly. +### Logs + +You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the corresponding logs for [Redis](#persistence-with-redis), if it is enabled, at `/var/log/supervisor/rspamd-redis.log`. We recommend inspecting these logs (with `docker exec -it cat /var/log/mail/rspamd.log`) in case Rspamd does not work as expected. + ### Modules You can find a list of all Rspamd modules [on their website][rspamd-docs-modules]. diff --git a/target/scripts/startup/setup.d/log.sh b/target/scripts/startup/setup.d/log.sh index eef76a3f..cf282966 100644 --- a/target/scripts/startup/setup.d/log.sh +++ b/target/scripts/startup/setup.d/log.sh @@ -13,31 +13,22 @@ function _setup_logs_general() { function _setup_logrotate() { _log 'debug' 'Setting up logrotate' - LOGROTATE='/var/log/mail/mail.log\n{\n compress\n copytruncate\n delaycompress\n' + if [[ ${LOGROTATE_INTERVAL} =~ ^(daily|weekly|monthly)$ ]]; then + _log 'trace' "Logrotate interval set to ${LOGROTATE_INTERVAL}" + else + _dms_panic__invalid_value 'LOGROTATE_INTERVAL' 'Setup -> Logrotate' + fi - case "${LOGROTATE_INTERVAL}" in - ( 'daily' ) - _log 'trace' 'Setting postfix logrotate interval to daily' - LOGROTATE="${LOGROTATE} rotate 4\n daily\n" - ;; - - ( 'weekly' ) - _log 'trace' 'Setting postfix logrotate interval to weekly' - LOGROTATE="${LOGROTATE} rotate 4\n weekly\n" - ;; - - ( 'monthly' ) - _log 'trace' 'Setting postfix logrotate interval to monthly' - LOGROTATE="${LOGROTATE} rotate 4\n monthly\n" - ;; - - ( * ) - _log 'warn' 'LOGROTATE_INTERVAL not found in _setup_logrotate' - ;; - - esac - - echo -e "${LOGROTATE}}" >/etc/logrotate.d/maillog + cat >/etc/logrotate.d/maillog << EOF +/var/log/mail/mail.log +{ + compress + copytruncate + delaycompress + rotate 4 + ${LOGROTATE_INTERVAL} +} +EOF } function _setup_mail_summary() { diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index d2389a07..3e943285 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -7,6 +7,7 @@ function _setup_rspamd() { __rspamd__log 'trace' '---------- Setup started ----------' __rspamd__run_early_setup_and_checks # must run first + __rspamd__setup_logfile __rspamd__setup_redis __rspamd__setup_postfix __rspamd__setup_clamav @@ -101,6 +102,20 @@ function __rspamd__run_early_setup_and_checks() { fi } +# Keep in sync with `target/scripts/startup/setup.d/log.sh:_setup_logrotate()` +function __rspamd__setup_logfile() { + cat >/etc/logrotate.d/rspamd << EOF +/var/log/mail/rspamd.log +{ + compress + copytruncate + delaycompress + rotate 4 + ${LOGROTATE_INTERVAL} +} +EOF +} + # Sets up Redis. In case the user does not use a dedicated Redis instance, we # supply a configuration for our local Redis instance which is started later. function __rspamd__setup_redis() { diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index d426831f..2dd8b917 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -101,8 +101,8 @@ startsecs=0 stopwaitsecs=55 autostart=false autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log +stdout_logfile=/var/log/mail/%(program_name)s.log +stderr_logfile=/var/log/mail/%(program_name)s.log command=/usr/bin/rspamd --no-fork --user=_rspamd --group=_rspamd [program:rspamd-redis] diff --git a/test/helper/common.bash b/test/helper/common.bash index cc0eda3d..8fb7854e 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -429,8 +429,11 @@ function _filter_service_log() { local SERVICE=${1:?Service name must be provided} local STRING=${2:?String to match must be provided} local CONTAINER_NAME=$(__handle_container_name "${3:-}") + local FILE="/var/log/supervisor/${SERVICE}.log" - _run_in_container grep -E "${STRING}" "/var/log/supervisor/${SERVICE}.log" + # Fallback to alternative log location: + [[ -f ${FILE} ]] || FILE="/var/log/mail/${SERVICE}.log" + _run_in_container grep -E "${STRING}" "${FILE}" } # Like `_filter_service_log` but asserts that the string was found. From 128e6b4d1f3873522806e067a0163cee637b8510 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 16 Oct 2023 09:51:48 +0200 Subject: [PATCH 040/125] chore: Add debug group (`packages.sh`) + more resilient rspamd setup (#3578) --- target/scripts/build/packages.sh | 26 ++++++++++--------- target/scripts/helpers/utils.sh | 14 +++++++--- .../startup/setup.d/security/rspamd.sh | 22 ++++++++++++---- .../set3/scripts/helper_functions.bats | 6 ++++- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 40e7365e..e9b2d479 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -43,10 +43,6 @@ function _install_postfix() { function _install_packages() { _log 'debug' 'Installing all packages now' - declare -a ANTI_VIRUS_SPAM_PACKAGES - declare -a CODECS_PACKAGES MISCELLANEOUS_PACKAGES - declare -a POSTFIX_PACKAGES MAIL_PROGRAMS_PACKAGES - ANTI_VIRUS_SPAM_PACKAGES=( amavisd-new clamav clamav-daemon pyzor razor spamassassin @@ -62,14 +58,13 @@ function _install_packages() { ) MISCELLANEOUS_PACKAGES=( - apt-transport-https bind9-dnsutils binutils bsd-mailx + apt-transport-https binutils bsd-mailx ca-certificates curl dbconfig-no-thanks - dumb-init ed gnupg iproute2 iputils-ping - libdate-manip-perl libldap-common - libmail-spf-perl libnet-dns-perl - locales logwatch netcat-openbsd - nftables rsyslog supervisor - uuid whois + dumb-init gnupg iproute2 libdate-manip-perl + libldap-common libmail-spf-perl + libnet-dns-perl locales logwatch + netcat-openbsd nftables rsyslog + supervisor uuid whois ) POSTFIX_PACKAGES=( @@ -82,12 +77,19 @@ function _install_packages() { opendmarc libsasl2-modules sasl2-bin ) + # `bind9-dnsutils` provides the `dig` command + # `iputils-ping` provides the `ping` command + DEBUG_PACKAGES=( + bind9-dnsutils iputils-ping less nano + ) + apt-get "${QUIET}" --no-install-recommends install \ "${ANTI_VIRUS_SPAM_PACKAGES[@]}" \ "${CODECS_PACKAGES[@]}" \ "${MISCELLANEOUS_PACKAGES[@]}" \ "${POSTFIX_PACKAGES[@]}" \ - "${MAIL_PROGRAMS_PACKAGES[@]}" + "${MAIL_PROGRAMS_PACKAGES[@]}" \ + "${DEBUG_PACKAGES[@]}" } function _install_dovecot() { diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index e44f0aff..f7095bf3 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -127,9 +127,17 @@ function _replace_by_env_in_file() { function _env_var_expect_zero_or_one() { local ENV_VAR_NAME=${1:?ENV var name must be provided to _env_var_expect_zero_or_one} - [[ ${!ENV_VAR_NAME} =~ ^(0|1)$ ]] && return 0 - _log 'warn' "The value of '${ENV_VAR_NAME}' is not zero or one ('${!ENV_VAR_NAME}'), but was expected to be" - return 1 + if [[ ! -v ${ENV_VAR_NAME} ]]; then + _log 'warn' "'${ENV_VAR_NAME}' is not set, but was expected to be" + return 1 + fi + + if [[ ! ${!ENV_VAR_NAME} =~ ^(0|1)$ ]]; then + _log 'warn' "The value of '${ENV_VAR_NAME}' (= '${!ENV_VAR_NAME}') is not 0 or 1, but was expected to be" + return 1 + fi + + return 0 } # Check if an environment variable's value is an integer. diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 3e943285..4199b077 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -43,6 +43,8 @@ function __rspamd__helper__enable_disable_module() { local LOCAL_OR_OVERRIDE=${3:-local} local MESSAGE='Enabling' + readonly MODULE ENABLE_MODULE LOCAL_OR_OVERRIDE + if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]]; then __rspamd__log 'warn' "__rspamd__helper__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not" return 1 @@ -64,10 +66,12 @@ EOF function __rspamd__run_early_setup_and_checks() { # Note: Variables not marked with `local` are # used in other functions as well. - RSPAMD_LOCAL_D='/etc/rspamd/local.d' - RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' + readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d/" + readonly RSPAMD_DMS_OVERRIDE_D mkdir -p /var/lib/rspamd/ : >/var/lib/rspamd/stats.ucl @@ -77,7 +81,7 @@ function __rspamd__run_early_setup_and_checks() { if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null; then ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" else - __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty? not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" + __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?; not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" fi fi @@ -195,6 +199,7 @@ function __rspamd__setup_default_modules() { metric_exporter ) + readonly -a DISABLE_MODULES local MODULE for MODULE in "${DISABLE_MODULES[@]}"; do __rspamd__helper__enable_disable_module "${MODULE}" 'false' @@ -211,6 +216,7 @@ function __rspamd__setup_learning() { __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' + readonly SIEVE_PIPE_BIN_DIR ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf @@ -264,6 +270,7 @@ function __rspamd__setup_greylisting() { # succeeds. function __rspamd__setup_hfilter_group() { local MODULE_FILE="${RSPAMD_LOCAL_D}/hfilter_group.conf" + readonly MODULE_FILE if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]]; then __rspamd__log 'debug' 'Hfilter (group) module is enabled' # Check if we received a number first @@ -284,6 +291,7 @@ function __rspamd__setup_hfilter_group() { function __rspamd__setup_check_authenticated() { local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf" + readonly MODULE_FILE if _env_var_expect_zero_or_one 'RSPAMD_CHECK_AUTHENTICATED' \ && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]] then @@ -320,8 +328,10 @@ function __rspamd__handle_user_modules_adjustments() { local VALUE=${4:?Value belonging to an option must be provided} # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty) VALUE=${VALUE% } - local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" + + readonly MODULE_FILE MODULE_LOG_NAME OPTION VALUE FILE + [[ -f ${FILE} ]] || touch "${FILE}" if grep -q -E "${OPTION}.*=.*" "${FILE}"; then @@ -335,6 +345,7 @@ function __rspamd__handle_user_modules_adjustments() { local RSPAMD_CUSTOM_COMMANDS_FILE="${RSPAMD_DMS_D}/custom-commands.conf" local RSPAMD_CUSTOM_COMMANDS_FILE_OLD="${RSPAMD_DMS_D}-modules.conf" + readonly RSPAMD_CUSTOM_COMMANDS_FILE RSPAMD_CUSTOM_COMMANDS_FILE_OLD # We check for usage of the previous location of the commands file. # This can be removed after the release of v14.0.0. @@ -347,6 +358,7 @@ function __rspamd__handle_user_modules_adjustments() { if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]]; then __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" + local COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do case "${COMMAND}" in ('disable-module') diff --git a/test/tests/parallel/set3/scripts/helper_functions.bats b/test/tests/parallel/set3/scripts/helper_functions.bats index 5a1fbf74..332de448 100644 --- a/test/tests/parallel/set3/scripts/helper_functions.bats +++ b/test/tests/parallel/set3/scripts/helper_functions.bats @@ -35,7 +35,11 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/ run _env_var_expect_zero_or_one TWO assert_failure - assert_output --partial "The value of 'TWO' is not zero or one ('2'), but was expected to be" + assert_output --partial "The value of 'TWO' (= '2') is not 0 or 1, but was expected to be" + + run _env_var_expect_zero_or_one UNSET + assert_failure + assert_output --partial "'UNSET' is not set, but was expected to be" run _env_var_expect_zero_or_one assert_failure From 4a58e2eb8342203d43f78b3a67fc5bbd1f3cd980 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 17 Oct 2023 02:20:50 +1300 Subject: [PATCH 041/125] docs: Revise `watchtower` page (#3583) - Better clarify images are only updated to newer image updates to the same tag of the running container(s). - Slight revisions to existing content. --- .../maintenance/update-and-cleanup.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/content/config/advanced/maintenance/update-and-cleanup.md b/docs/content/config/advanced/maintenance/update-and-cleanup.md index b972c4a5..42f039fc 100644 --- a/docs/content/config/advanced/maintenance/update-and-cleanup.md +++ b/docs/content/config/advanced/maintenance/update-and-cleanup.md @@ -6,6 +6,8 @@ title: 'Maintenance | Update and Cleanup' !!! example "Automatic image updates + cleanup" + Run a `watchtower` container with access to `docker.sock`, enabling the service to manage Docker: + ```yaml title="compose.yaml" services: watchtower: @@ -17,26 +19,35 @@ title: 'Maintenance | Update and Cleanup' - /var/run/docker.sock:/var/run/docker.sock ``` +!!! tip "The image tag used for a container is monitored for updates (eg: `:latest`, `:edge`, `:13`)" + + The automatic update support is **only for updates to that specific image tag**. + + - Your container will not update to a new major version tag (_unless using `:latest`_). + - Omit the minor or patch portion of the semver tag to receive updates for the omitted portion (_eg: `13` will represent the latest minor + patch release of `v13`_). + !!! tip "Updating only specific containers" - The default `watchtower` service will check every 24 hours for any new image updates to pull, **not only the images** defined within your `compose.yaml`. + By default the `watchtower` service will check every 24 hours for new image updates to pull, based on currently running containers (_**not restricted** to only those running within your `compose.yaml`_). - The images to update can be restricted with a custom command that provides a list of containers names and other config options. Configuration is detailed in the [`watchtower` docs][watchtower-docs]. + Images eligible for updates can configured with a [custom `command`][docker-docs-compose-command] that provides a list of container names, or via other supported options (eg: labels). This configuration is detailed in the [`watchtower` docs][watchtower-docs]. !!! info "Manual cleanup" `watchtower` also supports running on-demand with `docker run` or `compose.yaml` via the `--run-once` option. - You can also directly invoke cleanup of Docker storage with: + You can alternatively invoke cleanup of Docker storage directly with: - [`docker image prune --all`][docker-docs-prune-image] - [`docker system prune --all`][docker-docs-prune-system] (_also removes unused containers, networks, build cache_). - - Avoid the `--all` option to only remove ["dangling" content][docker-prune-dangling] (_eg: Orphaned images_). + + If you omit the `--all` option, this will instead only remove ["dangling" content][docker-prune-dangling] (_eg: Orphaned images_). [watchtower-dockerhub]: https://hub.docker.com/r/containrrr/watchtower [watchtower-cleanup]: https://containrrr.github.io/watchtower/arguments/#cleanup [watchtower-docs]: https://containrrr.dev/watchtower/ +[docker-docs-compose-command]: https://docs.docker.com/compose/compose-file/05-services/#command [docker-docs-prune-image]: https://docs.docker.com/engine/reference/commandline/image_prune/ [docker-docs-prune-system]: https://docs.docker.com/engine/reference/commandline/system_prune/ [docker-prune-dangling]: https://stackoverflow.com/questions/45142528/what-is-a-dangling-image-and-what-is-an-unused-image/60756668#60756668 From f6e556e33f86591ee189d318c27fcc97a59c9e10 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Tue, 17 Oct 2023 10:24:01 +0200 Subject: [PATCH 042/125] docs: fix path to rspamd.log (#3585) --- docs/content/config/best-practices/dkim_dmarc_spf.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index eda98600..da496aa5 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -202,7 +202,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: When `check_pubkey = true;` is set, Rspamd will query the DNS record for each DKIM selector, verifying each public key matches the private key configured. - If there is a mismatch, a warning will be emitted to the Rspamd log `/var/log/supervisor/rspamd.log`. + If there is a mismatch, a warning will be emitted to the Rspamd log `/var/log/mail/rspamd.log`. ### DNS Record { #dkim-dns } @@ -265,7 +265,7 @@ When mail signed with your DKIM key is sent from your mail server, the receiver [MxToolbox has a DKIM Verifier][mxtoolbox-dkim-verifier] that you can use to check your DKIM DNS record(s). -When using Rspamd, we recommend you turn on `check_pubkey = true;` in `dkim_signing.conf`. Rspamd will then check whether your private key matches your public key, and you can check possible mismatches by looking at `/var/log/supervisor/rspamd.log`. +When using Rspamd, we recommend you turn on `check_pubkey = true;` in `dkim_signing.conf`. Rspamd will then check whether your private key matches your public key, and you can check possible mismatches by looking at `/var/log/mail/rspamd.log`. ## DMARC From 811a7698459ebd40cbd520740c2074f4ad7402a6 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 18 Oct 2023 10:47:43 +1300 Subject: [PATCH 043/125] ci: Update `eclint` to `2.7.2` (#3584) This release contains a bugfix for charset detection, resolving a linting bug affecting the test-files directory. --- test/linting/.ecrc.json | 1 - test/linting/lint.sh | 2 +- .../email-templates/send-privacy-email.txt | 2 +- test/test-files/ssl/example.test/README.md | 62 +++++++++---------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/test/linting/.ecrc.json b/test/linting/.ecrc.json index ba92bf77..374615fe 100644 --- a/test/linting/.ecrc.json +++ b/test/linting/.ecrc.json @@ -3,7 +3,6 @@ "Exclude": [ "^test/bats/", "^test/test_helper/bats-(assert|support)", - "^test/test-files/", "\\.git/" ] } diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 88103ddd..7c281e8f 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -10,7 +10,7 @@ shopt -s inherit_errexit REPOSITORY_ROOT=$(realpath "$(dirname "$(readlink -f "${0}")")"/../../) LOG_LEVEL=${LOG_LEVEL:-debug} HADOLINT_VERSION='2.12.0' -ECLINT_VERSION='2.7.0' +ECLINT_VERSION='2.7.2' SHELLCHECK_VERSION='0.9.0' # shellcheck source=./../../target/scripts/helpers/log.sh diff --git a/test/test-files/email-templates/send-privacy-email.txt b/test/test-files/email-templates/send-privacy-email.txt index 2273a46c..0c51ec5b 100644 --- a/test/test-files/email-templates/send-privacy-email.txt +++ b/test/test-files/email-templates/send-privacy-email.txt @@ -7,7 +7,7 @@ data From: Some User To: Some User User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:52.0) - Gecko/20100101 Thunderbird/52.2.1 + Gecko/20100101 Thunderbird/52.2.1 Subject: Test ESMTP Auth LOGIN and remove privacy This is a test mail. diff --git a/test/test-files/ssl/example.test/README.md b/test/test-files/ssl/example.test/README.md index 274f6338..e1f8cdaf 100644 --- a/test/test-files/ssl/example.test/README.md +++ b/test/test-files/ssl/example.test/README.md @@ -71,21 +71,21 @@ Certificate: X509v3 Subject Alternative Name: DNS:example.test, DNS:mail.example.test Signature Algorithm: SHA256-RSA - 50:47:7b:59:26:9d:8d:f7:e4:dc:03:94:b0:35:e4:03:b7:94: - 16:7e:b6:79:c5:bb:e7:61:db:ca:e6:22:cc:c8:a0:9f:9d:b0: - 7c:12:43:ec:a7:f3:fe:ad:0a:44:69:69:7f:c7:31:f7:3f:e8: - 98:a7:37:43:bd:fb:5b:c6:85:85:91:dc:29:23:cb:6b:a9:aa: - f0:f0:62:79:ce:43:8c:5f:28:49:ee:a1:d4:16:67:6b:59:c3: - 15:65:e3:d3:3b:35:da:59:35:33:2a:5e:8a:59:ff:14:b9:51: - a5:8e:0b:7c:1b:a1:b1:f4:89:1a:3f:2f:d7:b1:8d:23:0a:7a: - 79:e1:c2:03:b5:2f:ee:34:16:a9:67:27:b6:10:67:5d:f4:1d: - d6:b3:e0:ab:80:3d:59:fc:bc:4b:1a:55:fb:36:75:ff:e3:88: - 73:e3:16:4d:2b:17:7b:2a:21:a3:18:14:04:19:b3:b8:11:39: - 55:3f:ce:21:b7:d3:5d:8d:78:d5:3a:e0:b2:17:41:ad:3c:8e: - a5:a2:ba:eb:3d:b6:9e:2c:ef:7d:d5:cc:71:cb:07:54:21:42: - 81:79:45:2b:93:74:93:a1:c9:f1:5e:5e:11:3d:ac:df:55:98: - 37:44:d2:55:a5:15:a9:33:79:6e:fe:49:6d:e5:7b:a0:1c:12: - c5:1b:4d:33 + 50:47:7b:59:26:9d:8d:f7:e4:dc:03:94:b0:35:e4:03:b7:94: + 16:7e:b6:79:c5:bb:e7:61:db:ca:e6:22:cc:c8:a0:9f:9d:b0: + 7c:12:43:ec:a7:f3:fe:ad:0a:44:69:69:7f:c7:31:f7:3f:e8: + 98:a7:37:43:bd:fb:5b:c6:85:85:91:dc:29:23:cb:6b:a9:aa: + f0:f0:62:79:ce:43:8c:5f:28:49:ee:a1:d4:16:67:6b:59:c3: + 15:65:e3:d3:3b:35:da:59:35:33:2a:5e:8a:59:ff:14:b9:51: + a5:8e:0b:7c:1b:a1:b1:f4:89:1a:3f:2f:d7:b1:8d:23:0a:7a: + 79:e1:c2:03:b5:2f:ee:34:16:a9:67:27:b6:10:67:5d:f4:1d: + d6:b3:e0:ab:80:3d:59:fc:bc:4b:1a:55:fb:36:75:ff:e3:88: + 73:e3:16:4d:2b:17:7b:2a:21:a3:18:14:04:19:b3:b8:11:39: + 55:3f:ce:21:b7:d3:5d:8d:78:d5:3a:e0:b2:17:41:ad:3c:8e: + a5:a2:ba:eb:3d:b6:9e:2c:ef:7d:d5:cc:71:cb:07:54:21:42: + 81:79:45:2b:93:74:93:a1:c9:f1:5e:5e:11:3d:ac:df:55:98: + 37:44:d2:55:a5:15:a9:33:79:6e:fe:49:6d:e5:7b:a0:1c:12: + c5:1b:4d:33 ``` @@ -139,10 +139,10 @@ Certificate: X509v3 Subject Alternative Name: DNS:example.test, DNS:mail.example.test Signature Algorithm: ECDSA-SHA256 - 30:46:02:21:00:f8:72:3d:90:7e:db:9e:7a:4f:6d:80:fb:fa: - dc:42:43:e2:dc:8f:6a:ec:18:c5:af:e1:ea:03:fd:66:78:a2: - 01:02:21:00:f7:86:58:81:17:f5:74:5b:14:c8:0f:93:e2:bb: - b8:e9:90:47:c0:f7:b1:60:82:d9:b4:1a:fc:fa:66:fa:48:5c + 30:46:02:21:00:f8:72:3d:90:7e:db:9e:7a:4f:6d:80:fb:fa: + dc:42:43:e2:dc:8f:6a:ec:18:c5:af:e1:ea:03:fd:66:78:a2: + 01:02:21:00:f7:86:58:81:17:f5:74:5b:14:c8:0f:93:e2:bb: + b8:e9:90:47:c0:f7:b1:60:82:d9:b4:1a:fc:fa:66:fa:48:5c ``` @@ -224,10 +224,10 @@ Certificate: X509v3 Subject Alternative Name: DNS:mail.example.test Signature Algorithm: ECDSA-SHA256 - 30:46:02:21:00:ad:08:7b:f0:82:41:2e:0e:cd:2b:f7:95:fd: - ee:73:d9:93:8d:74:7c:ef:29:4d:d5:da:33:04:f0:b6:b1:6b: - 13:02:21:00:d7:f1:95:db:be:18:b8:db:77:b9:57:07:e6:b9: - 5a:3d:00:34:d3:f5:eb:18:67:9b:ba:bf:88:62:72:e9:c9:99 + 30:46:02:21:00:ad:08:7b:f0:82:41:2e:0e:cd:2b:f7:95:fd: + ee:73:d9:93:8d:74:7c:ef:29:4d:d5:da:33:04:f0:b6:b1:6b: + 13:02:21:00:d7:f1:95:db:be:18:b8:db:77:b9:57:07:e6:b9: + 5a:3d:00:34:d3:f5:eb:18:67:9b:ba:bf:88:62:72:e9:c9:99 ``` @@ -268,10 +268,10 @@ Certificate: X509v3 Subject Key Identifier: DE:90:B3:B9:4D:C1:B3:EE:77:00:88:8B:69:EC:71:C4:30:F9:F6:7F Signature Algorithm: ECDSA-SHA256 - 30:44:02:20:3f:3b:90:e7:ca:82:70:8e:3f:2e:72:2a:b9:27: - 46:ac:e9:e2:4a:db:56:02:bc:a2:b2:99:e4:8d:10:7a:d5:73: - 02:20:72:25:64:b6:1c:aa:a6:c3:14:e1:66:35:bf:a1:db:90: - ea:49:59:f9:44:e8:63:de:a8:c0:bb:9b:21:08:59:87 + 30:44:02:20:3f:3b:90:e7:ca:82:70:8e:3f:2e:72:2a:b9:27: + 46:ac:e9:e2:4a:db:56:02:bc:a2:b2:99:e4:8d:10:7a:d5:73: + 02:20:72:25:64:b6:1c:aa:a6:c3:14:e1:66:35:bf:a1:db:90: + ea:49:59:f9:44:e8:63:de:a8:c0:bb:9b:21:08:59:87 ``` @@ -337,10 +337,10 @@ Certificate: X509v3 Subject Alternative Name: DNS:*.example.test Signature Algorithm: ECDSA-SHA256 - 30:46:02:21:00:f2:50:c0:b5:c9:24:e5:e9:36:a6:7b:35:5d: - 38:a7:7d:81:af:02:fc:9d:fd:79:f4:2d:4c:8a:04:55:44:a8: - 3a:02:21:00:b1:2d:d2:25:18:2d:35:19:20:97:78:f1:d5:18: - 9f:11:d5:97:a9:dc:64:95:2a:6c:9d:4e:78:69:c1:92:23:23 + 30:46:02:21:00:f2:50:c0:b5:c9:24:e5:e9:36:a6:7b:35:5d: + 38:a7:7d:81:af:02:fc:9d:fd:79:f4:2d:4c:8a:04:55:44:a8: + 3a:02:21:00:b1:2d:d2:25:18:2d:35:19:20:97:78:f1:d5:18: + 9f:11:d5:97:a9:dc:64:95:2a:6c:9d:4e:78:69:c1:92:23:23 ``` From eacc379cf12661a1795b382a667da39650154334 Mon Sep 17 00:00:00 2001 From: allddd <117767298+allddd@users.noreply.github.com> Date: Sun, 22 Oct 2023 15:16:41 +0200 Subject: [PATCH 044/125] feat: Postfix permit DSN (Delivery Status Notification) only on authenticated ports (465 + 587) (#3572) * add POSTFIX_DSN * add tests for POSTFIX_DSN * Revert "add POSTFIX_DSN" This reverts commit d5bd0e911737eb8b9efa9566caac6e95c6570af2. * discard DSN requests on unauthenticated ports * make tests work with overrides instead of ENV * Apply suggestions from code review * fix test inconsistencies --------- Co-authored-by: allddd Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 11 +++ target/postfix/main.cf | 1 + target/postfix/master.cf | 2 + test/config/dsn/postfix-main.cf | 1 + test/config/dsn/postfix-master.cf | 2 + .../email-templates/dsn-authenticated.txt | 14 +++ .../email-templates/dsn-unauthenticated.txt | 12 +++ test/tests/parallel/set3/mta/dsn.bats | 95 +++++++++++++++++++ 8 files changed, 138 insertions(+) create mode 100644 test/config/dsn/postfix-main.cf create mode 100644 test/config/dsn/postfix-master.cf create mode 100644 test/test-files/email-templates/dsn-authenticated.txt create mode 100644 test/test-files/email-templates/dsn-unauthenticated.txt create mode 100644 test/tests/parallel/set3/mta/dsn.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ce7b6fb..a2e15c77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ All notable changes to this project will be documented in this file. The format ### Breaking - The environment variable `ENABLE_LDAP=1` has been changed to `ACCOUNT_PROVISIONER=LDAP`. +- Postfix now defaults to supporting DSNs (_[Delivery Status Notifications](https://github.com/docker-mailserver/docker-mailserver/pull/3572#issuecomment-1751880574)_) only for authenticated users. This is a security measure to reduce spammer abuse of your DMS instance as a backscatter source. + - If you need to modify this change, please let us know by opening an issue / discussion. + - You can [opt-out (_enable DSNs_) via the `postfix-main.cf` override support](https://docker-mailserver.github.io/docker-mailserver/v12.1/config/advanced/override-defaults/postfix/) using the contents: `smtpd_discard_ehlo_keywords =`. + - Likewise for authenticated users, the submission(s) ports (465 + 587) are configured internally via `master.cf` to keep DSNs enabled (_since authentication protects from abuse_). + + If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: + + ``` + submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + ``` ### Added diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 518ad326..405dc0fb 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -54,6 +54,7 @@ smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_una smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, reject_unauth_pipelining, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination, reject_unauth_pipelining smtpd_sender_restrictions = $dms_smtpd_sender_restrictions +smtpd_discard_ehlo_keywords = silent-discard, dsn disable_vrfy_command = yes # Custom defined parameters for DMS: diff --git a/target/postfix/master.cf b/target/postfix/master.cf index 6f8877f6..e5b955a4 100644 --- a/target/postfix/master.cf +++ b/target/postfix/master.cf @@ -24,6 +24,7 @@ submission inet n - n - - smtpd -o smtpd_client_restrictions=permit_sasl_authenticated,reject -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o smtpd_sender_restrictions=$mua_sender_restrictions + -o smtpd_discard_ehlo_keywords= -o milter_macro_daemon_name=ORIGINATING -o cleanup_service_name=sender-cleanup @@ -37,6 +38,7 @@ submissions inet n - n - - smtpd -o smtpd_client_restrictions=permit_sasl_authenticated,reject -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o smtpd_sender_restrictions=$mua_sender_restrictions + -o smtpd_discard_ehlo_keywords= -o milter_macro_daemon_name=ORIGINATING -o cleanup_service_name=sender-cleanup diff --git a/test/config/dsn/postfix-main.cf b/test/config/dsn/postfix-main.cf new file mode 100644 index 00000000..1cb0db1e --- /dev/null +++ b/test/config/dsn/postfix-main.cf @@ -0,0 +1 @@ +smtpd_discard_ehlo_keywords = diff --git a/test/config/dsn/postfix-master.cf b/test/config/dsn/postfix-master.cf new file mode 100644 index 00000000..bb6aad15 --- /dev/null +++ b/test/config/dsn/postfix-master.cf @@ -0,0 +1,2 @@ +submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn +submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn diff --git a/test/test-files/email-templates/dsn-authenticated.txt b/test/test-files/email-templates/dsn-authenticated.txt new file mode 100644 index 00000000..c187bd67 --- /dev/null +++ b/test/test-files/email-templates/dsn-authenticated.txt @@ -0,0 +1,14 @@ +EHLO mail +AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu +bXlwYXNzd29yZA== +MAIL FROM: user1@localhost.localdomain +RCPT TO: user1@localhost.localdomain NOTIFY=success,failure +DATA +From: Existing Local User +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message +This is a test mail. + +. +QUIT diff --git a/test/test-files/email-templates/dsn-unauthenticated.txt b/test/test-files/email-templates/dsn-unauthenticated.txt new file mode 100644 index 00000000..8232ea68 --- /dev/null +++ b/test/test-files/email-templates/dsn-unauthenticated.txt @@ -0,0 +1,12 @@ +HELO mail.external.tld +MAIL FROM: user@external.tld +RCPT TO: user1@localhost.localdomain NOTIFY=success,failure +DATA +From: Docker Mail Server +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message +This is a test mail. + +. +QUIT diff --git a/test/tests/parallel/set3/mta/dsn.bats b/test/tests/parallel/set3/mta/dsn.bats new file mode 100644 index 00000000..dcbb79b6 --- /dev/null +++ b/test/tests/parallel/set3/mta/dsn.bats @@ -0,0 +1,95 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +BATS_TEST_NAME_PREFIX='[DSN] ' +CONTAINER1_NAME='dms-test_dsn_send_always' +CONTAINER2_NAME='dms-test_dsn_send_auth' +CONTAINER3_NAME='dms-test_dsn_send_none' +# A similar line is added to the log when a DSN (Delivery Status Notification) is sent: +# +# postfix/bounce[1023]: C943BA6B46: sender delivery status notification: DBF86A6B4CO +# +LOG_DSN='delivery status notification' + +function setup_file() { + local CUSTOM_SETUP_ARGUMENTS=( + # Required only for delivery via nc (_send_email) + --env PERMIT_DOCKER=container + ) + + export CONTAINER_NAME=${CONTAINER1_NAME} + _init_with_defaults + # Unset `smtpd_discard_ehlo_keywords` to allow DSNs by default on any `smtpd` service: + cp "${TEST_TMP_CONFIG}/dsn/postfix-main.cf" "${TEST_TMP_CONFIG}/postfix-main.cf" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_service postfix + _wait_for_smtp_port_in_container + + export CONTAINER_NAME=${CONTAINER2_NAME} + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_service postfix + _wait_for_smtp_port_in_container + + export CONTAINER_NAME=${CONTAINER3_NAME} + _init_with_defaults + # Mirror default main.cf (disable DSN on ports 465 + 587 too): + cp "${TEST_TMP_CONFIG}/dsn/postfix-master.cf" "${TEST_TMP_CONFIG}/postfix-master.cf" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_service postfix + _wait_for_smtp_port_in_container +} + +function teardown_file() { + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" "${CONTAINER3_NAME}" +} + +@test "should always send a DSN when requested" { + export CONTAINER_NAME=${CONTAINER1_NAME} + + _send_email 'email-templates/dsn-unauthenticated' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _wait_for_empty_mail_queue_in_container + + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + _should_output_number_of_lines 3 +} + +# Defaults test case +@test "should only send a DSN when requested from ports 465/587" { + export CONTAINER_NAME=${CONTAINER2_NAME} + + _send_email 'email-templates/dsn-unauthenticated' + _wait_for_empty_mail_queue_in_container + + # DSN requests can now only be made on ports 465 and 587, + # so grep should not find anything. + # + # Although external requests are discarded, anyone who has requested a DSN + # will still receive it, but it will come from the sending mail server, not this one. + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + assert_failure + + # These ports are excluded via master.cf. + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _wait_for_empty_mail_queue_in_container + + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + _should_output_number_of_lines 2 +} + +@test "should never send a DSN" { + export CONTAINER_NAME=${CONTAINER3_NAME} + + _send_email 'email-templates/dsn-unauthenticated' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _wait_for_empty_mail_queue_in_container + + # DSN requests are rejected regardless of origin. + # This is usually a bad idea, as you won't get them either. + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + assert_failure +} From eb7b1882e16b1c2076f2d5b77b3696fe17b27453 Mon Sep 17 00:00:00 2001 From: georglauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:42:01 +0200 Subject: [PATCH 045/125] update `contributors.yml` action --- .github/workflows/contributors.yml | 75 +- CONTRIBUTORS.md | 1885 +--------------------------- 2 files changed, 15 insertions(+), 1945 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 9028a60b..e784ae0e 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -10,12 +10,6 @@ permissions: pull-requests: write statuses: write -env: - # Assign commit authorship to official Github Actions bot: - GIT_USER: github-actions[bot] - GIT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com - BRANCH_NAME: contributors-update - jobs: add-contributors: name: 'Add Contributors' @@ -24,64 +18,15 @@ jobs: - name: 'Checkout' uses: actions/checkout@v4 - - name: 'Checkout New Branch and Push It' - run: | - git checkout -b ${{ env.BRANCH_NAME }} - git push --force https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git HEAD:${{ env.BRANCH_NAME }} - git checkout master - - # See https://github.com/marketplace/actions/auto-add-contributors for reference of the action. - # - # This action is not well documented, but it does the job for now. We pin the version in order - # to not have any issues in the future. - name: 'Update CONTRIBUTORS.md' - uses: BobAnkh/add-contributors@v0.2.2 + uses: akhilmhdh/contributors-readme-action@v2.3.6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - ACCESS_TOKEN: ${{secrets.GITHUB_TOKEN}} - BRANCH: ${{ env.BRANCH_NAME }} - COMMIT_MESSAGE: 'docs: update `CONTRIBUTORS.md`' - PATH: /CONTRIBUTORS.md - CONTRIBUTOR: '## Contributors' - COLUMN_PER_ROW: 6 - IMG_WIDTH: 100 - FONT_SIZE: 14 - AVATAR_SHAPE: round - - # See https://github.com/marketplace/actions/create-pull-request for reference of the action. - - name: 'Create Pull Request' - uses: peter-evans/create-pull-request@v5.0.2 - id: create-pr - with: - token: ${{ secrets.GITHUB_TOKEN }} - base: master - branch: ${{ env.BRANCH_NAME }} - title: 'docs: update `CONTRIBUTORS.md`' - commit-message: 'docs: update `CONTRIBUTORS.md`' - delete-branch: true - committer: ${{ env.GIT_USER }} <${{ env.GIT_EMAIL }}> - author: ${{ env.GIT_USER }} <${{ env.GIT_EMAIL }}> - signoff: true - body: | - Updated `CONTRIBUTORS.md` via the CI workflow: [`contributors.yml`][workflow]. - - [workflow]: https://github.com/docker-mailserver/docker-mailserver/blob/master/.github/workflows/contributors.yml - - # See https://github.com/marketplace/actions/set-commit-status for reference of the action. - # - # GH Actions are limited when it comes to actions triggering other actions. Hence, - # this whole workflow will not trigger a `pull_request` event without a PAT. The lint - # workflow, which is required due to branch protection, is not important for this type - # of PR, so we skip it and pretend it was successful. - - name: 'Set Status for Linting Actions to Success (Skipped)' - uses: myrotvorets/set-commit-status-action@v2.0.0 - continue-on-error: true - with: - token: ${{ secrets.GITHUB_TOKEN }} - # Skipped workflows are still assigned a "success" status: - status: success - # This should be the correct commit SHA on ${{ env.BRANCH_NAME }}: - sha: ${{ steps.create-pr.outputs.pull-request-head-sha }} - # Name of status check to add/update: - context: lint - # Optional message/note we can inline to the right of the context name in the UI: - description: Lint skipped. Not relevant. + readme_path: CONTRIBUTORS.md + collaborators: all + commit_message: 'docs: updated `CONTRIBUTORS.md`' + committer_username: github-actions[bot] + committer_email: 41898282+github-actions[bot]@users.noreply.github.com + pr_title_on_protected: 'docs: update `CONTRIBUTORS.md' + auto_detect_branch_protection: true diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index cbd2653b..08412f24 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,1880 +1,9 @@ +# Contributors + Thanks goes to these wonderful people ✨ -## Contributors - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - Thomas -
- Thomas VIAL - - 		- - Georg -
- Georg Lauterbach - - 		- - Casper/ -
- Casper - - 		- - Erik -
- Erik Wramner - - 		- - Brennan -
- Brennan Kinney - - 		- - Jean-Denis -
- Jean-Denis Vauguet - -
- - Martin -
- Martin Schulze - - 		- - Frederic -
- Frederic Werner - - 		- - Josef -
- Josef Friedrich - - 		- - Johan -
- Johan Smits - - 		- - youtous/ -
- youtous - - 		- - 17Halbe/ -
- 17Halbe - -
- - Nathan -
- Nathan Pierce - - 		- - Thorsten -
- Thorsten von Eicken - - 		- - Germain -
- Germain Masse - - 		- - 00angus/ -
- 00angus - - 		- - Paul -
- Paul Steinlechner - - 		- - Dominik -
- Dominik Winter - -
- - Paul -
- Paul Adams - - 		- - Felix -
- Felix Bartels - - 		- - Sebastian -
- Sebastian Wiesendahl - - 		- - Steve -
- Steve Johnson - - 		- - André -
- André Stein - - 		- - William -
- William Desportes - -
- - omarc1492/ -
- omarc1492 - - 		- - Christian -
- Christian Glahn - - 		- - Marek -
- Marek Walczak - - 		- - Kai -
- Kai Ren - - 		- - Kyle -
- Kyle Ondy - - 		- - Michael/ -
- Michael - -
- - lukas/ -
- lukas - - 		- - Sascha -
- Sascha Scandella - - 		- - Lukáš -
- Lukáš Vasek - - 		- - Andreas -
- Andreas Perhab - - 		- - vortex852456/ -
- vortex852456 - - 		- - Christian -
- Christian Grasso - -
- - Hans-Cees -
- Hans-Cees Speel - - 		- - Jack -
- Jack Pearson - - 		- - Dashamir -
- Dashamir Hoxha - - 		- - GAVARD -
- GAVARD Ewann - - 		- - Jack -
- Jack Twilley - - 		- - Luke -
- Luke Cyca - -
- - Oleg -
- Oleg Kainov - - 		- - Robert -
- Robert Dolca - - 		- - Thomas -
- Thomas Kilian - - 		- - Tobias -
- Tobias Rittig - - 		- - akmet/ -
- akmet - - 		- - Arne -
- Arne Kepp - -
- - Dennis -
- Dennis Stumm - - 		- - Moritz -
- Moritz Marquardt - - 		- - pyy/ -
- pyy - - 		- - Anne/ -
- Anne - - 		- - Birkenstab/ -
- Birkenstab - - 		- - Brandon -
- Brandon Schmitt - -
- - Cédric -
- Cédric Laubacher - - 		- - GrupoCITEC/ -
- GrupoCITEC - - 		- - Jairo -
- Jairo Llopis - - 		- - James/ -
- James - - 		- - Jarrod -
- Jarrod Smith - - 		- - Patrizio -
- Patrizio Bekerle - -
- - Rubytastic2/ -
- Rubytastic2 - - 		- - Semir -
- Semir Patel - - 		- - Wolfgang -
- Wolfgang Ocker - - 		- - Zehir/ -
- Zehir - - 		- - guardiande/ -
- guardiande - - 		- - kamuri/ -
- kamuri - -
- - davidszp/ -
- davidszp - - 		- - Andreas -
- Andreas Gerstmayr - - 		- - Marko -
- Marko J - - 		- - Michael -
- Michael Schmoock - - 		- - VanVan/ -
- VanVan - - 		- - Alexander -
- Alexander Elbracht - -
- - Amin -
- Amin Vakil - - 		- - Andrew -
- Andrew Low - - 		- - Ask -
- Ask Bjørn Hansen - - 		- - Ben/ -
- Ben - - 		- - Christian -
- Christian Raue - - 		- - Daniel -
- Daniel Panteleit - -
- - Darren -
- Darren McGrandle - - 		- - Dominik -
- Dominik Bruhn - - 		- - DuncanvR/ -
- DuncanvR - - 		- - Emanuele -
- Emanuele Mazzotta - - 		- - FL42/ -
- FL42 - - 		- - Guillaume -
- Guillaume Simon - -
- - Ikko -
- Ikko Eltociear Ashimine - - 		- - James -
- James Fryer - - 		- - Millaguie/ -
- Millaguie - - 		- - Jeremy -
- Jeremy Shipman - - 		- - Jonas -
- Jonas Kalderstam - - 		- - Louis/ -
- Louis - -
- - martinwepner/ -
- martinwepner - - 		- - Michael -
- Michael Als - - 		- - Morgan -
- Morgan Kesler - - 		- - Pablo -
- Pablo Castorino - - 		- - Philipp -
- Philipp Fruck - - 		- - Rainer -
- Rainer Rillke - -
- - Bob -
- Bob Gregor - - 		- - r-pufky/ -
- r-pufky - - 		- - andymel/ -
- andymel - - 		- - bigpigeon/ -
- bigpigeon - - 		- - engelant/ -
- engelant - - 		- - j-marz/ -
- j-marz - -
- - lokipo/ -
- lokipo - - 		- - msheakoski/ -
- msheakoski - - 		- - Felix/ -
- Felix - - 		- - Leon -
- Leon Busch-George - - 		- - Marius -
- Marius Panneck - - 		- - Thomas -
- Thomas Willems - -
- - 0xflotus/ -
- 0xflotus - - 		- - Ivan -
- Ivan Fokeev - - 		- - 20th/ -
- 20th - - 		- - 2b/ -
- 2b - - 		- - Max:/ -
- Max: - - 		- - Achim -
- Achim Christ - -
- - Adrian -
- Adrian Pistol - - 		- - Alexander -
- Alexander Kachkaev - - 		- - Alexander -
- Alexander Neu - - 		- - Bedniakov -
- Bedniakov Aleksei - - 		- - Andreas -
- Andreas Egli - - 		- - Andrew -
- Andrew Cornford - -
- - Andrey -
- Andrey Likhodievskiy - - 		- - Arash -
- Arash Fatahzade - - 		- - Arthur -
- Arthur Outhenin-Chalandre - - 		- - Astro/ -
- Astro - - 		- - Benedict -
- Benedict Endemann - - 		- - Bogdan/ -
- Bogdan - -
- - Charles -
- Charles Harris - - 		- - Christian -
- Christian Musa - - 		- - Christoph/ -
- Christoph - - 		- - Claus -
- Claus Beerta - - 		- - Damian -
- Damian Moore - - 		- - espitall/ -
- espitall - -
- - Daniel -
- Daniel Karski - - 		- - Daniele -
- Daniele Bellavista - - 		- - Daniël -
- Daniël van den Berg - - 		- - Dingoz/ -
- Dingoz - - 		- - Dmitry -
- Dmitry R. - - 		- - Dorian -
- Dorian Ayllón - -
- - Edmond -
- Edmond Varga - - 		- - Eduard -
- Eduard Knysh - - 		- - Elisei -
- Elisei Roca - - 		- - Erick -
- Erick Calder - - 		- - Erik -
- Erik Brakkee - - 		- - Huncode/ -
- Huncode - -
- - Florian/ -
- Florian - - 		- - Florian -
- Florian Roks - - 		- - Franz -
- Franz Keferböck - - 		- - Frugan/ -
- Frugan - - 		- - Gabriel -
- Gabriel Euzet - - 		- - Gabriel -
- Gabriel Landais - -
- - GiovanH/ -
- GiovanH - - 		- - H4R0/ -
- H4R0 - - 		- - Harry -
- Harry Youd - - 		- - Hugues -
- Hugues Granger - - 		- - Ian -
- Ian Andrews - - 		- - Influencer/ -
- Influencer - -
- - jcalfee/ -
- jcalfee - - 		- - JS -
- JS Légaré - - 		- - Jeidnx/ -
- Jeidnx - - 		- - JiLleON/ -
- JiLleON - - 		- - Jiří -
- Jiří Kozlovský - - 		- - jmccl/ -
- jmccl - -
- - Jurek -
- Jurek Barth - - 		- - JOnathan -
- JOnathan duMonT - - 		- - Kaan/ -
- Kaan - - 		- - Karthik -
- Karthik K - - 		- - KCrawley/ -
- KCrawley - - 		- - Khue -
- Khue Doan - -
- - Lars -
- Lars Pötter - - 		- - Leo -
- Leo Winter - - 		- - Lin -
- Lin Han - - 		- - MadsRC/ -
- MadsRC - - 		- - Mathieu -
- Mathieu Brunot - - 		- - Maximilian -
- Maximilian Hippler - -
- - Michael -
- Michael G. - - 		- - Michael -
- Michael Jensen - - 		- - Michel -
- Michel Albert - - 		- - Mohammed -
- Mohammed Chotia - - 		- - Mohammed -
- Mohammed Noureldin - - 		- - Moritz -
- Moritz Poldrack - -
- - Naveen/ -
- Naveen - - 		- - Nicholas -
- Nicholas Pepper - - 		- - Nick -
- Nick Pappas - - 		- - Nils -
- Nils Knappmeier - - 		- - Olivier -
- Olivier Picquenot - - 		- - Orville -
- Orville Q. Song - -
- - Ovidiu -
- Ovidiu Predescu - - 		- - Petar -
- Petar Šegina - - 		- - Peter -
- Peter Hartmann - - 		- - Pierre-Yves -
- Pierre-Yves Rofes - - 		- - Remo -
- Remo E - - 		- - René -
- René Plötz - -
- - Roman -
- Roman Seyffarth - - 		- - Sam -
- Sam Collins - - 		- - Scott -
- Scott Weldon - - 		- - Sebastian -
- Sebastian Straub - - 		- - Serge -
- Serge van den Boom - - 		- - Sergey -
- Sergey Nazaryev - -
- - Shyim/ -
- Shyim - - 		- - Simon -
- Simon J Mudd - - 		- - Simon -
- Simon Schröter - - 		- - Stephan/ -
- Stephan - - 		- - Stig -
- Stig Otnes Kolstad - - 		- - Sven -
- Sven Kauber - -
- - Sylvain -
- Sylvain Benner - - 		- - Sylvain -
- Sylvain Dumont - - 		- - TechnicLab/ -
- TechnicLab - - 		- - Thomas -
- Thomas Schmit - - 		- - Tin/ -
- Tin - - 		- - Torben -
- Torben Weibert - -
- - Toru -
- Toru Hisai - - 		- - Trangar/ -
- Trangar - - 		- - Twist235/ -
- Twist235 - - 		- - Vasiliy -
- Vasiliy Gokoyev - - 		- - Victoria -
- Victoria Brekenfeld - - 		- - Vilius/ -
- Vilius - -
- - Wim/ -
- Wim - - 		- - Y.C.Huang/ -
- Y.C.Huang - - 		- - arcaine2/ -
- arcaine2 - - 		- - awb99/ -
- awb99 - - 		- - brainkiller/ -
- brainkiller - - 		- - cternes/ -
- cternes - -
- - dborowy/ -
- dborowy - - 		- - dimalo/ -
- dimalo - - 		- - eleith/ -
- eleith - - 		- - helmutundarnold/ -
- helmutundarnold - - 		- - hnws/ -
- hnws - - 		- - i-C-o-d-e-r/ -
- i-C-o-d-e-r - -
- - idaadi/ -
- idaadi - - 		- - ixeft/ -
- ixeft - - 		- - jjtt/ -
- jjtt - - 		- - jose -
- jose nazario - - 		- - landergate/ -
- landergate - - 		- - magnus -
- magnus anderssen - -
- - marios88/ -
- marios88 - - 		- - matrixes/ -
- matrixes - - 		- - mchamplain/ -
- mchamplain - - 		- - Jason -
- Jason Miller - - 		- - mplx/ -
- mplx - - 		- - odinis/ -
- odinis - -
- - okami/ -
- okami - - 		- - olaf-mandel/ -
- olaf-mandel - - 		- - ontheair81/ -
- ontheair81 - - 		- - pravynandas/ -
- pravynandas - - 		- - presocratics/ -
- presocratics - - 		- - rhyst/ -
- rhyst - -
- - schnippl0r/ -
- schnippl0r - - 		- - smargold476/ -
- smargold476 - - 		- - sportshead/ -
- sportshead - - 		- - squash/ -
- squash - - 		- - strarsis/ -
- strarsis - - 		- - tamueller/ -
- tamueller - -
- - vivacarvajalito/ -
- vivacarvajalito - - 		- - wolkenschieber/ -
- wolkenschieber - - 		- - worldworm/ -
- worldworm - -
+ + ## Further Contributors @@ -1898,8 +27,4 @@ Also thanks goes to these wonderful people, that have contributed in various oth This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! -____ - -Note: - -We started using [all-contributors](https://github.com/all-contributors/all-contributors) in July 2021. We will add contributors with their future PRs or Issues. Code contributions are added automatically. If you are [one of the 200+](https://github.com/docker-mailserver/docker-mailserver/graphs/contributors) that contributed to the project in the past and would like to see your name here too, please reach out! +Note: We started using [all-contributors](https://github.com/all-contributors/all-contributors) in July 2021. We will add contributors with their future PRs or Issues. Code contributions are added automatically. If you are [one of the 200+](https://github.com/docker-mailserver/docker-mailserver/graphs/contributors) that contributed to the project in the past and would like to see your name here too, please reach out! From d988d8a8d1be8bd07c271eaa9911f15f5cd033c7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 22 Oct 2023 18:11:16 +0200 Subject: [PATCH 046/125] docs: updated `CONTRIBUTORS.md` (#3596) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 1937 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1937 insertions(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 08412f24..538f3740 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -3,6 +3,1943 @@ Thanks goes to these wonderful people ✨ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + casperklein +
+ Casper + + 		+ + fbartels +
+ Felix Bartels + + 		+ + NorseGaud +
+ Nathan Pierce + + 		+ + williamdes +
+ William Desportes + + 		+ + wernerfred +
+ Frederic Werner + + 		+ + georglauterbach +
+ Georg Lauterbach + +
+ + tomav +
+ Thomas VIAL + + 		+ + erik-wramner +
+ Erik Wramner + + 		+ + polarathene +
+ Brennan Kinney + + 		+ + chikamichi +
+ Jean-Denis Vauguet + + 		+ + martin-schulze-vireso +
+ Martin Schulze + + 		+ + Josef-Friedrich +
+ Josef Friedrich + +
+ + johansmitsnl +
+ Johan Smits + + 		+ + youtous +
+ Null + + 		+ + 17Halbe +
+ Null + + 		+ + tve +
+ Thorsten Von Eicken + + 		+ + gmasse +
+ Germain Masse + + 		+ + 00angus +
+ Null + +
+ + alinmear +
+ Paul Steinlechner + + 		+ + ap-wtioit +
+ Andreas Perhab + + 		+ + dominikwinter +
+ Dominik Winter + + 		+ + crazystick +
+ Paul Adams + + 		+ + swiesend +
+ Sebastian Wiesendahl + + 		+ + svenyonson +
+ Steve Johnson + +
+ + stonemaster +
+ André Stein + + 		+ + omarc1492 +
+ Null + + 		+ + phish108 +
+ Christian Glahn + + 		+ + mwlczk +
+ Marek Walczak + + 		+ + tyranron +
+ Kai Ren + + 		+ + KyleOndy +
+ Kyle Ondy + +
+ + MichaelSp +
+ Michael + + 		+ + mindrunner +
+ Lukas + + 		+ + m-a-v +
+ Sascha Scandella + + 		+ + bilak +
+ Lukáš Vasek + + 		+ + vortex852456 +
+ Null + + 		+ + chris54721 +
+ Christian Grasso + +
+ + hanscees +
+ Hans-Cees Speel + + 		+ + jrpear +
+ Jack Pearson + + 		+ + dashohoxha +
+ Dashamir Hoxha + + 		+ + egavard +
+ GAVARD Ewann + + 		+ + mathuin +
+ Jack Twilley + + 		+ + jamebus +
+ James + +
+ + lukecyca +
+ Luke Cyca + + 		+ + okainov +
+ Oleg Kainov + + 		+ + robertdolca +
+ Robert Dolca + + 		+ + kiliant +
+ Thomas Kilian + + 		+ + diiigle +
+ Tobias Rittig + + 		+ + akmet +
+ Akmet + +
+ + arneke +
+ Arne Kepp + + 		+ + dennis95stumm +
+ Dennis Stumm + + 		+ + moqmar +
+ Moritz Marquardt + + 		+ + pyy +
+ Null + + 		+ + voordev +
+ Anne + + 		+ + Birkenstab +
+ Null + +
+ + BrandonSchmitt +
+ Brandon Schmitt + + 		+ + Starbix +
+ Cédric Laubacher + + 		+ + citec +
+ GrupoCITEC + + 		+ + yajo +
+ Jairo Llopis + + 		+ + MakerMatrix +
+ Jarrod Smith + + 		+ + pbek +
+ Patrizio Bekerle + +
+ + Rubytastic2 +
+ Null + + 		+ + analogue +
+ Semir Patel + + 		+ + weo +
+ Wolfgang Ocker + + 		+ + Zehir +
+ Zehir + + 		+ + guardiande +
+ Null + + 		+ + kamuri +
+ Null + +
+ + davidszp +
+ Null + + 		+ + andreasgerstmayr +
+ Andreas Gerstmayr + + 		+ + mjung +
+ Marko J + + 		+ + m-schmoock +
+ Michael Schmoock + + 		+ + VanVan +
+ Null + + 		+ + elbracht +
+ Alexander Elbracht + +
+ + aminvakil +
+ Amin Vakil + + 		+ + andrewlow +
+ Andrew Low + + 		+ + abh +
+ Ask Bjørn Hansen + + 		+ + ubenmackin +
+ Ben + + 		+ + craue +
+ Christian Raue + + 		+ + danielpanteleit +
+ Daniel Panteleit + +
+ + dmcgrandle +
+ Darren McGrandle + + 		+ + theomega +
+ Dominik Bruhn + + 		+ + DuncanvR +
+ Null + + 		+ + emazzotta +
+ Emanuele Mazzotta + + 		+ + fl42 +
+ FL42 + + 		+ + ipernet +
+ Guillaume Simon + +
+ + H4R0 +
+ Null + + 		+ + eltociear +
+ Ikko Eltociear Ashimine + + 		+ + jamesfryer +
+ James Fryer + + 		+ + millaguie +
+ Millaguie + + 		+ + jedateach +
+ Jeremy Shipman + + 		+ + spacecowboy +
+ Jonas Kalderstam + +
+ + artonge +
+ Louis + + 		+ + martinwepner +
+ Null + + 		+ + nueaf +
+ Michael Als + + 		+ + keslerm +
+ Morgan Kesler + + 		+ + castorinop +
+ Pablo Castorino + + 		+ + p-fruck +
+ Philipp Fruck + +
+ + Rillke +
+ Rainer Rillke + + 		+ + reneploetz +
+ René Plötz + + 		+ + bobbravo2 +
+ Bob Gregor + + 		+ + r-pufky +
+ Robert Pufky + + 		+ + vincentDcmps +
+ Vincent Ducamps + + 		+ + andymel123 +
+ Andymel + +
+ + bigpigeon +
+ Bigpigeon + + 		+ + engelant +
+ Null + + 		+ + j-marz +
+ Null + + 		+ + lokipo +
+ Null + + 		+ + msheakoski +
+ Null + + 		+ + GoliathLabs +
+ Felix + +
+ + yogo1212 +
+ Leon Busch-George + + 		+ + mpanneck +
+ Marius Panneck + + 		+ + willtho89 +
+ Thomas Willems + + 		+ + tbutter +
+ Thomas Butter + + 		+ + 0xflotus +
+ 0xflotus + + 		+ + ifokeev +
+ Johan Fokeev + +
+ + 20th +
+ Null + + 		+ + 2b +
+ Null + + 		+ + askz +
+ Max: + + 		+ + acch +
+ Achim Christ + + 		+ + vifino +
+ Adrian Pistol + + 		+ + kachkaev +
+ Alexander Kachkaev + +
+ + alexanderneu +
+ Alexander Neu + + 		+ + ch3sh1r +
+ Bedniakov Aleksei + + 		+ + eglia +
+ Andreas Egli + + 		+ + groupmsl +
+ Andrew Cornford + + 		+ + green-anger +
+ Andrey Likhodievskiy + + 		+ + iRhonin +
+ Arash Fatahzade + +
+ + MrFreezeex +
+ Arthur Outhenin-Chalandre + + 		+ + arunvc +
+ Arun + + 		+ + astrocket +
+ Astro + + 		+ + baxerus +
+ Benedict Endemann + + 		+ + spock +
+ Bogdan + + 		+ + erdos4d +
+ Charles Harris + +
+ + crash7 +
+ Christian Musa + + 		+ + auchri +
+ Christoph + + 		+ + arkanovicz +
+ Claude Brisson + + 		+ + CBeerta +
+ Claus Beerta + + 		+ + damianmoore +
+ Damian Moore + + 		+ + espitall +
+ Null + +
+ + dkarski +
+ Daniel Karski + + 		+ + dbellavista +
+ Daniele Bellavista + + 		+ + danielvandenberg95 +
+ Daniël Van Den Berg + + 		+ + mlatorre31 +
+ Dingoz + + 		+ + mazzz1y +
+ Dmitry R. + + 		+ + aydodo +
+ Dorian Ayllón + +
+ + vedtam +
+ Edmond Varga + + 		+ + edvorg +
+ Eduard Knyshov + + 		+ + eliroca +
+ Elisei Roca + + 		+ + ekkis +
+ Erick Calder + + 		+ + ErikEngerd +
+ Erik Brakkee + + 		+ + huncode +
+ Huncode + +
+ + felixn +
+ Felix N + + 		+ + flole +
+ Florian + + 		+ + froks +
+ Florian Roks + + 		+ + fkefer +
+ Franz Keferböck + + 		+ + frugan-it +
+ Frugan + + 		+ + Marsu31 +
+ Gabriel Euzet + +
+ + glandais +
+ Gabriel Landais + + 		+ + GiovanH +
+ GiovanH + + 		+ + harryyoud +
+ Harry Youd + + 		+ + HeySora +
+ HeySora + + 		+ + sirgantrithon +
+ Ian Andrews + + 		+ + Influencer +
+ Influencer + +
+ + jcalfee +
+ Null + + 		+ + init-js +
+ JS Légaré + + 		+ + Jeidnx +
+ Jeidnx + + 		+ + JiLleON +
+ Null + + 		+ + jirislav +
+ Jiří Kozlovský + + 		+ + jmccl +
+ Null + +
+ + jurekbarth +
+ Jurek Barth + + 		+ + JOduMonT +
+ JOnathan DuMonT + + 		+ + Kaan88 +
+ Kaan + + 		+ + akkumar +
+ Karthik K + + 		+ + KCrawley +
+ Null + + 		+ + khuedoan +
+ Khue Doan + +
+ + JustAnother1 +
+ Lars Pötter + + 		+ + LeoWinterDE +
+ Leo Winter + + 		+ + linhandev +
+ Lin Han + + 		+ + luke- +
+ Lucas Bartholemy + + 		+ + LucidityCrash +
+ Null + + 		+ + MadsRC +
+ MadsRC + +
+ + madmath03 +
+ Mathieu Brunot + + 		+ + maxemann96 +
+ Maximilian Hippler + + 		+ + dragetd +
+ Michael G. + + 		+ + michaeljensen +
+ Michael Jensen + + 		+ + exhuma +
+ Michel Albert + + 		+ + milas +
+ Milas Bowman + +
+ + mcchots +
+ Mohammed Chotia + + 		+ + MohammedNoureldin +
+ Mohammed Noureldin + + 		+ + mpldr +
+ Moritz Poldrack + + 		+ + naveensrinivasan +
+ Naveen + + 		+ + neuralp +
+ Nicholas Pepper + + 		+ + radicand +
+ Nick Pappas + +
+ + nilshoell +
+ Nils Höll + + 		+ + nknapp +
+ Nils Knappmeier + + 		+ + pcqnt +
+ Olivier Picquenot + + 		+ + OrvilleQ +
+ Orville Q. Song + + 		+ + ovidiucp +
+ Ovidiu Predescu + + 		+ + mrPjer +
+ Petar Šegina + +
+ + peter-hartmann +
+ Peter Hartmann + + 		+ + piwai +
+ Pierre-Yves Rofes + + 		+ + remoe +
+ Remo E + + 		+ + romansey +
+ Roman Seyffarth + + 		+ + MightySCollins +
+ Sam Collins + + 		+ + 501st-alpha1 +
+ Scott Weldon + +
+ + klamann +
+ Sebastian Straub + + 		+ + svdb0 +
+ Serge Van Den Boom + + 		+ + 3ap +
+ Sergey Nazaryev + + 		+ + shyim +
+ Shyim + + 		+ + sjmudd +
+ Simon J Mudd + + 		+ + simonsystem +
+ Simon Schröter + +
+ + stephan-devop +
+ Stephan + + 		+ + stigok +
+ Stig Otnes Kolstad + + 		+ + 5ven +
+ Sven Kauber + + 		+ + syl20bnr +
+ Sylvain Benner + + 		+ + sylvaindumont +
+ Sylvain Dumont + + 		+ + TechnicLab +
+ Null + +
+ + thomasschmit +
+ Thomas Schmit + + 		+ + Thiritin +
+ Tin + + 		+ + tweibert +
+ Torben Weibert + + 		+ + torus +
+ Toru Hisai + + 		+ + VictorKoenders +
+ Trangar + + 		+ + Twist235 +
+ Null + +
+ + k3it +
+ Vasiliy Gokoyev + + 		+ + Drakulix +
+ Victoria Brekenfeld + + 		+ + vilisas +
+ Vilius + + 		+ + 42wim +
+ Wim + + 		+ + ShiriNmi1520 +
+ Y.C.Huang + + 		+ + allddd +
+ Allddd + +
+ + arcaine2 +
+ Null + + 		+ + awb99 +
+ Awb99 + + 		+ + brainkiller +
+ Null + + 		+ + cternes +
+ Null + + 		+ + dborowy +
+ Null + + 		+ + dimalo +
+ Null + +
+ + eleith +
+ Eleith + + 		+ + ghnp5 +
+ Null + + 		+ + helmutundarnold +
+ Null + + 		+ + hnws +
+ Null + + 		+ + i-C-o-d-e-r +
+ Null + + 		+ + idaadi +
+ Null + +
+ + ixeft +
+ Null + + 		+ + jjtt +
+ Null + + 		+ + paralax +
+ Jose Nazario + + 		+ + jpduyx +
+ Null + + 		+ + landergate +
+ Null + + 		+ + callmemagnus +
+ Magnus Anderssen + +
+ + marios88 +
+ Null + + 		+ + matrixes +
+ Null + + 		+ + mchamplain +
+ Mchamplain + + 		+ + millerjason +
+ Jason Miller + + 		+ + mplx +
+ Null + + 		+ + odinis +
+ Null + +
+ + okamidash +
+ Okami + + 		+ + olaf-mandel +
+ Null + + 		+ + ontheair81 +
+ Null + + 		+ + pravynandas +
+ Null + + 		+ + presocratics +
+ Null + + 		+ + rhyst +
+ Null + +
+ + rmlhuk +
+ Null + + 		+ + rriski +
+ Null + + 		+ + schnippl0r +
+ Null + + 		+ + smargold476 +
+ Null + + 		+ + sportshead +
+ Null + + 		+ + squash +
+ Null + +
+ + strarsis +
+ Null + + 		+ + tamueller +
+ Null + + 		+ + vivacarvajalito +
+ Null + + 		+ + wligtenberg +
+ Null + + 		+ + wolkenschieber +
+ Null + + 		+ + worldworm +
+ Null + +
## Further Contributors From cb62ce20e6f49d211a8f264b11feaed65582a338 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:31:22 +0200 Subject: [PATCH 047/125] bugfix: change Rspamd DKIM default config location (#3597) Instead of using `etc/rspamd/override.d/dkim_signing.conf`, we will now be using `/tmp/docker-mailserver/rspamd/override.d/dkim_signing.conf`. The new location is persisted (and linked again during startup) and hence better suited. --- target/bin/rspamd-dkim | 57 ++++++++++++++----- .../startup/setup.d/security/rspamd.sh | 3 +- .../parallel/set1/spam_virus/rspamd_dkim.bats | 6 +- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index de7129de..576c55ab 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -138,24 +138,55 @@ function _parse_arguments() { shift 2 done + return 0 +} + +function _preflight_checks() { if [[ ${KEYTYPE} == 'ed25519' ]] && [[ ${KEYSIZE} -ne 2048 ]]; then _exit_with_error "Chosen keytype does not accept the 'keysize' argument" fi - return 0 + if [[ ! -d /tmp/docker-mailserver ]]; then + _log 'warn' "The directory '/tmp/docker-mailserver' does not seem to be mounted by a volume - the Rspamd (DKIM) configuration will not be persisted" + fi + + # Note: Variables not marked with `local` are used + # in other functions (after this function was called). + # Also keep in sync with: target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks + local RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + local RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" + readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + + mkdir -p "${RSPAMD_DMS_DKIM_D}" "${RSPAMD_DMS_OVERRIDE_D}" + chown _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" + + # Mimmick target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks where + # ${RSPAMD_OVERRIDE_D} is linked to ${RSPAMD_DMS_OVERRIDE_D}, but not if + # + # 1. ${RSPAMD_OVERRIDE_D} has already been linked to ${RSPAMD_DMS_OVERRIDE_D} + # 2. ${RSPAMD_OVERRIDE_D} has contents already + # + # If 1. is true, then we're good since DMS' default setup linked the directory already and we will save + # a persisted location in every case. If 1. is false, 2. should be false as well since by default, + # ${RSPAMD_OVERRIDE_D} has no contents - we're good as well. What should logically never happen is + # that 1. is false but 2. is true; this case is caught nevertheless and a warning is emitted. + if [[ ! -h "${RSPAMD_OVERRIDE_D}" ]]; then + if rmdir "${RSPAMD_OVERRIDE_D}" &>/dev/null; then + ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" + else + _log 'warn' "Could not link '${RSPAMD_OVERRIDE_D}' to '${RSPAMD_DMS_OVERRIDE_D}' (as '${RSPAMD_OVERRIDE_D}' does not appear to be empty, which is unexpected) - you will need to restart DMS for changes to take effect" + fi + fi } function _create_keys() { - # Note: Variables not marked with `local` are used - # in other functions (after this function was called). - BASE_DIR='/tmp/docker-mailserver/rspamd/dkim' - if [[ ${KEYTYPE} == 'rsa' ]]; then - local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${KEYSIZE}-${SELECTOR}-${DOMAIN}" + local BASE_FILE_NAME="${RSPAMD_DMS_DKIM_D}/${KEYTYPE}-${KEYSIZE}-${SELECTOR}-${DOMAIN}" KEYTYPE_OPTIONS=('-b' "${KEYSIZE}") _log 'info' "Creating DKIM keys of type '${KEYTYPE}' and length '${KEYSIZE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" else - local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${SELECTOR}-${DOMAIN}" + local BASE_FILE_NAME="${RSPAMD_DMS_DKIM_D}/${KEYTYPE}-${SELECTOR}-${DOMAIN}" KEYTYPE_OPTIONS=('-t' "${KEYTYPE}") _log 'info' "Creating DKIM keys of type '${KEYTYPE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" fi @@ -164,9 +195,6 @@ function _create_keys() { PUBLIC_KEY_DNS_FILE="${BASE_FILE_NAME}.public.dns.txt" PRIVATE_KEY_FILE="${BASE_FILE_NAME}.private.txt" - mkdir -p "${BASE_DIR}" - chown _rspamd:_rspamd "${BASE_DIR}" - # shellcheck disable=SC2310 if __do_as_rspamd_user rspamadm \ dkim_keygen \ @@ -186,8 +214,8 @@ function _create_keys() { function _check_permissions() { # shellcheck disable=SC2310 - if ! __do_as_rspamd_user ls "${BASE_DIR}" >/dev/null; then - _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to list files in the keys directory ('${BASE_DIR}') - Rspamd may experience permission errors later" + if ! __do_as_rspamd_user ls "${RSPAMD_DMS_DKIM_D}" >/dev/null; then + _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to list files in the keys directory ('${RSPAMD_DMS_DKIM_D}') - Rspamd may experience permission errors later" elif ! __do_as_rspamd_user cat "${PRIVATE_KEY_FILE}" >/dev/null; then _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to read the private key file - Rspamd may experience permission errors later" else @@ -196,11 +224,11 @@ function _check_permissions() { } function _setup_default_signing_conf() { - local DEFAULT_CONFIG_FILE='/etc/rspamd/override.d/dkim_signing.conf' + local DEFAULT_CONFIG_FILE="${RSPAMD_DMS_OVERRIDE_D}/dkim_signing.conf" if [[ -f ${DEFAULT_CONFIG_FILE} ]]; then _log 'debug' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default" else - _log 'info' "Supplying a default configuration ('${DEFAULT_CONFIG_FILE}')" + _log 'info' "Supplying a default configuration (to '${DEFAULT_CONFIG_FILE}')" cat >"${DEFAULT_CONFIG_FILE}" << EOF # documentation: https://rspamd.com/doc/modules/dkim_signing.html @@ -254,6 +282,7 @@ function _final_steps() { _obtain_hostname_and_domainname _require_n_parameters_or_print_usage 0 "${@}" _parse_arguments "${@}" +_preflight_checks _create_keys _check_permissions _setup_default_signing_conf diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 4199b077..38b83444 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -70,7 +70,7 @@ function __rspamd__run_early_setup_and_checks() { readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d/" + local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" readonly RSPAMD_DMS_OVERRIDE_D mkdir -p /var/lib/rspamd/ @@ -82,6 +82,7 @@ function __rspamd__run_early_setup_and_checks() { ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" else __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?; not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" + __rspamd__log 'warn' "Note that using '${RSPAMD_DMS_OVERRIDE_D}' and placing files manually in '${RSPAMD_OVERRIDE_D}' is not supported" fi fi diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index cd880e52..76c301ea 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -4,8 +4,8 @@ load "${REPOSITORY_ROOT}/test/helper/setup" BATS_TEST_NAME_PREFIX='[Rspamd] (DKIM) ' CONTAINER_NAME='dms-test_rspamd-dkim' -DOMAIN_NAME='fixed.com' -SIGNING_CONF_FILE='/etc/rspamd/override.d/dkim_signing.conf' +DOMAIN_NAME='example.test' +SIGNING_CONF_FILE='/tmp/docker-mailserver/rspamd/override.d/dkim_signing.conf' function setup_file() { _init_with_defaults @@ -59,7 +59,7 @@ function teardown_file() { _default_teardown ; } __create_key assert_success __log_is_free_of_warnings_and_errors - assert_output --partial "Supplying a default configuration ('${SIGNING_CONF_FILE}')" + assert_output --partial "Supplying a default configuration (to '${SIGNING_CONF_FILE}')" refute_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" assert_output --partial "Finished DKIM key creation" _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" From 097dc6c9a4b48d5955c4d9c40c191bfb4e0ca29d Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Thu, 26 Oct 2023 02:22:36 +0200 Subject: [PATCH 048/125] docs(bin/setup): Add an example for an alias with multiple recipients (#3600) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- target/bin/addalias | 3 +++ 1 file changed, 3 insertions(+) diff --git a/target/bin/addalias b/target/bin/addalias index 60f8aa03..9bcb6cec 100755 --- a/target/bin/addalias +++ b/target/bin/addalias @@ -33,6 +33,9 @@ ${ORANGE}EXAMPLES${RESET} ${LWHITE}./setup.sh alias add alias@example.com recipient@example.com${RESET} Add the alias 'alias@example.com' for the mail account 'recipient@example.com'. + ${LWHITE}./setup.sh alias add alias@example.com 'recipient@example.com, another-recipient@example.com'${RESET} + Multiple recipients are separated by comma. + ${ORANGE}EXIT STATUS${RESET} Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain errors, the script will exit early with exit status 1. From 5efd2497864ab9bcf185b7901503786054b4a791 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 29 Oct 2023 17:04:07 +0100 Subject: [PATCH 049/125] docs: updated `CONTRIBUTORS.md` (#3606) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 110 ++++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 538f3740..615cc80e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1201,14 +1201,21 @@ Thanks goes to these wonderful people ✨ Jiří Kozlovský + + + jsonn +
+ Joerg Sonnenberger + + + jmccl
Null - - + jurekbarth @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
Null - + + khuedoan
Khue Doan - - + JustAnother1 @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
Null - + + MadsRC
MadsRC - - + madmath03 @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
Michel Albert - + + milas
Milas Bowman - - + mcchots @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
Nicholas Pepper - + + radicand
Nick Pappas - - + nilshoell @@ -1415,15 +1422,15 @@ Thanks goes to these wonderful people ✨
Ovidiu Predescu - + + mrPjer
Petar Šegina - - + peter-hartmann @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
Sam Collins - + + 501st-alpha1
Scott Weldon - - + klamann @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
Simon J Mudd - + + simonsystem
Simon Schröter - - + stephan-devop @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
Sylvain Dumont - + + TechnicLab
Null - - + thomasschmit @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
Trangar - + + Twist235
Null - - + k3it @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
Y.C.Huang - + + allddd
Allddd - - + arcaine2 @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
Null - + + dimalo
Null - - + eleith @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
Null - + + idaadi
Null - - + ixeft @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
Null - + + callmemagnus
Magnus Anderssen - - + marios88 @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
Null - + + odinis
Null - - + okamidash @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
Null - + + rhyst
Null - - + rmlhuk @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
Null - + + squash
Null - - + strarsis @@ -1931,7 +1938,8 @@ Thanks goes to these wonderful people ✨
Null - + + worldworm From f674232f7116acd1d5b2758c52ab5f5827d4d928 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:20:37 +0100 Subject: [PATCH 050/125] misc: final Rspamd adjustments for v13 (#3599) * outsource Rspamd ENVs into explicit helper This will allow us to uniformly source the helper and get the values from everywhere consistently. This is more than desirable since we will be using these values not only for the Rspamd setup, but also for DKIM management and during change-detection. * integrate Rspamd into changedetection We outsource one more function to reside in the helper script for Rspamd so that we can call this function from the Rspamd setup and from the changedetection functionality too. * realize deprecation of old commands file for Rspamd THIS IS A BREAKING CHANGE! This change realizes the log message: "Using old file location now (deprecated) - this will prevent startup in v13.0.0" Startup will now fail. * added '--force' option to Rspamd DKIM script * use new helper to get ENVs for Rspamd in DKIM script * remove the need for linking directories This was unnecessary, as explained in https://github.com/docker-mailserver/docker-mailserver/pull/3597#discussion_r1369413599 * Apply suggestions from code review review by @polarathene * apply more review feedback from @polarathene - - * update documentation --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 +- docs/content/config/security/rspamd.md | 4 +- target/bin/rspamd-dkim | 63 ++++----- target/scripts/check-for-changes.sh | 32 +++++ target/scripts/helpers/change-detection.sh | 6 + target/scripts/helpers/index.sh | 1 + target/scripts/helpers/rspamd.sh | 105 +++++++++++++++ .../startup/setup.d/security/rspamd.sh | 123 ++---------------- .../parallel/set1/spam_virus/rspamd_dkim.bats | 18 ++- .../parallel/set1/spam_virus/rspamd_full.bats | 5 +- 10 files changed, 205 insertions(+), 156 deletions(-) create mode 100644 target/scripts/helpers/rspamd.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index a2e15c77..0b875e2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,13 @@ All notable changes to this project will be documented in this file. The format If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: - ``` + ```cf submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn ``` +- using the old path for the Rspamd custom commands file (`/tmp/docker-mailserver/rspamd-modules.conf`), which was deprecated, will now prevent startup; use `/tmp/docker-mailserver/rspamd/custom-commands.conf` instead + ### Added - New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 94be2db9..5867b1e8 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -107,11 +107,11 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo !!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" -If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/` (a directory that is linked to `/etc/rspamd/override.d/`, if it exists) to override Rspamd and DMS default settings. This directory will not do a complete file override, but a [forced override of the specific settings in that file][rspamd-docs-override-dir]. +If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. !!! warning "Clashing Overrides" - Note that when also [using the `rspamd-commands` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. + Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. [rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 576c55ab..357c2a9c 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -27,9 +27,10 @@ ${ORANGE}DESCRIPTION${RESET} ${ORANGE}OPTIONS${RESET} ${BLUE}Generic Program Information${RESET} - -v Enable verbose logging (setting the log level to 'debug'). - -vv Enable very verbose logging (setting the log level to 'trace'). - help Print the usage information. + -f | --force Overwrite existing files if there are any + -v Enable verbose logging (setting the log level to 'debug'). + -vv Enable very verbose logging (setting the log level to 'trace'). + help Print the usage information. ${BLUE}Configuration adjustments${RESET} keytype Set the type of key you want to use. @@ -69,6 +70,7 @@ function __do_as_rspamd_user() { } function _parse_arguments() { + FORCE=0 KEYTYPE='rsa' KEYSIZE='2048' SELECTOR='mail' @@ -112,6 +114,12 @@ function _parse_arguments() { exit 0 ;; + ( '-f' | '--force' ) + FORCE=1 + shift 1 + continue + ;; + ( '-vv' ) # shellcheck disable=SC2034 LOG_LEVEL='trace' @@ -132,7 +140,6 @@ function _parse_arguments() { __usage _exit_with_error "Unknown option(s) '${1}' ${2:+"and '${2}'"}" ;; - esac shift 2 @@ -150,34 +157,10 @@ function _preflight_checks() { _log 'warn' "The directory '/tmp/docker-mailserver' does not seem to be mounted by a volume - the Rspamd (DKIM) configuration will not be persisted" fi - # Note: Variables not marked with `local` are used - # in other functions (after this function was called). - # Also keep in sync with: target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks - local RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - local RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" - readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + _rspamd_get_envs mkdir -p "${RSPAMD_DMS_DKIM_D}" "${RSPAMD_DMS_OVERRIDE_D}" chown _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" - - # Mimmick target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks where - # ${RSPAMD_OVERRIDE_D} is linked to ${RSPAMD_DMS_OVERRIDE_D}, but not if - # - # 1. ${RSPAMD_OVERRIDE_D} has already been linked to ${RSPAMD_DMS_OVERRIDE_D} - # 2. ${RSPAMD_OVERRIDE_D} has contents already - # - # If 1. is true, then we're good since DMS' default setup linked the directory already and we will save - # a persisted location in every case. If 1. is false, 2. should be false as well since by default, - # ${RSPAMD_OVERRIDE_D} has no contents - we're good as well. What should logically never happen is - # that 1. is false but 2. is true; this case is caught nevertheless and a warning is emitted. - if [[ ! -h "${RSPAMD_OVERRIDE_D}" ]]; then - if rmdir "${RSPAMD_OVERRIDE_D}" &>/dev/null; then - ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" - else - _log 'warn' "Could not link '${RSPAMD_OVERRIDE_D}' to '${RSPAMD_DMS_OVERRIDE_D}' (as '${RSPAMD_OVERRIDE_D}' does not appear to be empty, which is unexpected) - you will need to restart DMS for changes to take effect" - fi - fi } function _create_keys() { @@ -195,6 +178,16 @@ function _create_keys() { PUBLIC_KEY_DNS_FILE="${BASE_FILE_NAME}.public.dns.txt" PRIVATE_KEY_FILE="${BASE_FILE_NAME}.private.txt" + if [[ -f ${PUBLIC_KEY_FILE} ]] || [[ -f ${PUBLIC_KEY_DNS_FILE} ]] || [[ -f ${PRIVATE_KEY_FILE} ]]; then + if [[ ${FORCE} -eq 0 ]]; then + _log 'error' "Not overwriting existing files (use '--force' to overwrite existing files)" + exit 1 + else + _log 'info' "Overwriting existing files as the '--force' option was supplied" + rm "${PUBLIC_KEY_FILE}" "${PUBLIC_KEY_DNS_FILE}" "${PRIVATE_KEY_FILE}" + fi + fi + # shellcheck disable=SC2310 if __do_as_rspamd_user rspamadm \ dkim_keygen \ @@ -226,7 +219,7 @@ function _check_permissions() { function _setup_default_signing_conf() { local DEFAULT_CONFIG_FILE="${RSPAMD_DMS_OVERRIDE_D}/dkim_signing.conf" if [[ -f ${DEFAULT_CONFIG_FILE} ]]; then - _log 'debug' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default" + _log 'info' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default ('--force' does not overwrite this file, manual adjustment required)" else _log 'info' "Supplying a default configuration (to '${DEFAULT_CONFIG_FILE}')" cat >"${DEFAULT_CONFIG_FILE}" << EOF @@ -253,7 +246,15 @@ domain { } EOF - chown _rspamd:_rspamd "${DEFAULT_CONFIG_FILE}" + + # We copy here immediately in order to not rely on the changedetector - this way, users + # can immediately use the new keys. The file should not already exist in ${RSPAMD_OVERRIDE_D} + # since it would have been copied already. + cp "${DEFAULT_CONFIG_FILE}" "${RSPAMD_OVERRIDE_D}/dkim_signing.conf" + chown _rspamd:_rspamd "${DEFAULT_CONFIG_FILE}" "${RSPAMD_OVERRIDE_D}/dkim_signing.conf" + + _log 'debug' 'Restarting Rspamd as initial DKIM configuration was suppplied' + supervisorctl restart rspamd fi } diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index a8cdeb68..66417c09 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -21,6 +21,10 @@ source /etc/dms-settings # usage with DMS_HOSTNAME, which should remove the need to call this: _obtain_hostname_and_domainname +# This is a helper to properly set all Rspamd-related environment variables +# correctly and in one place. +_rspamd_get_envs + # verify checksum file exists; must be prepared by start-mailserver.sh if [[ ! -f ${CHKSUM_FILE} ]]; then _exit_with_error "'${CHKSUM_FILE}' is missing" 0 @@ -49,6 +53,7 @@ function _check_for_changes() { # Handle any changes _ssl_changes _postfix_dovecot_changes + _rspamd_changes _log_with_date 'debug' 'Reloading services due to detected changes' @@ -174,6 +179,33 @@ function _ssl_changes() { # They presently have no special handling other than to trigger a change that will restart Postfix/Dovecot. } +function _rspamd_changes() { + # RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + if [[ ${CHANGED} =~ ${RSPAMD_DMS_D}/.* ]]; then + + # "${RSPAMD_DMS_D}/override.d" + if [[ ${CHANGED} =~ ${RSPAMD_DMS_OVERRIDE_D}/.* ]]; then + _log_with_date 'trace' 'Rspamd - Copying configuration overrides' + rm "${RSPAMD_OVERRIDE_D}"/* + cp "${RSPAMD_DMS_OVERRIDE_D}"/* "${RSPAMD_OVERRIDE_D}" + fi + + # "${RSPAMD_DMS_D}/custom-commands.conf" + if [[ ${CHANGED} =~ ${RSPAMD_DMS_CUSTOM_COMMANDS_F} ]]; then + _log_with_date 'trace' 'Rspamd - Generating new configuration from custom commands' + _rspamd_handle_user_modules_adjustments + fi + + # "${RSPAMD_DMS_D}/dkim" + if [[ ${CHANGED} =~ ${RSPAMD_DMS_DKIM_D} ]]; then + _log_with_date 'trace' 'Rspamd - DKIM files updated' + fi + + _log_with_date 'debug' 'Rspamd configuration has changed - restarting service' + supervisorctl restart rspamd + fi +} + while true; do _check_for_changes sleep 2 diff --git a/target/scripts/helpers/change-detection.sh b/target/scripts/helpers/change-detection.sh index e396bfe2..08f6906c 100644 --- a/target/scripts/helpers/change-detection.sh +++ b/target/scripts/helpers/change-detection.sh @@ -40,6 +40,12 @@ function _monitored_files_checksums() { "${DMS_DIR}/dovecot-quotas.cf" "${DMS_DIR}/dovecot-masters.cf" ) + + # Check whether Rspamd is used and if so, monitor it's changes as well + if [[ ${ENABLE_RSPAMD} -eq 1 ]] && [[ -d ${RSPAMD_DMS_D} ]]; then + readarray -d '' STAGING_FILES_RSPAMD < <(find "${RSPAMD_DMS_D}" -type f -name "*.sh" -print0) + STAGING_FILES+=("${STAGING_FILES_RSPAMD[@]}") + fi fi # SSL certs: diff --git a/target/scripts/helpers/index.sh b/target/scripts/helpers/index.sh index 8d876739..0d77c9c4 100644 --- a/target/scripts/helpers/index.sh +++ b/target/scripts/helpers/index.sh @@ -16,6 +16,7 @@ function _import_scripts() { source "${PATH_TO_SCRIPTS}/network.sh" source "${PATH_TO_SCRIPTS}/postfix.sh" source "${PATH_TO_SCRIPTS}/relay.sh" + source "${PATH_TO_SCRIPTS}/rspamd.sh" source "${PATH_TO_SCRIPTS}/ssl.sh" source "${PATH_TO_SCRIPTS}/utils.sh" diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh new file mode 100644 index 00000000..868e3d3a --- /dev/null +++ b/target/scripts/helpers/rspamd.sh @@ -0,0 +1,105 @@ +#! /bin/bash + +# shellcheck disable=SC2034 # VAR appears unused. + +function _rspamd_get_envs() { + readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' + readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + + readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" + readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + + readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" +} + +# Parses `RSPAMD_DMS_CUSTOM_COMMANDS_F` and executed the directives given by the file. +# To get a detailed explanation of the commands and how the file works, visit +# https://docker-mailserver.github.io/docker-mailserver/latest/config/security/rspamd/#with-the-help-of-a-custom-file +function _rspamd_handle_user_modules_adjustments() { + # Adds an option with a corresponding value to a module, or, in case the option + # is already present, overwrites it. + # + # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ + # @param ${2} = module name as it should appear in the log + # @param ${3} = option name in the module + # @param ${4} = value of the option + # + # ## Note + # + # While this function is currently bound to the scope of `_rspamd_handle_user_modules_adjustments`, + # it is written in a versatile way (taking 4 arguments instead of assuming `ARGUMENT2` / `ARGUMENT3` + # are set) so that it may be used elsewhere if needed. + function __add_or_replace() { + local MODULE_FILE=${1:?Module file name must be provided} + local MODULE_LOG_NAME=${2:?Module log name must be provided} + local OPTION=${3:?Option name must be provided} + local VALUE=${4:?Value belonging to an option must be provided} + # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty) + VALUE=${VALUE% } + local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" + + readonly MODULE_FILE MODULE_LOG_NAME OPTION VALUE FILE + + [[ -f ${FILE} ]] || touch "${FILE}" + + if grep -q -E "${OPTION}.*=.*" "${FILE}"; then + __rspamd__log 'trace' "Overwriting option '${OPTION}' with value '${VALUE}' for ${MODULE_LOG_NAME}" + sed -i -E "s|([[:space:]]*${OPTION}).*|\1 = ${VALUE};|g" "${FILE}" + else + __rspamd__log 'trace' "Setting option '${OPTION}' for ${MODULE_LOG_NAME} to '${VALUE}'" + echo "${OPTION} = ${VALUE};" >>"${FILE}" + fi + } + + # We check for usage of the previous location of the commands file. + # TODO This can be removed after the release of v14.0.0. + local RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD="${RSPAMD_DMS_D}-modules.conf" + readonly RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD + if [[ -f ${RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD} ]]; then + _dms_panic__general "Old custom command file location '${RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD}' is deprecated (use '${RSPAMD_DMS_CUSTOM_COMMANDS_F}' now)" 'Rspamd setup' + fi + + if [[ -f "${RSPAMD_DMS_CUSTOM_COMMANDS_F}" ]]; then + __rspamd__log 'debug' "Found file '${RSPAMD_DMS_CUSTOM_COMMANDS_F}' - parsing and applying it" + + local COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 + while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do + case "${COMMAND}" in + ('disable-module') + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' + ;; + + ('enable-module') + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'true' 'override' + ;; + + ('set-option-for-module') + __add_or_replace "${ARGUMENT1}.conf" "module '${ARGUMENT1}'" "${ARGUMENT2}" "${ARGUMENT3}" + ;; + + ('set-option-for-controller') + __add_or_replace 'worker-controller.inc' 'controller worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" + ;; + + ('set-option-for-proxy') + __add_or_replace 'worker-proxy.inc' 'proxy worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" + ;; + + ('set-common-option') + __add_or_replace 'options.inc' 'common options' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" + ;; + + ('add-line') + __rspamd__log 'trace' "Adding complete line to '${ARGUMENT1}'" + echo "${ARGUMENT2}${ARGUMENT3+ ${ARGUMENT3}}" >>"${RSPAMD_OVERRIDE_D}/${ARGUMENT1}" + ;; + + (*) + __rspamd__log 'warn' "Command '${COMMAND}' is invalid" + continue + ;; + esac + done < <(_get_valid_lines_from_file "${RSPAMD_DMS_CUSTOM_COMMANDS_F}") + fi +} diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 38b83444..19ce75dc 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -1,12 +1,17 @@ #!/bin/bash -# Function called during global setup to handle the complete setup of Rspamd. +# This file is executed during startup of DMS. Hence, the `index.sh` helper has already +# been sourced, and thus, all helper functions from `rspamd.sh` are available. + +# Function called during global setup to handle the complete setup of Rspamd. Functions +# with a single `_` prefix are sourced from the `rspamd.sh` helper. function _setup_rspamd() { if _env_var_expect_zero_or_one 'ENABLE_RSPAMD' && [[ ${ENABLE_RSPAMD} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Rspamd' __rspamd__log 'trace' '---------- Setup started ----------' - __rspamd__run_early_setup_and_checks # must run first + _rspamd_get_envs # must run first + __rspamd__run_early_setup_and_checks # must run second __rspamd__setup_logfile __rspamd__setup_redis __rspamd__setup_postfix @@ -16,7 +21,7 @@ function _setup_rspamd() { __rspamd__setup_greylisting __rspamd__setup_hfilter_group __rspamd__setup_check_authenticated - __rspamd__handle_user_modules_adjustments # must run last + _rspamd_handle_user_modules_adjustments # must run last __rspamd__log 'trace' '---------- Setup finished ----------' else @@ -64,26 +69,11 @@ EOF # Run miscellaneous early setup tasks and checks, such as creating files needed at runtime # or checking for other anti-spam/anti-virus software. function __rspamd__run_early_setup_and_checks() { - # Note: Variables not marked with `local` are - # used in other functions as well. - readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' - readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - - local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" - readonly RSPAMD_DMS_OVERRIDE_D - mkdir -p /var/lib/rspamd/ : >/var/lib/rspamd/stats.ucl if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]]; then - __rspamd__log 'debug' "Found directory '${RSPAMD_DMS_OVERRIDE_D}' - linking it to '${RSPAMD_OVERRIDE_D}'" - if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null; then - ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" - else - __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?; not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" - __rspamd__log 'warn' "Note that using '${RSPAMD_DMS_OVERRIDE_D}' and placing files manually in '${RSPAMD_OVERRIDE_D}' is not supported" - fi + cp "${RSPAMD_DMS_OVERRIDE_D}"/* "${RSPAMD_OVERRIDE_D}" fi if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then @@ -304,98 +294,3 @@ function __rspamd__setup_check_authenticated() { "${MODULE_FILE}" fi } - -# Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. -# To get a detailed explanation of the commands and how the file works, visit -# https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file -function __rspamd__handle_user_modules_adjustments() { - # Adds an option with a corresponding value to a module, or, in case the option - # is already present, overwrites it. - # - # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ - # @param ${2} = module name as it should appear in the log - # @param ${3} = option name in the module - # @param ${4} = value of the option - # - # ## Note - # - # While this function is currently bound to the scope of `__rspamd__handle_user_modules_adjustments`, - # it is written in a versatile way (taking 4 arguments instead of assuming `ARGUMENT2` / `ARGUMENT3` - # are set) so that it may be used elsewhere if needed. - function __add_or_replace() { - local MODULE_FILE=${1:?Module file name must be provided} - local MODULE_LOG_NAME=${2:?Module log name must be provided} - local OPTION=${3:?Option name must be provided} - local VALUE=${4:?Value belonging to an option must be provided} - # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty) - VALUE=${VALUE% } - local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" - - readonly MODULE_FILE MODULE_LOG_NAME OPTION VALUE FILE - - [[ -f ${FILE} ]] || touch "${FILE}" - - if grep -q -E "${OPTION}.*=.*" "${FILE}"; then - __rspamd__log 'trace' "Overwriting option '${OPTION}' with value '${VALUE}' for ${MODULE_LOG_NAME}" - sed -i -E "s|([[:space:]]*${OPTION}).*|\1 = ${VALUE};|g" "${FILE}" - else - __rspamd__log 'trace' "Setting option '${OPTION}' for ${MODULE_LOG_NAME} to '${VALUE}'" - echo "${OPTION} = ${VALUE};" >>"${FILE}" - fi - } - - local RSPAMD_CUSTOM_COMMANDS_FILE="${RSPAMD_DMS_D}/custom-commands.conf" - local RSPAMD_CUSTOM_COMMANDS_FILE_OLD="${RSPAMD_DMS_D}-modules.conf" - readonly RSPAMD_CUSTOM_COMMANDS_FILE RSPAMD_CUSTOM_COMMANDS_FILE_OLD - - # We check for usage of the previous location of the commands file. - # This can be removed after the release of v14.0.0. - if [[ -f ${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} ]]; then - __rspamd__log 'warn' "Detected usage of old file location for modules adjustment ('${RSPAMD_CUSTOM_COMMANDS_FILE_OLD}') - please use the new location ('${RSPAMD_CUSTOM_COMMANDS_FILE}')" - __rspamd__log 'warn' "Using old file location now (deprecated) - this will prevent startup in v13.0.0" - RSPAMD_CUSTOM_COMMANDS_FILE=${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} - fi - - if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]]; then - __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" - - local COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 - while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do - case "${COMMAND}" in - ('disable-module') - __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' - ;; - - ('enable-module') - __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'true' 'override' - ;; - - ('set-option-for-module') - __add_or_replace "${ARGUMENT1}.conf" "module '${ARGUMENT1}'" "${ARGUMENT2}" "${ARGUMENT3}" - ;; - - ('set-option-for-controller') - __add_or_replace 'worker-controller.inc' 'controller worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" - ;; - - ('set-option-for-proxy') - __add_or_replace 'worker-proxy.inc' 'proxy worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" - ;; - - ('set-common-option') - __add_or_replace 'options.inc' 'common options' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" - ;; - - ('add-line') - __rspamd__log 'trace' "Adding complete line to '${ARGUMENT1}'" - echo "${ARGUMENT2}${ARGUMENT3+ ${ARGUMENT3}}" >>"${RSPAMD_OVERRIDE_D}/${ARGUMENT1}" - ;; - - (*) - __rspamd__log 'warn' "Command '${COMMAND}' is invalid" - continue - ;; - esac - done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") - fi -} diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 76c301ea..215b334d 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -68,8 +68,14 @@ function teardown_file() { _default_teardown ; } local INITIAL_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") __create_key + assert_failure + assert_output --partial "Not overwriting existing files (use '--force' to overwrite existing files)" + + # the same as before, but with the '--force' option + __create_key 'rsa' 'mail' "${DOMAIN_NAME}" '2048' '--force' __log_is_free_of_warnings_and_errors refute_output --partial "Supplying a default configuration ('${SIGNING_CONF_FILE}')" + assert_output --partial "Overwriting existing files as the '--force' option was supplied" assert_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" assert_output --partial "Finished DKIM key creation" local SECOND_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") @@ -188,11 +194,15 @@ function __create_key() { local SELECTOR=${2:-mail} local DOMAIN=${3:-${DOMAIN_NAME}} local KEYSIZE=${4:-2048} + local FORCE=${5:-} - _run_in_container setup config dkim \ - keytype "${KEYTYPE}" \ - keysize "${KEYSIZE}" \ - selector "${SELECTOR}" \ + # Not quoting is intended here as we would othewise provide + # the argument "''" (empty string), which would cause errors + # shellcheck disable=SC2086 + _run_in_container setup config dkim ${FORCE} \ + keytype "${KEYTYPE}" \ + keysize "${KEYSIZE}" \ + selector "${SELECTOR}" \ domain "${DOMAIN}" } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 3fbf59d2..09d42d46 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -66,12 +66,9 @@ function teardown_file() { _default_teardown ; } assert_output 'rspamd_milter = inet:localhost:11332' } -@test "'/etc/rspamd/override.d/' is linked correctly" { +@test "contents of '/etc/rspamd/override.d/' are copied" { local OVERRIDE_D='/etc/rspamd/override.d' - _run_in_container_bash "[[ -h ${OVERRIDE_D} ]]" - assert_success - _run_in_container_bash "[[ -f ${OVERRIDE_D}/testmodule_complicated.conf ]]" assert_success } From 8c0777b6691fbfc091c9c67d9eb525a45b2b5127 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:49:36 +0100 Subject: [PATCH 051/125] docs: improve docs about how to work with logs (#3626) * improvide docs about how to work with logs Most importantly, 1. I added information on the recently adopted `less` / `nano` 2. I added information about `/var/log/mail/` * fix typos * Apply suggestions from code review * Update docs/content/config/debugging.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/debugging.md | 25 ++++++++++++++++--------- docs/content/config/security/rspamd.md | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index a17fd5ff..24e41566 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -29,22 +29,29 @@ These links may advise how the provider can unblock the port through additional 1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. 2. **Use error logs as a search query**: Try [finding an _existing issue_][gh-issues] or _search engine result_ from any errors in your container log output. Often you'll find answers or more insights. If you still need to open an issue, sharing links from your search may help us assist you. The mail server log can be acquired by running `docker log ` (_or `docker logs -f ` if you want to follow the log_). -3. **Understand the basics of mail servers**: Especially for beginners, make sure you read our [Introduction][docs-introduction] and [Usage][docs-usage] articles. -4. **Search the whole FAQ**: Our [FAQ][docs-faq] contains answers for common problems. Make sure you go through the list. -5. **Reduce the scope**: Ensure that you can run a basic setup of DMS first. Then incrementally restore parts of your original configuration until the problem is reproduced again. If you're new to DMS, it is common to find the cause is misunderstanding how to configure a minimal setup. +3. **Inspect the logs of the service that is failing**: We provide a dedicated paragraph on this topic [further down below](#logs). +4. **Understand the basics of mail servers**: Especially for beginners, make sure you read our [Introduction][docs-introduction] and [Usage][docs-usage] articles. +5. **Search the whole FAQ**: Our [FAQ][docs-faq] contains answers for common problems. Make sure you go through the list. +6. **Reduce the scope**: Ensure that you can run a basic setup of DMS first. Then incrementally restore parts of your original configuration until the problem is reproduced again. If you're new to DMS, it is common to find the cause is misunderstanding how to configure a minimal setup. ### Debug a running container -To get a shell inside the container run: `docker exec -it bash`. +#### General -If you need more flexibility than `docker logs` offers, within the container `/var/log/mail/mail.log` and `/var/log/supervisor/` are the most useful locations to get relevant DMS logs. Use the `tail` or `cat` commands to view their contents. - -To install additional software: +To get a shell inside the container run: `docker exec -it bash`. To install additional software, run: 1. `apt-get update` to update repository metadata. -2. `apt-get install ` +2. `apt-get install ` to install a package, e.g., `apt-get install neovim` if you want to use NeoVim instead of `nano` (which is shipped by default). -For example a text editor you can use in the terminal: `apt-get install nano` +#### Logs + +If you need more flexibility than what the `docker logs` command offers, then the most useful locations to get relevant DMS logs within the container are: + +- `/var/log/mail/mail.log` +- `/var/log/mail/mail/.log` +- `/var/log/supervisor/.log` + +You may use `nano` (a text editor) to edit files, while `less` (a file viewer) and `tail`/`cat` are useful tools to inspect the contents of logs. ## Compatibility diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 5867b1e8..442e5e78 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -71,7 +71,7 @@ DMS does not supply custom values for DNS servers to Rspamd. If you need to use ### Logs -You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the corresponding logs for [Redis](#persistence-with-redis), if it is enabled, at `/var/log/supervisor/rspamd-redis.log`. We recommend inspecting these logs (with `docker exec -it cat /var/log/mail/rspamd.log`) in case Rspamd does not work as expected. +You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the corresponding logs for [Redis](#persistence-with-redis), if it is enabled, at `/var/log/supervisor/rspamd-redis.log`. We recommend inspecting these logs (with `docker exec -it less /var/log/mail/rspamd.log`) in case Rspamd does not work as expected. ### Modules From 460f2d5be5b9ee52b3b30ace3a365a3854f5181a Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Wed, 8 Nov 2023 17:57:16 +0100 Subject: [PATCH 052/125] docs: correct Rspamd directory name (#3629) --- docs/content/config/advanced/optional-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 94097a8e..8a43e4db 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -12,7 +12,7 @@ This is a list of all configuration files and directories which are optional or - **sieve-pipe:** directory for sieve pipe scripts. (Docs: [Sieve][docs-sieve]) - **opendkim:** DKIM directory. Auto-configurable via [`setup.sh config dkim`][docs-setupsh]. (Docs: [DKIM][docs-dkim]) - **ssl:** SSL Certificate directory if `SSL_TYPE` is set to `self-signed` or `custom`. (Docs: [SSL][docs-ssl]) -- **Rspamd:** Override directory for custom settings when using Rspamd (Docs: [Rspamd][docs-rspamd-override-d]) +- **rspamd:** Override directory for custom settings when using Rspamd (Docs: [Rspamd][docs-rspamd-override-d]) ## Files From 290355cf5abc23fc0c5d95d79bd88f1084e7a2df Mon Sep 17 00:00:00 2001 From: Zepmann Date: Wed, 8 Nov 2023 22:18:17 +0100 Subject: [PATCH 053/125] docs: Add Dovecot Lua auth guide + required package (#3579) * Dovecot: add deb package dovecot-lua to support Lua scripting * Adding documentation for Lua authentication * Updated documentation and made a better distinction between Dovecot packages for officially supported features and for community supported features. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/examples/use-cases/auth-lua.md | 162 ++++++++++++++++++++ docs/mkdocs.yml | 1 + target/scripts/build/packages.sh | 11 +- 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 docs/content/examples/use-cases/auth-lua.md diff --git a/docs/content/examples/use-cases/auth-lua.md b/docs/content/examples/use-cases/auth-lua.md new file mode 100644 index 00000000..82586885 --- /dev/null +++ b/docs/content/examples/use-cases/auth-lua.md @@ -0,0 +1,162 @@ +--- +title: 'Examples | Use Cases | Lua Authentication' +--- + +## Introduction + +Dovecot has the ability to let users create their own custom user provisioning and authentication providers in [Lua](https://en.wikipedia.org/wiki/Lua_(programming_language)#Syntax). This allows any data source that can be approached from Lua to be used for authentication, including web servers. It is possible to do more with Dovecot and Lua, but other use cases fall outside of the scope of this documentation page. + +!!! warning "Community contributed guide" + Dovecot authentication via Lua scripting is not officially supported in DMS. No assistance will be provided should you encounter any issues. + + DMS provides the required packages to support this guide. Note that these packages will be removed should they introduce any future maintenance burden. + + The example in this guide relies on the current way in which DMS works with Dovecot configuration files. Changes to this to accommodate new authentication methods such as OpenID Connect will likely break this example in the future. This guide is updated on a best-effort base. + +Dovecot's Lua support can be used for user provisioning (userdb functionality) and/or password verification (passdb functionality). Consider using other userdb and passdb options before considering Lua, since Lua does require the use of additional (unsupported) program code that might require maintenance when updating DMS. + +Each implementation of Lua-based authentication is custom. Therefore it is impossible to write documentation that covers every scenario. Instead, this page describes a single example scenario. If that scenario is followed, you will learn vital aspects that are necessary to kickstart your own Lua development: + +- How to override Dovecot's default configuration to disable parts that conflict with your scenario. +- How to make Dovecot use your Lua script. +- How to add your own Lua script and any libraries it uses. +- How to debug your Lua script. + +## The example scenario + +This scenario starts with [DMS being configured to use LDAP][docs::auth-ldap] for mailbox identification, user authorization and user authentication. In this scenario, [Nextcloud](https://nextcloud.com/) is also a service that uses the same LDAP server for user identification, authorization and authentication. + +The goal of this scenario is to have Dovecot not authenticate the user against LDAP, but against Nextcloud using an [application password](https://docs.nextcloud.com/server/latest/user_manual/en/session_management.html#managing-devices). The idea behind this is that a compromised mailbox password does not compromise the user's account entirely. To make this work, Nextcloud is configured to [deny the use of account passwords by clients](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/config_sample_php_parameters.html#token-auth-enforced) and to [disable account password reset through mail verification](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/config_sample_php_parameters.html#lost-password-link). + +If the application password is configured correctly, an adversary can only use it to access the user's mailbox on DMS, and CalDAV and CardDAV data on Nextcloud. File access through WebDAV can be disabled for the application password used to access mail. Having CalDAV and CardDAV compromised by the same password is a minor setback. If an adversary gets access to a Nextcloud application password through a device of the user, it is likely that the adversary also gets access to the user's calendars and contact lists anyway (locally or through the same account settings used for mail and CalDAV/CardDAV synchronization). The user's stored files in Nextcloud, the LDAP account password and any other services that rely on it would still be protected. A bonus is that a user is able to revoke and renew the mailbox password in Nextcloud for whatever reason, through a friendly user interface with all the security measures with which the Nextcloud instance is configured (e.g. verification of the current account password). + +A drawback of this method is that any (compromised) Nextcloud application password can be used to access the user's mailbox. This introduces a risk that a Nextcloud application password used for something else (e.g. WebDAV file access) is compromised and used to access the user's mailbox. Discussion of that risk and possible mitigations fall outside of the scope of this scenario. + +To answer the questions asked earlier for this specific scenario: + +1. Do I want to use Lua to identify mailboxes and verify that users are are authorized to use mail services? **No. Provisioning is done through LDAP.** +1. Do I want to use Lua to verify passwords that users authenticate with for IMAP/POP3/SMTP in their mail clients? **Yes. Password authentication is done through Lua against Nextcloud.** +1. If the answer is 'yes' to question 1 or 2: are there other methods that better facilitate my use case instead of custom scripts which rely on me being a developer and not just a user? **No. Only HTTP can be used to authenticate against Nextcloud, which is not supported natively by Dovecot or DMS.** + +While it is possible to extend the authentication methods which Nextcloud can facilitate with [Nextcloud apps](https://apps.nextcloud.com/), there is currently a mismatch between what DMS supports and what Nextcloud applications can provide. This might change in the future. For now, Lua will be used to bridge the gap between DMS and Nextcloud for authentication only (Dovecot passdb), while LDAP will still be used to identify mailboxes and verify authorization (Dovecot userdb). + +## Modify Dovecot's configuration + +???+ example "Add to DMS volumes in `compose.yaml`" + + ```yaml + # All new volumes are marked :ro to configure them as read-only, since their contents are not changed from inside the container + volumes: + # Configuration override to disable LDAP authentication + - ./docker-data/dms/config/dovecot/auth-ldap.conf.ext:/etc/dovecot/conf.d/auth-ldap.conf.ext:ro + # Configuration addition to enable Lua authentication + - ./docker-data/dms/config/dovecot/auth-lua-httpbasic.conf:/etc/dovecot/conf.d/auth-lua-httpbasic.conf:ro + # Directory containing Lua scripts + - ./docker-data/dms/config/dovecot/lua/:/etc/dovecot/lua/:ro + ``` + +Create a directory for Lua scripts: +```bash +mkdir -p ./docker-data/dms/config/dovecot/lua +``` + +Create configuration file `./docker-data/dms/config/dovecot/auth-ldap.conf.ext` for LDAP user provisioning: +``` +userdb { + driver = ldap + args = /etc/dovecot/dovecot-ldap.conf.ext +} +``` + +Create configuration file `./docker-data/dms/config/dovecot/auth-lua-httpbasic.conf` for Lua user authentication: +``` +passdb { + driver = lua + args = file=/etc/dovecot/lua/auth-httpbasic.lua blocking=yes +} +``` + +That is all for configuring Dovecot. + +## Create the Lua script + +Create Lua file `./docker-data/dms/config/dovecot/lua/auth-httpbasic.lua` with contents: + +```lua +local http_url = "https://nextcloud.example.com/remote.php/dav/" +local http_method = "PROPFIND" +local http_status_ok = 207 +local http_status_failure = 401 +local http_header_forwarded_for = "X-Forwarded-For" + +package.path = package.path .. ";/etc/dovecot/lua/?.lua" +local base64 = require("base64") + +local http_client = dovecot.http.client { + timeout = 1000; + max_attempts = 1; + debug = false; +} + +function script_init() + return 0 +end + +function script_deinit() +end + +function auth_passdb_lookup(req) + local auth_request = http_client:request { + url = http_url; + method = http_method; + } + auth_request:add_header("Authorization", "Basic " .. (base64.encode(req.user .. ":" .. req.password))) + auth_request:add_header(http_header_forwarded_for, req.remote_ip) + local auth_response = auth_request:submit() + local resp_status = auth_response:status() + local reason = auth_response:reason() + + local returnStatus = dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE + local returnDesc = http_method .. " - " .. http_url .. " - " .. resp_status .. " " .. reason + if resp_status == http_status_ok + then + returnStatus = dovecot.auth.PASSDB_RESULT_OK + returnDesc = "nopassword=y" + elseif resp_status == http_status_failure + then + returnStatus = dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH + returnDesc = "" + end + return returnStatus, returnDesc +end +``` + +Replace the hostname in the URL to the actual hostname of Nextcloud. + +Dovecot [provides an HTTP client for use in Lua](https://doc.dovecot.org/admin_manual/lua/#dovecot.http.client). Aside of that, Lua by itself is pretty barebones. It chooses library compactness over included functionality. You can see that in that a separate library is referenced to add support for Base64 encoding, which is required for [HTTP basic access authentication](https://en.wikipedia.org/wiki/Basic_access_authentication). This library (also a Lua script) is not included. It must be downloaded and stored in the same directory: + +```bash +cd ./docker-data/dms/config/dovecot/lua +curl -JLO https://raw.githubusercontent.com/iskolbin/lbase64/master/base64.lua +``` + +Only use native (pure Lua) libraries as dependencies if possible, such as `base64.lua` from the example. This ensures maximum compatibility. Performance is less of an issue since Lua scripts written for Dovecot probably won't be long or complex, and there won't be a lot of data processing by Lua itself. + +## Debugging a Lua script + +To see which Lua version is used by Dovecot if you plan to do something that is version dependent, run: + +```bash +docker exec CONTAINER_NAME strings /usr/lib/dovecot/libdovecot-lua.so|grep '^LUA_' +``` + +While Dovecot logs the status of authentication attempts for any passdb backend, Dovecot will also log Lua scripting errors and messages sent to Dovecot's [Lua API log functions](https://doc.dovecot.org/admin_manual/lua/#dovecot.i_debug). The combined DMS log (including that of Dovecot) can be viewed using `docker logs CONTAINER_NAME`. If the log is too noisy (_due to other processes in the container also logging to it_), `docker exec CONTAINER_NAME cat /var/log/mail/mail.log` can be used to view the log of Dovecot and Postfix specifically. + +If working with HTTP in Lua, setting `debug = true;` when initiating `dovecot.http.client` will create debug log messages for every HTTP request and response. + +Note that Lua runs compiled bytecode, and that scripts will be compiled when they are initially started. Once compiled, the bytecode is cached and changes in the Lua script will not be processed automatically. Dovecot will reload its configuration and clear its cached Lua bytecode when running `docker exec CONTAINER_NAME dovecot reload`. A (changed) Lua script will be compiled to bytecode the next time it is executed after running the Dovecot reload command. + +[docs::auth-ldap]: ../../config/advanced/auth-ldap.md +[docs::dovecot-override-configuration]: ../../config/advanced/override-defaults/dovecot.md#override-configuration +[docs::dovecot-add-configuration]: ../../config/advanced/override-defaults/dovecot.md#add-configuration +[docs::faq-alter-running-dms-instance-without-container-relaunch]: ../../faq.md#how-to-alter-a-running-dms-instance-without-relaunching-the-container diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index ba920774..fdc2bc59 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -163,6 +163,7 @@ nav: - 'Forward-Only Mail-Server with LDAP': examples/use-cases/forward-only-mailserver-with-ldap-authentication.md - 'Customize IMAP Folders': examples/use-cases/imap-folders.md - 'iOS Mail Push Support': examples/use-cases/ios-mail-push-support.md + - 'Lua Authentication': examples/use-cases/auth-lua.md - 'FAQ' : faq.md - 'Contributing': - 'General Information': contributing/general.md diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index e9b2d479..a025c3b4 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -95,13 +95,19 @@ function _install_packages() { function _install_dovecot() { declare -a DOVECOT_PACKAGES + # Dovecot packages for officially supported features. DOVECOT_PACKAGES=( dovecot-core dovecot-imapd dovecot-ldap dovecot-lmtpd dovecot-managesieved dovecot-pop3d dovecot-sieve dovecot-solr ) - if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]]; then + # Dovecot packages for community supported features. + DOVECOT_PACKAGES+=(dovecot-auth-lua) + + # Dovecot's deb community repository only provides x86_64 packages, so do not include it + # when building for another architecture. + if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]] && [[ "$(uname --machine)" == "x86_64" ]]; then _log 'trace' 'Using Dovecot community repository' curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg @@ -109,6 +115,9 @@ function _install_dovecot() { _log 'trace' 'Updating Dovecot package signatures' apt-get "${QUIET}" update + + # Additional community package needed for Lua support if the Dovecot community repository is used. + DOVECOT_PACKAGES+=(dovecot-lua) fi _log 'debug' 'Installing Dovecot' From 0703e014925824a07ba8da8239e7cb2468405b03 Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Fri, 10 Nov 2023 01:03:21 +0100 Subject: [PATCH 054/125] docs: Clarify default for ENV `FETCHMAIL_PARALLEL` (#3603) - Make this easier to find when browsing the example environment file. - Adjust ENV documentation to properly mark the actual default value. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/environment.md | 6 ++++-- mailserver.env | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 678ec965..284549f1 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -577,8 +577,10 @@ Note: activate this only if you are confident in your bayes database for identif ##### FETCHMAIL_PARALLEL - **0** => `fetchmail` runs with a single config file `/etc/fetchmailrc` - **1** => `/etc/fetchmailrc` is split per poll entry. For every poll entry a separate fetchmail instance is started to allow having multiple imap idle configurations defined. +- **0** => `fetchmail` runs with a single config file `/etc/fetchmailrc` +- 1 => `/etc/fetchmailrc` is split per poll entry. For every poll entry a separate fetchmail instance is started to [allow having multiple imap idle connections per server][fetchmail-imap-workaround] (_when poll entries reference the same IMAP server_). + +[fetchmail-imap-workaround]: https://otremba.net/wiki/Fetchmail_(Debian)#Immediate_Download_via_IMAP_IDLE Note: The defaults of your fetchmailrc file need to be at the top of the file. Otherwise it won't be added correctly to all separate `fetchmail` instances. #### Getmail diff --git a/mailserver.env b/mailserver.env index 1a57ceca..957a632e 100644 --- a/mailserver.env +++ b/mailserver.env @@ -397,6 +397,10 @@ ENABLE_FETCHMAIL=0 # The interval to fetch mail in seconds FETCHMAIL_POLL=300 +# Use multiple fetchmail instances (1 per poll entry in fetchmail.cf) +# Supports multiple IMAP IDLE connections when a server is used across multiple poll entries +# https://otremba.net/wiki/Fetchmail_(Debian)#Immediate_Download_via_IMAP_IDLE +FETCHMAIL_PARALLEL=0 # Enable or disable `getmail`. # From 26214491efb2ec07242c24a9191633f0d72998d9 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 10 Nov 2023 19:57:17 +0100 Subject: [PATCH 055/125] fix: Drop special bits from Postfix `maildrop/` and `public/` directory permissions (#3625) * update K8s deployment Because `allowPrivilegeEscalation` controls SUID/SGID, we require it when postdrop is invoked. * correct permissions for maildrop/public The reason our permissions previously worked out as that in setups where SUID/SGID worked, the binaries used to place files in these directories already have SGID set; the current set of permissions makes less sense (as explained in this comment: https://github.com/docker-mailserver/docker-mailserver/issues/3619#issuecomment-1793816412) Since the binaries used to place files inside these directories alredy have SUID/SGID set, we do not require these bits (or the sticky bit) to be set on the directories. * Apply suggestions from code review --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/advanced/kubernetes.md | 5 ++++- target/scripts/startup/setup.d/mail_state.sh | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 93cc0884..8a47bffc 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -190,7 +190,10 @@ spec: imagePullPolicy: IfNotPresent securityContext: - allowPrivilegeEscalation: false + # Required to support SGID via `postdrop` executable + # in `/var/mail-state` for Postfix (maildrop + public dirs): + # https://github.com/docker-mailserver/docker-mailserver/pull/3625 + allowPrivilegeEscalation: true readOnlyRootFilesystem: false runAsUser: 0 runAsGroup: 0 diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index ffc31791..73c2515b 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -105,10 +105,10 @@ function _setup_save_states() { # These two require the postdrop(103) group: chgrp -R postdrop "${STATEDIR}"/spool-postfix/{maildrop,public} - # After changing the group, special bits (set-gid, sticky) may be stripped, restore them: - # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3149#issuecomment-1454981309 - chmod 1730 "${STATEDIR}/spool-postfix/maildrop" - chmod 2710 "${STATEDIR}/spool-postfix/public" + # These permissions rely on the `postdrop` binary having the SGID bit set. + # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3625 + chmod 730 "${STATEDIR}/spool-postfix/maildrop" + chmod 710 "${STATEDIR}/spool-postfix/public" elif [[ ${ONE_DIR} -eq 1 ]]; then _log 'warn' "'ONE_DIR=1' but no volume was mounted to '${STATEDIR}'" else From 5f2fb72c9ccfc1f24b581ac6f7f80837ed434c66 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:34:46 +0100 Subject: [PATCH 056/125] Rspamd: add check for DKIM private key files' permissions (#3627) * added check for Rspamd DKIM on startup The newly added function `__rspamd__check_dkim_permissions` performs a check on DKIM private key files. This is useful to prevent issues like #3621 in the future. The function is deliberately kept simple and may not catch every single misconfiguration in terms of permissions and ownership, but it should be quite accurate. Please note that the Rspamd setup does NOT change at all, and the checks will not abort the setup in case they fail. A simple warning is emmited. * add more documentation to Rspamd functions * Apply suggestions from code review * improve `__do_as_rspamd_user` * rework check similar to review suggestion see https://github.com/docker-mailserver/docker-mailserver/pull/3627#discussion_r1388697547 --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- target/bin/rspamd-dkim | 7 ---- target/scripts/helpers/rspamd.sh | 12 ++++++ .../startup/setup.d/security/rspamd.sh | 41 +++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 357c2a9c..6dfcc1a0 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -62,13 +62,6 @@ ${ORANGE}EXIT STATUS${RESET} " } -function __do_as_rspamd_user() { - local COMMAND=${1:?Command required when using __do_as_rspamd_user} - _log 'trace' "Running '${*}' as user '_rspamd' now" - shift 1 - su -l '_rspamd' -s "$(command -v "${COMMAND}")" -- "${@}" -} - function _parse_arguments() { FORCE=0 KEYTYPE='rsa' diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh index 868e3d3a..2f4dcc46 100644 --- a/target/scripts/helpers/rspamd.sh +++ b/target/scripts/helpers/rspamd.sh @@ -2,6 +2,18 @@ # shellcheck disable=SC2034 # VAR appears unused. +# Perform a specific command as the Rspamd user (`_rspamd`). This is useful +# in case you want to have correct permissions on newly created files or if +# you want to check whether Rspamd can perform a specific action. +function __do_as_rspamd_user() { + _log 'trace' "Running '${*}' as user '_rspamd'" + su _rspamd -s /bin/bash -c "${*}" +} + +# Calling this function brings common Rspamd-related environment variables +# into the current context. The environment variables are `readonly`, i.e. +# they cannot be modified. Use this function when you require common directory +# names, file names, etc. function _rspamd_get_envs() { readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 19ce75dc..239397e5 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -23,6 +23,9 @@ function _setup_rspamd() { __rspamd__setup_check_authenticated _rspamd_handle_user_modules_adjustments # must run last + # only performing checks, no further setup handled from here onwards + __rspamd__check_dkim_permissions + __rspamd__log 'trace' '---------- Setup finished ----------' else _log 'debug' 'Rspamd is disabled' @@ -280,6 +283,12 @@ function __rspamd__setup_hfilter_group() { fi } +# If 'RSPAMD_CHECK_AUTHENTICATED' is enabled, then content checks for all users, i.e. +# also for authenticated users, are performed. +# +# The default that DMS ships does not check authenticated users. In case the checks are +# enabled, this function will remove the part of the Rspamd configuration that disables +# checks for authenticated users. function __rspamd__setup_check_authenticated() { local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf" readonly MODULE_FILE @@ -294,3 +303,35 @@ function __rspamd__setup_check_authenticated() { "${MODULE_FILE}" fi } + +# This function performs a simple check: go through DKIM configuration files, acquire +# all private key file locations and check whether they exist and whether they can be +# accessed by Rspamd. +function __rspamd__check_dkim_permissions() { + local DKIM_CONF_FILES DKIM_KEY_FILES + [[ -f ${RSPAMD_LOCAL_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_LOCAL_D}/dkim_signing.conf") + [[ -f ${RSPAMD_OVERRIDE_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_OVERRIDE_D}/dkim_signing.conf") + + # Here, we populate DKIM_KEY_FILES which we later iterate over. DKIM_KEY_FILES + # contains all keys files configured by the user. + local FILE + for FILE in "${DKIM_CONF_FILES[@]}"; do + readarray -t DKIM_KEY_FILES_TMP < <(grep -o -E 'path = .*' "${FILE}" | cut -d '=' -f 2 | tr -d ' ";') + DKIM_KEY_FILES+=("${DKIM_KEY_FILES_TMP[@]}") + done + + for FILE in "${DKIM_KEY_FILES[@]}"; do + if [[ -f ${FILE} ]]; then + __rspamd__log 'trace' "Checking DKIM file '${FILE}'" + # See https://serverfault.com/a/829314 for an explanation on `-exec false {} +` + # We additionally resolve symbolic links to check the permissions of the actual files + if find "$(realpath -eL "${FILE}")" -user _rspamd -or -group _rspamd -or -perm -o=r -exec false {} +; then + __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' does not appear to have correct permissions/ownership for Rspamd to use it" + else + __rspamd__log 'trace' "DKIM file '${FILE}' permissions and ownership appear correct" + fi + else + __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' is configured for usage, but does not appear to exist" + fi + done +} From d2efedf91c8622c6db09bd3fd524b1643e92fe1b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 13:23:21 +0000 Subject: [PATCH 057/125] docs: updated `CONTRIBUTORS.md` (#3637) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 173 +++++++++++++++++++++++++----------------------- 1 file changed, 90 insertions(+), 83 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 615cc80e..3fcdd463 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -427,14 +427,21 @@ Thanks goes to these wonderful people ✨ Jarrod Smith + + + jsonn +
+ Joerg Sonnenberger + + + pbek
Patrizio Bekerle - - + Rubytastic2 @@ -469,15 +476,15 @@ Thanks goes to these wonderful people ✨
Null - + + kamuri
Null - - + davidszp @@ -512,15 +519,15 @@ Thanks goes to these wonderful people ✨
Null - + + elbracht
Alexander Elbracht - - + aminvakil @@ -555,15 +562,15 @@ Thanks goes to these wonderful people ✨
Christian Raue - + + danielpanteleit
Daniel Panteleit - - + dmcgrandle @@ -598,15 +605,15 @@ Thanks goes to these wonderful people ✨
FL42 - + + ipernet
Guillaume Simon - - + H4R0 @@ -641,15 +648,15 @@ Thanks goes to these wonderful people ✨
Jeremy Shipman - + + spacecowboy
Jonas Kalderstam - - + artonge @@ -684,15 +691,15 @@ Thanks goes to these wonderful people ✨
Pablo Castorino - + + p-fruck
Philipp Fruck - - + Rillke @@ -727,15 +734,15 @@ Thanks goes to these wonderful people ✨
Vincent Ducamps - + + andymel123
Andymel - - + bigpigeon @@ -770,15 +777,15 @@ Thanks goes to these wonderful people ✨
Null - + + GoliathLabs
Felix - - + yogo1212 @@ -813,15 +820,15 @@ Thanks goes to these wonderful people ✨
0xflotus - + + ifokeev
Johan Fokeev - - + 20th @@ -856,15 +863,15 @@ Thanks goes to these wonderful people ✨
Adrian Pistol - + + kachkaev
Alexander Kachkaev - - + alexanderneu @@ -899,15 +906,15 @@ Thanks goes to these wonderful people ✨
Andrey Likhodievskiy - + + iRhonin
Arash Fatahzade - - + MrFreezeex @@ -942,15 +949,15 @@ Thanks goes to these wonderful people ✨
Bogdan - + + erdos4d
Charles Harris - - + crash7 @@ -985,15 +992,15 @@ Thanks goes to these wonderful people ✨
Damian Moore - + + espitall
Null - - + dkarski @@ -1028,15 +1035,15 @@ Thanks goes to these wonderful people ✨
Dmitry R. - + + aydodo
Dorian Ayllón - - + vedtam @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
Erik Brakkee - + + huncode
Huncode - - + felixn @@ -1109,20 +1116,20 @@ Thanks goes to these wonderful people ✨ - - frugan-it + + frugan-dev
Frugan - + + Marsu31
Gabriel Euzet - - + glandais @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
Ian Andrews - + + Influencer
Influencer - - + jcalfee @@ -1200,13 +1207,6 @@ Thanks goes to these wonderful people ✨
Jiří Kozlovský - - - - jsonn -
- Joerg Sonnenberger - @@ -1639,6 +1639,13 @@ Thanks goes to these wonderful people ✨ + + + Zepmann +
+ Null + + allddd @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
Null - + + dborowy
Null - - + dimalo @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
Null - + + i-C-o-d-e-r
Null - - + idaadi @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
Null - + + landergate
Null - - + callmemagnus @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
Jason Miller - + + mplx
Null - - + odinis @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
Null - + + presocratics
Null - - + rhyst @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
Null - + + sportshead
Null - - + squash @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
Null - + + wolkenschieber
Null - - + worldworm From f5a7e9d119ff944de8cfbc2fa70abdb2b8b79bc0 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:15:39 +0100 Subject: [PATCH 058/125] contributors: fix spelling & remove dedicated AllContributors section (#3638) With the latest `contributors.yml` workflow, everyone is included in the list of contributors. Hence, we do not need the extra section anymore. --- .github/workflows/contributors.yml | 3 ++- CONTRIBUTORS.md | 24 ------------------------ 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index e784ae0e..6f8476f2 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -25,8 +25,9 @@ jobs: with: readme_path: CONTRIBUTORS.md collaborators: all + use_username: true commit_message: 'docs: updated `CONTRIBUTORS.md`' committer_username: github-actions[bot] committer_email: 41898282+github-actions[bot]@users.noreply.github.com - pr_title_on_protected: 'docs: update `CONTRIBUTORS.md' + pr_title_on_protected: 'docs: update `CONTRIBUTORS.md`' auto_detect_branch_protection: true diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 3fcdd463..b7b7ce35 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1956,27 +1956,3 @@ Thanks goes to these wonderful people ✨ - -## Further Contributors - -Also thanks goes to these wonderful people, that have contributed in various other ways than code lines ✨ - -[Emoji Key ✨ (and Contribution Types)](https://allcontributors.org/docs/en/emoji-key) - - - - - - - - -

matrixes
📝
- - - - - - -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! - -Note: We started using [all-contributors](https://github.com/all-contributors/all-contributors) in July 2021. We will add contributors with their future PRs or Issues. Code contributions are added automatically. If you are [one of the 200+](https://github.com/docker-mailserver/docker-mailserver/graphs/contributors) that contributed to the project in the past and would like to see your name here too, please reach out! From 218b56b123b6b74d6ac604db37ba21ae00090f2b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:16:52 +0100 Subject: [PATCH 059/125] docs: updated `CONTRIBUTORS.md` (#3639) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 530 ++++++++++++++++++++++++------------------------ 1 file changed, 265 insertions(+), 265 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index b7b7ce35..893b9572 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -9,42 +9,42 @@ Thanks goes to these wonderful people ✨ casperklein
- Casper + casperklein fbartels
- Felix Bartels + fbartels NorseGaud
- Nathan Pierce + NorseGaud williamdes
- William Desportes + williamdes wernerfred
- Frederic Werner + wernerfred georglauterbach
- Georg Lauterbach + georglauterbach @@ -52,42 +52,42 @@ Thanks goes to these wonderful people ✨ tomav
- Thomas VIAL + tomav erik-wramner
- Erik Wramner + erik-wramner polarathene
- Brennan Kinney + polarathene chikamichi
- Jean-Denis Vauguet + chikamichi martin-schulze-vireso
- Martin Schulze + martin-schulze-vireso Josef-Friedrich
- Josef Friedrich + Josef-Friedrich @@ -95,42 +95,42 @@ Thanks goes to these wonderful people ✨ johansmitsnl
- Johan Smits + johansmitsnl youtous
- Null + youtous 17Halbe
- Null + 17Halbe tve
- Thorsten Von Eicken + tve gmasse
- Germain Masse + gmasse 00angus
- Null + 00angus @@ -138,42 +138,42 @@ Thanks goes to these wonderful people ✨ alinmear
- Paul Steinlechner + alinmear ap-wtioit
- Andreas Perhab + ap-wtioit dominikwinter
- Dominik Winter + dominikwinter crazystick
- Paul Adams + crazystick swiesend
- Sebastian Wiesendahl + swiesend svenyonson
- Steve Johnson + svenyonson @@ -181,42 +181,42 @@ Thanks goes to these wonderful people ✨ stonemaster
- André Stein + stonemaster omarc1492
- Null + omarc1492 phish108
- Christian Glahn + phish108 mwlczk
- Marek Walczak + mwlczk tyranron
- Kai Ren + tyranron KyleOndy
- Kyle Ondy + KyleOndy @@ -224,42 +224,42 @@ Thanks goes to these wonderful people ✨ MichaelSp
- Michael + MichaelSp mindrunner
- Lukas + mindrunner m-a-v
- Sascha Scandella + m-a-v bilak
- Lukáš Vasek + bilak vortex852456
- Null + vortex852456 chris54721
- Christian Grasso + chris54721 @@ -267,42 +267,42 @@ Thanks goes to these wonderful people ✨ hanscees
- Hans-Cees Speel + hanscees jrpear
- Jack Pearson + jrpear dashohoxha
- Dashamir Hoxha + dashohoxha egavard
- GAVARD Ewann + egavard mathuin
- Jack Twilley + mathuin jamebus
- James + jamebus @@ -310,42 +310,42 @@ Thanks goes to these wonderful people ✨ lukecyca
- Luke Cyca + lukecyca okainov
- Oleg Kainov + okainov robertdolca
- Robert Dolca + robertdolca kiliant
- Thomas Kilian + kiliant diiigle
- Tobias Rittig + diiigle akmet
- Akmet + akmet @@ -353,42 +353,42 @@ Thanks goes to these wonderful people ✨ arneke
- Arne Kepp + arneke dennis95stumm
- Dennis Stumm + dennis95stumm moqmar
- Moritz Marquardt + moqmar pyy
- Null + pyy voordev
- Anne + voordev Birkenstab
- Null + Birkenstab @@ -396,42 +396,42 @@ Thanks goes to these wonderful people ✨ BrandonSchmitt
- Brandon Schmitt + BrandonSchmitt Starbix
- Cédric Laubacher + Starbix citec
- GrupoCITEC + citec yajo
- Jairo Llopis + yajo MakerMatrix
- Jarrod Smith + MakerMatrix jsonn
- Joerg Sonnenberger + jsonn @@ -439,28 +439,28 @@ Thanks goes to these wonderful people ✨ pbek
- Patrizio Bekerle + pbek Rubytastic2
- Null + Rubytastic2 analogue
- Semir Patel + analogue weo
- Wolfgang Ocker + weo @@ -474,7 +474,7 @@ Thanks goes to these wonderful people ✨ guardiande
- Null + guardiande @@ -482,42 +482,42 @@ Thanks goes to these wonderful people ✨ kamuri
- Null + kamuri davidszp
- Null + davidszp andreasgerstmayr
- Andreas Gerstmayr + andreasgerstmayr mjung
- Marko J + mjung m-schmoock
- Michael Schmoock + m-schmoock VanVan
- Null + VanVan @@ -525,42 +525,42 @@ Thanks goes to these wonderful people ✨ elbracht
- Alexander Elbracht + elbracht aminvakil
- Amin Vakil + aminvakil andrewlow
- Andrew Low + andrewlow abh
- Ask Bjørn Hansen + abh ubenmackin
- Ben + ubenmackin craue
- Christian Raue + craue @@ -568,42 +568,42 @@ Thanks goes to these wonderful people ✨ danielpanteleit
- Daniel Panteleit + danielpanteleit dmcgrandle
- Darren McGrandle + dmcgrandle theomega
- Dominik Bruhn + theomega DuncanvR
- Null + DuncanvR emazzotta
- Emanuele Mazzotta + emazzotta fl42
- FL42 + fl42 @@ -611,42 +611,42 @@ Thanks goes to these wonderful people ✨ ipernet
- Guillaume Simon + ipernet H4R0
- Null + H4R0 eltociear
- Ikko Eltociear Ashimine + eltociear jamesfryer
- James Fryer + jamesfryer millaguie
- Millaguie + millaguie jedateach
- Jeremy Shipman + jedateach @@ -654,42 +654,42 @@ Thanks goes to these wonderful people ✨ spacecowboy
- Jonas Kalderstam + spacecowboy artonge
- Louis + artonge martinwepner
- Null + martinwepner nueaf
- Michael Als + nueaf keslerm
- Morgan Kesler + keslerm castorinop
- Pablo Castorino + castorinop @@ -697,42 +697,42 @@ Thanks goes to these wonderful people ✨ p-fruck
- Philipp Fruck + p-fruck Rillke
- Rainer Rillke + Rillke reneploetz
- René Plötz + reneploetz bobbravo2
- Bob Gregor + bobbravo2 r-pufky
- Robert Pufky + r-pufky vincentDcmps
- Vincent Ducamps + vincentDcmps @@ -740,42 +740,42 @@ Thanks goes to these wonderful people ✨ andymel123
- Andymel + andymel123 bigpigeon
- Bigpigeon + bigpigeon engelant
- Null + engelant j-marz
- Null + j-marz lokipo
- Null + lokipo msheakoski
- Null + msheakoski @@ -783,35 +783,35 @@ Thanks goes to these wonderful people ✨ GoliathLabs
- Felix + GoliathLabs yogo1212
- Leon Busch-George + yogo1212 mpanneck
- Marius Panneck + mpanneck willtho89
- Thomas Willems + willtho89 tbutter
- Thomas Butter + tbutter @@ -826,42 +826,42 @@ Thanks goes to these wonderful people ✨ ifokeev
- Johan Fokeev + ifokeev 20th
- Null + 20th 2b
- Null + 2b askz
- Max: + askz acch
- Achim Christ + acch vifino
- Adrian Pistol + vifino @@ -869,42 +869,42 @@ Thanks goes to these wonderful people ✨ kachkaev
- Alexander Kachkaev + kachkaev alexanderneu
- Alexander Neu + alexanderneu ch3sh1r
- Bedniakov Aleksei + ch3sh1r eglia
- Andreas Egli + eglia groupmsl
- Andrew Cornford + groupmsl green-anger
- Andrey Likhodievskiy + green-anger @@ -912,42 +912,42 @@ Thanks goes to these wonderful people ✨ iRhonin
- Arash Fatahzade + iRhonin MrFreezeex
- Arthur Outhenin-Chalandre + MrFreezeex arunvc
- Arun + arunvc astrocket
- Astro + astrocket baxerus
- Benedict Endemann + baxerus spock
- Bogdan + spock @@ -955,42 +955,42 @@ Thanks goes to these wonderful people ✨ erdos4d
- Charles Harris + erdos4d crash7
- Christian Musa + crash7 auchri
- Christoph + auchri arkanovicz
- Claude Brisson + arkanovicz CBeerta
- Claus Beerta + CBeerta damianmoore
- Damian Moore + damianmoore @@ -998,42 +998,42 @@ Thanks goes to these wonderful people ✨ espitall
- Null + espitall dkarski
- Daniel Karski + dkarski dbellavista
- Daniele Bellavista + dbellavista danielvandenberg95
- Daniël Van Den Berg + danielvandenberg95 mlatorre31
- Dingoz + mlatorre31 mazzz1y
- Dmitry R. + mazzz1y @@ -1041,42 +1041,42 @@ Thanks goes to these wonderful people ✨ aydodo
- Dorian Ayllón + aydodo vedtam
- Edmond Varga + vedtam edvorg
- Eduard Knyshov + edvorg eliroca
- Elisei Roca + eliroca ekkis
- Erick Calder + ekkis ErikEngerd
- Erik Brakkee + ErikEngerd @@ -1084,42 +1084,42 @@ Thanks goes to these wonderful people ✨ huncode
- Huncode + huncode felixn
- Felix N + felixn flole
- Florian + flole froks
- Florian Roks + froks fkefer
- Franz Keferböck + fkefer frugan-dev
- Frugan + frugan-dev @@ -1127,14 +1127,14 @@ Thanks goes to these wonderful people ✨ Marsu31
- Gabriel Euzet + Marsu31 glandais
- Gabriel Landais + glandais @@ -1148,7 +1148,7 @@ Thanks goes to these wonderful people ✨ harryyoud
- Harry Youd + harryyoud @@ -1162,7 +1162,7 @@ Thanks goes to these wonderful people ✨ sirgantrithon
- Ian Andrews + sirgantrithon @@ -1177,14 +1177,14 @@ Thanks goes to these wonderful people ✨ jcalfee
- Null + jcalfee init-js
- JS Légaré + init-js @@ -1198,14 +1198,14 @@ Thanks goes to these wonderful people ✨ JiLleON
- Null + JiLleON jirislav
- Jiří Kozlovský + jirislav @@ -1213,42 +1213,42 @@ Thanks goes to these wonderful people ✨ jmccl
- Null + jmccl jurekbarth
- Jurek Barth + jurekbarth JOduMonT
- JOnathan DuMonT + JOduMonT Kaan88
- Kaan + Kaan88 akkumar
- Karthik K + akkumar KCrawley
- Null + KCrawley @@ -1256,42 +1256,42 @@ Thanks goes to these wonderful people ✨ khuedoan
- Khue Doan + khuedoan JustAnother1
- Lars Pötter + JustAnother1 LeoWinterDE
- Leo Winter + LeoWinterDE linhandev
- Lin Han + linhandev luke-
- Lucas Bartholemy + luke- LucidityCrash
- Null + LucidityCrash @@ -1306,35 +1306,35 @@ Thanks goes to these wonderful people ✨ madmath03
- Mathieu Brunot + madmath03 maxemann96
- Maximilian Hippler + maxemann96 dragetd
- Michael G. + dragetd michaeljensen
- Michael Jensen + michaeljensen exhuma
- Michel Albert + exhuma @@ -1342,42 +1342,42 @@ Thanks goes to these wonderful people ✨ milas
- Milas Bowman + milas mcchots
- Mohammed Chotia + mcchots MohammedNoureldin
- Mohammed Noureldin + MohammedNoureldin mpldr
- Moritz Poldrack + mpldr naveensrinivasan
- Naveen + naveensrinivasan neuralp
- Nicholas Pepper + neuralp @@ -1385,42 +1385,42 @@ Thanks goes to these wonderful people ✨ radicand
- Nick Pappas + radicand nilshoell
- Nils Höll + nilshoell nknapp
- Nils Knappmeier + nknapp pcqnt
- Olivier Picquenot + pcqnt OrvilleQ
- Orville Q. Song + OrvilleQ ovidiucp
- Ovidiu Predescu + ovidiucp @@ -1428,42 +1428,42 @@ Thanks goes to these wonderful people ✨ mrPjer
- Petar Šegina + mrPjer peter-hartmann
- Peter Hartmann + peter-hartmann piwai
- Pierre-Yves Rofes + piwai remoe
- Remo E + remoe romansey
- Roman Seyffarth + romansey MightySCollins
- Sam Collins + MightySCollins @@ -1471,42 +1471,42 @@ Thanks goes to these wonderful people ✨ 501st-alpha1
- Scott Weldon + 501st-alpha1 klamann
- Sebastian Straub + klamann svdb0
- Serge Van Den Boom + svdb0 3ap
- Sergey Nazaryev + 3ap shyim
- Shyim + shyim sjmudd
- Simon J Mudd + sjmudd @@ -1514,42 +1514,42 @@ Thanks goes to these wonderful people ✨ simonsystem
- Simon Schröter + simonsystem stephan-devop
- Stephan + stephan-devop stigok
- Stig Otnes Kolstad + stigok 5ven
- Sven Kauber + 5ven syl20bnr
- Sylvain Benner + syl20bnr sylvaindumont
- Sylvain Dumont + sylvaindumont @@ -1557,42 +1557,42 @@ Thanks goes to these wonderful people ✨ TechnicLab
- Null + TechnicLab thomasschmit
- Thomas Schmit + thomasschmit Thiritin
- Tin + Thiritin tweibert
- Torben Weibert + tweibert torus
- Toru Hisai + torus VictorKoenders
- Trangar + VictorKoenders @@ -1600,42 +1600,42 @@ Thanks goes to these wonderful people ✨ Twist235
- Null + Twist235 k3it
- Vasiliy Gokoyev + k3it Drakulix
- Victoria Brekenfeld + Drakulix vilisas
- Vilius + vilisas 42wim
- Wim + 42wim ShiriNmi1520
- Y.C.Huang + ShiriNmi1520 @@ -1643,42 +1643,42 @@ Thanks goes to these wonderful people ✨ Zepmann
- Null + Zepmann allddd
- Allddd + allddd arcaine2
- Null + arcaine2 awb99
- Awb99 + awb99 brainkiller
- Null + brainkiller cternes
- Null + cternes @@ -1686,42 +1686,42 @@ Thanks goes to these wonderful people ✨ dborowy
- Null + dborowy dimalo
- Null + dimalo eleith
- Eleith + eleith ghnp5
- Null + ghnp5 helmutundarnold
- Null + helmutundarnold hnws
- Null + hnws @@ -1729,42 +1729,42 @@ Thanks goes to these wonderful people ✨ i-C-o-d-e-r
- Null + i-C-o-d-e-r idaadi
- Null + idaadi ixeft
- Null + ixeft jjtt
- Null + jjtt paralax
- Jose Nazario + paralax jpduyx
- Null + jpduyx @@ -1772,42 +1772,42 @@ Thanks goes to these wonderful people ✨ landergate
- Null + landergate callmemagnus
- Magnus Anderssen + callmemagnus marios88
- Null + marios88 matrixes
- Null + matrixes mchamplain
- Mchamplain + mchamplain millerjason
- Jason Miller + millerjason @@ -1815,42 +1815,42 @@ Thanks goes to these wonderful people ✨ mplx
- Null + mplx odinis
- Null + odinis okamidash
- Okami + okamidash olaf-mandel
- Null + olaf-mandel ontheair81
- Null + ontheair81 pravynandas
- Null + pravynandas @@ -1858,42 +1858,42 @@ Thanks goes to these wonderful people ✨ presocratics
- Null + presocratics rhyst
- Null + rhyst rmlhuk
- Null + rmlhuk rriski
- Null + rriski schnippl0r
- Null + schnippl0r smargold476
- Null + smargold476 @@ -1901,42 +1901,42 @@ Thanks goes to these wonderful people ✨ sportshead
- Null + sportshead squash
- Null + squash strarsis
- Null + strarsis tamueller
- Null + tamueller vivacarvajalito
- Null + vivacarvajalito wligtenberg
- Null + wligtenberg @@ -1944,14 +1944,14 @@ Thanks goes to these wonderful people ✨ wolkenschieber
- Null + wolkenschieber worldworm
- Null + worldworm From d8ebf591f9ff320821c5779521fed1505f1b2916 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 14 Nov 2023 22:34:56 +0100 Subject: [PATCH 060/125] docs: correct path for logs (#3640) --- docs/content/config/debugging.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 24e41566..9888a9e9 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -47,8 +47,7 @@ To get a shell inside the container run: `docker exec -it bash` If you need more flexibility than what the `docker logs` command offers, then the most useful locations to get relevant DMS logs within the container are: -- `/var/log/mail/mail.log` -- `/var/log/mail/mail/.log` +- `/var/log/mail/.log` - `/var/log/supervisor/.log` You may use `nano` (a text editor) to edit files, while `less` (a file viewer) and `tail`/`cat` are useful tools to inspect the contents of logs. From 042bd872bf8500d748f60468534fbb701ed09896 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:03:32 +0100 Subject: [PATCH 061/125] chore(deps): Bump docker/build-push-action from 5.0.0 to 5.1.0 (#3645) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index c7dc6e3f..0f375d45 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 5d68cd73..0ed2fd3e 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 3c6e1288..5b8bac62 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index ff8c8c1a..b39cced8 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . tags: mailserver-testing:ci From 020542a66cadeaaab69027587e52433814e7612b Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 23 Nov 2023 14:05:51 +1300 Subject: [PATCH 062/125] docs: Debugging - Add macOS suggestion to use `gRPC FUSE` file sharing implementation (#3652) --- docs/content/config/debugging.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 9888a9e9..a01dbe6a 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -73,6 +73,7 @@ This could be from outdated software, or running a system that isn't able to pro ### System +- **macOS:** DMS has limited support for macOS. Often an issue encountered is due to permissions related to the `volumes` config in `compose.yaml`. You may have luck [trying `gRPC FUSE`][gh-macos-support] as the file sharing implementation; [`VirtioFS` is the successor][docker-macos-virtiofs] but presently appears incompatible with DMS. - **Kernel:** Some systems provide [kernels with modifications (_replacing defaults and backporting patches_)][network::kernels-modified] to support running legacy software or kernels, complicating compatibility. This can be commonly experienced with products like NAS. - **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. - **Container runtime:** Docker and Podman for example have subtle differences. DMS docs are primarily focused on Docker, but we try to document known issues where relevant. @@ -85,13 +86,16 @@ This could be from outdated software, or running a system that isn't able to pro [network::kernels-modified]: https://github.com/docker-mailserver/docker-mailserver/pull/2662#issuecomment-1168435970 [network::kernel-nftables]: https://unix.stackexchange.com/questions/596493/can-nftables-and-iptables-ip6tables-rules-be-applied-at-the-same-time-if-so-wh/596497#596497 -[docs-faq]: ../faq.md [docs-environment-log-level]: ./environment.md#log_level +[docs-faq]: ../faq.md [docs-ipv6]: ./advanced/ipv6.md [docs-introduction]: ../introduction.md +[docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container [docs-usage]: ../usage.md + [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues +[gh-macos-support]: https://github.com/docker-mailserver/docker-mailserver/issues/3648#issuecomment-1822774080 [gh-discuss-roundcube-fail2ban]: https://github.com/orgs/docker-mailserver/discussions/3273#discussioncomment-5654603 [docker-rootless-interface]: https://github.com/moby/moby/issues/45742 -[docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container +[docker-macos-virtiofs]: https://www.docker.com/blog/speed-boost-achievement-unlocked-on-docker-desktop-4-6-for-mac/ From 2a716cf4a4e0eec9ec12d99fcc6e9c5e625a63f8 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 23 Nov 2023 22:47:11 +1300 Subject: [PATCH 063/125] docs: Dovecot Sieve - Adjust to new path for user home folder (#3650) --- docs/content/config/advanced/mail-sieve.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index fe540efb..b3cef249 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -11,9 +11,9 @@ There are global and user specific filters which are filtering the incoming emai Global filters are applied to EVERY incoming mail for EVERY email address. To specify a global Sieve filter provide a `docker-data/dms/config/before.dovecot.sieve` or a `docker-data/dms/config/after.dovecot.sieve` file with your filter rules. -If any filter in this filtering chain discards an incoming mail, the delivery process will stop as well and the mail will not reach any following filters(e.g. global-before stops an incoming spam mail: The mail will get discarded and a user-specific filter won't get applied.) +If any filter in this filtering chain discards an incoming mail, the delivery process will stop as well and the mail will not reach any following filters (e.g. global-before stops an incoming spam mail: The mail will get discarded and a user-specific filter won't get applied.) -To specify a user-defined Sieve filter place a `.dovecot.sieve` file into a virtual user's mail folder e.g. `/var/mail/example.com/user1/.dovecot.sieve`. If this file exists dovecot will apply the filtering rules. +To specify a user-defined Sieve filter place a `.dovecot.sieve` file into a virtual user's mail folder (e.g. `/var/mail/example.com/user1/home/.dovecot.sieve`). If this file exists dovecot will apply the filtering rules. It's even possible to install a user provided Sieve filter at startup during users setup: simply include a Sieve file in the `docker-data/dms/config/` path for each user login that needs a filter. The file name provided should be in the form `.dovecot.sieve`, so for example for `user1@example.com` you should provide a Sieve file named `docker-data/dms/config/user1@example.com.dovecot.sieve`. From 7d1fcb75d76db7a2d30441afa3074681273a45c8 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 25 Nov 2023 22:57:34 +1300 Subject: [PATCH 064/125] docs: Debugging - Correctly starting DMS (#3654) --- docs/content/config/debugging.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index a01dbe6a..d1e29376 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -14,6 +14,27 @@ This page contains valuable information when it comes to resolving issues you en - Check that all published DMS ports are actually open and not blocked by your ISP / hosting provider. - SSL errors are likely the result of a wrong setup on the user side and not caused by DMS itself. +- Ensure that you have correctly started DMS. Many problems related to configuration are due to this. + +!!! danger "Correctly starting DMS" + + Use the [`--force-recreate`][docker-docs::force-recreate] option to avoid configuration mishaps: `docker compose up --force-recreate` + + Alternatively, always use `docker compose down` to stop DMS. **Do not** rely on `CTRL + C`, `docker compose stop`, or `docker compose restart`. + + --- + + DMS setup scripts are run when a container starts, but may fail to work properly if you do the following: + + - Stopping a container with commands like: `docker stop` or `docker compose up` stopped via `CTRL + C` instead of `docker compose down`. + - Restarting a container. + + Volumes persist data across container instances, however the same container instance will keep internal changes not stored in a volume until the container is removed. + + Due to this, DMS setup scripts may modify configuration it has already modified in the past. + + - This is brittle as some changes are naive by assuming they are applied to the original configs from the image. + - Volumes in `compose.yaml` are expected to persist any important data. Thus it should be safe to throwaway the container created each time, avoiding this config problem. ### Mail sent from DMS does not arrive at destination @@ -99,3 +120,4 @@ This could be from outdated software, or running a system that isn't able to pro [docker-rootless-interface]: https://github.com/moby/moby/issues/45742 [docker-macos-virtiofs]: https://www.docker.com/blog/speed-boost-achievement-unlocked-on-docker-desktop-4-6-for-mac/ +[docker-docs::force-recreate]: https://docs.docker.com/compose/reference/up/ From cedd360ebd8d66b9bc7ef1ab986abc9c8ffeebc4 Mon Sep 17 00:00:00 2001 From: Jean-Kevin KPADEY <9912558+mivek@users.noreply.github.com> Date: Sat, 25 Nov 2023 11:02:42 +0100 Subject: [PATCH 065/125] docs: add a new example explaining how to integrate crowdsec with (#3651) --- docs/content/examples/tutorials/crowdsec.md | 74 +++++++++++++++++++++ docs/mkdocs.yml | 1 + 2 files changed, 75 insertions(+) create mode 100644 docs/content/examples/tutorials/crowdsec.md diff --git a/docs/content/examples/tutorials/crowdsec.md b/docs/content/examples/tutorials/crowdsec.md new file mode 100644 index 00000000..2e8efe06 --- /dev/null +++ b/docs/content/examples/tutorials/crowdsec.md @@ -0,0 +1,74 @@ +--- +title: 'Tutorials | Crowdsec' +--- + +!!! quote "What is Crowdsec?" + + Crowdsec is an open source software that detects and blocks attackers using log analysis. + It has access to a global community-wide IP reputation database. + + [Source](https://www.crowdsec.net) + +## Installation + +Crowdsec supports multiple [installation methods][crowdsec-installation-docs], however this page will use the docker installation. + + +### Docker mailserver + +In your `compose.yaml` for the DMS service, add a bind mount volume for `/var/log/mail`. This is to share the DMS logs to a separate crowdsec container. + +!!! example + ```yaml + services: + mailserver: + - /docker-data/dms/mail-logs/:/var/log/mail/ + ``` + +### Crowdsec + +The crowdsec container should also bind mount the same host path for the DMS logs that was added in the DMS example above. + +```yaml +services: + image: crowdsecurity/crowdsec + restart: unless-stopped + ports: + - "8080:8080" + - "6060:6060" + volumes: + - /docker-data/dms/mail-logs/:/var/log/dms:ro + - ./acquis.d:/etc/crowdsec/acquis.d + - crowdsec-db:/var/lib/crowdsec/data/ + environment: + # These collection contains parsers and scenarios for postfix and dovecot + COLLECTIONS: crowdsecurity/postfix crowdsecurity/dovecot + TZ: Europe/Paris +volumes: + crowdsec-db: +``` + +## Configuration + +Configure crowdsec to read and parse DMS logs file. + +!!! example + + Create the file `dms.yml` in `./acquis.d/` + + ```yaml + --- + source: file + filenames: + - /var/log/dms/mail.log + labels: + type: syslog + ``` + +!!! warning Bouncers + + Crowdsec on its own is just a detection software, the remediation is done by components called bouncers. + This page does not explain how to install or configure a bouncer. It can be found in [crowdsec documentation][crowdsec-bouncer-docs]. + +[crowdsec-installation-docs]: https://doc.crowdsec.net/docs/getting_started/install_crowdsec +[crowdsec-bouncer-docs]: https://doc.crowdsec.net/docs/bouncers/intro diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index fdc2bc59..bd41a798 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -157,6 +157,7 @@ nav: - 'Tutorials': - 'Basic Installation': examples/tutorials/basic-installation.md - 'Mailserver behind Proxy': examples/tutorials/mailserver-behind-proxy.md + - 'Crowdsec': examples/tutorials/crowdsec.md - 'Building your own Docker image': examples/tutorials/docker-build.md - 'Blog Posts': examples/tutorials/blog-posts.md - 'Use Cases': From ba814f421302e3545cfee342f508f01d0ae98ce0 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 26 Nov 2023 01:34:00 +1300 Subject: [PATCH 066/125] ci: Linter EC should use `/check` as the mount path (#3655) --- test/linting/lint.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 7c281e8f..30cab39f 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -17,11 +17,14 @@ SHELLCHECK_VERSION='0.9.0' source "${REPOSITORY_ROOT}/target/scripts/helpers/log.sh" function _eclint() { + # `/check` is used instead of `/ci` as the mount path due to: + # https://github.com/editorconfig-checker/editorconfig-checker/issues/268#issuecomment-1826200253 + # `.ecrc.json` continues to explicitly ignores the `.git/` path to avoid any potential confusion if docker run --rm --tty \ - --volume "${REPOSITORY_ROOT}:/ci:ro" \ - --workdir "/ci" \ + --volume "${REPOSITORY_ROOT}:/check:ro" \ + --workdir "/check" \ --name dms-test_eclint \ - "mstruebing/editorconfig-checker:${ECLINT_VERSION}" ec -config "/ci/test/linting/.ecrc.json" + "mstruebing/editorconfig-checker:${ECLINT_VERSION}" ec -config "/check/test/linting/.ecrc.json" then _log 'info' 'ECLint succeeded' else From b663e10841d7d95f25d51efb7696c0c5b588d55f Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 26 Nov 2023 02:00:56 +0100 Subject: [PATCH 067/125] release: v13.0.0 (#3641) * adjust PR template I am tired of writing `CHANGELOG.md` myself --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Casper --- .github/pull_request_template.md | 1 + CHANGELOG.md | 131 +++++++++++++++++++++++++++---- VERSION | 2 +- 3 files changed, 118 insertions(+), 16 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 26dc801e..040d67dd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -26,3 +26,4 @@ Fixes # - [ ] I have made corresponding changes to the documentation (README.md or the documentation under `docs/`) - [ ] If necessary I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes +- [ ] **I have added information about changes made in this PR to `CHANGELOG.md`** diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b875e2b..71c9de3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,30 +2,131 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v12.1.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.0.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.0) + ### Breaking -- The environment variable `ENABLE_LDAP=1` has been changed to `ACCOUNT_PROVISIONER=LDAP`. -- Postfix now defaults to supporting DSNs (_[Delivery Status Notifications](https://github.com/docker-mailserver/docker-mailserver/pull/3572#issuecomment-1751880574)_) only for authenticated users. This is a security measure to reduce spammer abuse of your DMS instance as a backscatter source. - - If you need to modify this change, please let us know by opening an issue / discussion. - - You can [opt-out (_enable DSNs_) via the `postfix-main.cf` override support](https://docker-mailserver.github.io/docker-mailserver/v12.1/config/advanced/override-defaults/postfix/) using the contents: `smtpd_discard_ehlo_keywords =`. - - Likewise for authenticated users, the submission(s) ports (465 + 587) are configured internally via `master.cf` to keep DSNs enabled (_since authentication protects from abuse_). +- **LDAP:** + - ENV `LDAP_SERVER_HOST`, `DOVECOT_URIS`, and `SASLAUTHD_LDAP_SERVER` will now log an error if the LDAP URI scheme is missing. Previously there was an implicit fallback to `ldap://` ([#3522](https://github.com/docker-mailserver/docker-mailserver/pull/3522)) + - `ENABLE_LDAP=1` is no longer supported, please use `ACCOUNT_PROVISIONER=LDAP` ([#3507](https://github.com/docker-mailserver/docker-mailserver/pull/3507)) +- **Rspamd:** + - The deprecated path for the Rspamd custom commands file (`/tmp/docker-mailserver/rspamd-modules.conf`) now prevents successful startup. The correct path is `/tmp/docker-mailserver/rspamd/custom-commands.conf`. +- **Dovecot:** + - Dovecot mail storage per account in `/var/mail` previously shared the same path for the accounts home directory ([#3335](https://github.com/docker-mailserver/docker-mailserver/pull/3335)) + - The home directory now is a subdirectory `home/`. This change better supports sieve scripts. + - **NOTE:** The change has not yet been implemented for `ACCOUNT_PROVISIONER=LDAP`. +- **Postfix:** + - `/etc/postfix/master.cf` has renamed the "smtps" service to "submissions" ([#3235](https://github.com/docker-mailserver/docker-mailserver/pull/3235)) + - This is the modern `/etc/services` name for port 465, aligning with the similar "submission" port 587. + - Postfix now defaults to supporting DSNs (_[Delivery Status Notifications](https://github.com/docker-mailserver/docker-mailserver/pull/3572#issuecomment-1751880574)_) only for authenticated users (_via ports 465 + 587_). This is a security measure to reduce spammer abuse of your DMS instance as a backscatter source. ([#3572](https://github.com/docker-mailserver/docker-mailserver/pull/3572)) + - If you need to modify this change, please let us know by opening an issue / discussion. + - You can [opt out (_enable DSNs_) via the `postfix-main.cf` override support](https://docker-mailserver.github.io/docker-mailserver/v12.1/config/advanced/override-defaults/postfix/) using the contents: `smtpd_discard_ehlo_keywords =`. + - Likewise for authenticated users, the submission(s) ports (465 + 587) are configured internally via `master.cf` to keep DSNs enabled (_since authentication protects from abuse_). - If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: + If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: - ```cf - submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn - submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn - ``` - -- using the old path for the Rspamd custom commands file (`/tmp/docker-mailserver/rspamd-modules.conf`), which was deprecated, will now prevent startup; use `/tmp/docker-mailserver/rspamd/custom-commands.conf` instead + ```cf + submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + ``` ### Added -- New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) +- **Features:** + - `getmail` as an alternative to `fetchmail` ([#2803](https://github.com/docker-mailserver/docker-mailserver/pull/2803)) + - `setup` CLI - `setup fail2ban` gained a new `status ` subcommand ([#3455](https://github.com/docker-mailserver/docker-mailserver/pull/3455)) +- **Environment Variables:** + - `MARK_SPAM_AS_READ`. When set to `1`, marks incoming spam as "read" to avoid unwanted "new mail" notifications for junk mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) + - `DMS_VMAIL_UID` and `DMS_VMAIL_GID` allow changing the default ID values (`5000:5000`) for the Dovecot vmail user and group ([#3550](https://github.com/docker-mailserver/docker-mailserver/pull/3550)) + - `RSPAMD_CHECK_AUTHENTICATED` allows authenticated users to avoid additional security checks by Rspamd ([#3440](https://github.com/docker-mailserver/docker-mailserver/pull/3440)) +- **Documentation:** + - Use-case examples / tutorials: + - iOS mail push support ([#3513](https://github.com/docker-mailserver/docker-mailserver/pull/3513)) + - Guide for setting up Dovecot Authentication via Lua ([#3579](https://github.com/docker-mailserver/docker-mailserver/pull/3579)) + - Guide for integrating with the Crowdsec service ([#3651](https://github.com/docker-mailserver/docker-mailserver/pull/3651)) + - Debugging page: + - New compatibility section ([#3404](https://github.com/docker-mailserver/docker-mailserver/pull/3404)) + - Now advises how to (re)start DMS correctly ([#3654](https://github.com/docker-mailserver/docker-mailserver/pull/3654)) + - Better communicate distinction between DMS FQDN and DMS mail accounts ([#3372](https://github.com/docker-mailserver/docker-mailserver/pull/3372)) + - Traefik example now includes `passthrough=true` on implicit ports ([#3568](https://github.com/docker-mailserver/docker-mailserver/pull/3568)) + - Rspamd docs have received a variety of revisions ([#3318](https://github.com/docker-mailserver/docker-mailserver/pull/3318), [#3325](https://github.com/docker-mailserver/docker-mailserver/pull/3325), [#3329](https://github.com/docker-mailserver/docker-mailserver/pull/3329)) + - IPv6 config examples with content tabs ([#3436](https://github.com/docker-mailserver/docker-mailserver/pull/3436)) + - Mention [internet.nl](https://internet.nl/test-mail/) as another testing service ([#3445](https://github.com/docker-mailserver/docker-mailserver/pull/3445)) + - `setup alias add ...` CLI help message now includes an example for aliasing to multiple recipients ([#3600](https://github.com/docker-mailserver/docker-mailserver/pull/3600)) + - `SPAMASSASSIN_SPAM_TO_INBOX=1`, now emits a debug log to raise awareness that `SA_KILL` will be ignored ([#3360](https://github.com/docker-mailserver/docker-mailserver/pull/3360)) + - `CLAMAV_MESSAGE_SIZE_LIMIT` now logs a warning when the value exceeds what ClamAV is capable of supporting (4GiB max scan size [#3332](https://github.com/docker-mailserver/docker-mailserver/pull/3332), 2GiB max file size [#3341](https://github.com/docker-mailserver/docker-mailserver/pull/3341)) + - Added note to caution against changing `mydestination` in Postfix's `main.cf` ([#3316](https://github.com/docker-mailserver/docker-mailserver/pull/3316)) +- **Internal:** + - Added a wrapper to update Postfix configuration safely ([#3484](https://github.com/docker-mailserver/docker-mailserver/pull/3484), [#3503](https://github.com/docker-mailserver/docker-mailserver/pull/3503)) + - Add debug group to `packages.sh` ([#3578](https://github.com/docker-mailserver/docker-mailserver/pull/3578)) +- **Tests:** + - Additional linting check for BASH syntax ([#3369](https://github.com/docker-mailserver/docker-mailserver/pull/3369)) + +### Updates + +- **Misc:** + - Changed `setup config dkim` default key size to `2048` (`open-dkim`) ([#3508](https://github.com/docker-mailserver/docker-mailserver/pull/3508)) +- **Postfix:** + - Dropped special bits from `maildrop/` and `public/` directory permissions ([#3625](https://github.com/docker-mailserver/docker-mailserver/pull/3625)) +- **Rspamd:** + - Adjusted learning of ham ([#3334](https://github.com/docker-mailserver/docker-mailserver/pull/3334)) + - Adjusted `antivirus.conf` ([#3331](https://github.com/docker-mailserver/docker-mailserver/pull/3331)) + - `logrotate` setup + Rspamd log path + tests log helper fallback path ([#3576](https://github.com/docker-mailserver/docker-mailserver/pull/3576)) + - Setup during container startup is now more resilient ([#3578](https://github.com/docker-mailserver/docker-mailserver/pull/3578)) + - Changed DKIM default config location ([#3597](https://github.com/docker-mailserver/docker-mailserver/pull/3597)) + - Removed the symlink for the `override.d/` directory in favor of using `cp`, integrated into the changedetector service, , added a `--force` option for the Rspamd DKIM management, and provided a dedicated helper script for common ENV variables ([#3599](https://github.com/docker-mailserver/docker-mailserver/pull/3599)) + - Required permissions are now verified for DKIM private key files ([#3627](https://github.com/docker-mailserver/docker-mailserver/pull/3627)) +- **Documentation:** + - Documentation aligned to Compose v2 conventions, `docker-compose` command changed to `docker compose`, `docker-compose.yaml` to `compose.yaml` ([#3295](https://github.com/docker-mailserver/docker-mailserver/pull/3295)) + - Restored missing edit button ([#3338](https://github.com/docker-mailserver/docker-mailserver/pull/3338)) + - Complete rewrite of the IPv6 page ([#3244](https://github.com/docker-mailserver/docker-mailserver/pull/3244), [#3531](https://github.com/docker-mailserver/docker-mailserver/pull/3531)) + - Complete rewrite of the "Update and Cleanup" maintenance page ([#3539](https://github.com/docker-mailserver/docker-mailserver/pull/3539), [#3583](https://github.com/docker-mailserver/docker-mailserver/pull/3583)) + - Improved debugging page advice on working with logs ([#3626](https://github.com/docker-mailserver/docker-mailserver/pull/3626), [#3640](https://github.com/docker-mailserver/docker-mailserver/pull/3640)) + - Clarified the default for ENV `FETCHMAIL_PARALLEL` ([#3603](https://github.com/docker-mailserver/docker-mailserver/pull/3603)) + - Removed port 25 from FAQ entry for mail client ports supporting authenticated submission ([#3496](https://github.com/docker-mailserver/docker-mailserver/pull/3496)) + - Updated home path in docs for Dovecot Sieve ([#3370](https://github.com/docker-mailserver/docker-mailserver/pull/3370), [#3650](https://github.com/docker-mailserver/docker-mailserver/pull/3650)) + - Fixed path to `rspamd.log` ([#3585](https://github.com/docker-mailserver/docker-mailserver/pull/3585)) + - "Optional Config" page now uses consistent lowercase convention for directory names ([#3629](https://github.com/docker-mailserver/docker-mailserver/pull/3629)) + - `CONTRIBUTORS.md`: Removed redundant "All Contributors" section ([#3638](https://github.com/docker-mailserver/docker-mailserver/pull/3638)) +- **Internal:** + - LDAP config improvements (Removed implicit `ldap://` LDAP URI scheme fallback) ([#3522](https://github.com/docker-mailserver/docker-mailserver/pull/3522)) + - Changed style conventions for internal scripts ([#3361](https://github.com/docker-mailserver/docker-mailserver/pull/3361), [#3364](https://github.com/docker-mailserver/docker-mailserver/pull/3364), [#3365](https://github.com/docker-mailserver/docker-mailserver/pull/3365), [#3366](https://github.com/docker-mailserver/docker-mailserver/pull/3366), [#3368](https://github.com/docker-mailserver/docker-mailserver/pull/3368), [#3464](https://github.com/docker-mailserver/docker-mailserver/pull/3464)) +- **CI / Automation:** + - `.gitattributes` now ensures files are committed with `eol=lf` ([#3527](https://github.com/docker-mailserver/docker-mailserver/pull/3527)) + - Revised the GitHub issue bug report template ([#3317](https://github.com/docker-mailserver/docker-mailserver/pull/3317), [#3381](https://github.com/docker-mailserver/docker-mailserver/pull/3381), [#3435](https://github.com/docker-mailserver/docker-mailserver/pull/3435)) + - Clarified that the issue tracker is not for personal support ([#3498](https://github.com/docker-mailserver/docker-mailserver/pull/3498), [#3502](https://github.com/docker-mailserver/docker-mailserver/pull/3502)) + - Bumped versions of miscellaneous software (also shoutout to @dependabot) ([#3371](https://github.com/docker-mailserver/docker-mailserver/pull/3371), [#3584](https://github.com/docker-mailserver/docker-mailserver/pull/3584), [#3504](https://github.com/docker-mailserver/docker-mailserver/pull/3504), [#3516](https://github.com/docker-mailserver/docker-mailserver/pull/3516)) +- **Tests:** + - Refactored LDAP tests to current conventions ([#3483](https://github.com/docker-mailserver/docker-mailserver/pull/3483)) + - Changed OpenLDAP image to `bitnami/openldap` ([#3494](https://github.com/docker-mailserver/docker-mailserver/pull/3494)) + - Revised LDAP config + setup ([#3514](https://github.com/docker-mailserver/docker-mailserver/pull/3514)) + - Added tests for the helper function `_add_to_or_update_postfix_main()` ([#3505](https://github.com/docker-mailserver/docker-mailserver/pull/3505)) + - EditorConfig Checker lint now uses a mount path to `/check` instead of `/ci` ([#3655](https://github.com/docker-mailserver/docker-mailserver/pull/3655)) + +### Fixed + +- **Security:** + - Fixed issue with concatenating `$dmarc_milter` and `$dkim_milter` in `main.cf` ([#3380](https://github.com/docker-mailserver/docker-mailserver/pull/3380)) + - Fixed Rspamd DKIM signing for inbound emails ([#3439](https://github.com/docker-mailserver/docker-mailserver/pull/3439), [#3453](https://github.com/docker-mailserver/docker-mailserver/pull/3453)) + - OpenDKIM key generation is no longer broken when Rspamd is also enabled ([#3535](https://github.com/docker-mailserver/docker-mailserver/pull/3535)) +- **Internal:** + - The "database" files (_for managing users and aliases_) now correctly filters within lookup query ([#3359](https://github.com/docker-mailserver/docker-mailserver/pull/3359)) + - `_setup_spam_to_junk()` no longer registered when `SMTP_ONLY=1` ([#3385](https://github.com/docker-mailserver/docker-mailserver/pull/3385)) + - Dovecot `fts_xapian` is now compiled from source to match the Dovecot package ABI ([#3373](https://github.com/docker-mailserver/docker-mailserver/pull/3373)) +- **CI:** + - Scheduled build now have the correct permissions to run successfully ([#3345](https://github.com/docker-mailserver/docker-mailserver/pull/3345)) +- **Documentation:** + - Miscellaneous spelling and wording improvements ([#3324](https://github.com/docker-mailserver/docker-mailserver/pull/3324), [#3330](https://github.com/docker-mailserver/docker-mailserver/pull/3330), [#3337](https://github.com/docker-mailserver/docker-mailserver/pull/3337), [#3339](https://github.com/docker-mailserver/docker-mailserver/pull/3339), [#3344](https://github.com/docker-mailserver/docker-mailserver/pull/3344), [#3367](https://github.com/docker-mailserver/docker-mailserver/pull/3367), [#3411](https://github.com/docker-mailserver/docker-mailserver/pull/3411), [#3443](https://github.com/docker-mailserver/docker-mailserver/pull/3443)) +- **Tests:** + - Run `pgrep` within the actual container ([#3553](https://github.com/docker-mailserver/docker-mailserver/pull/3553)) + - `lmtp_ip.bats` improved partial failure output ([#3552](https://github.com/docker-mailserver/docker-mailserver/pull/3552)) + - Improvements to LDIF test data ([#3506](https://github.com/docker-mailserver/docker-mailserver/pull/3506)) + - Normalized for `.gitattributes` + improved `eclint` coverage ([#3566](https://github.com/docker-mailserver/docker-mailserver/pull/3566)) + - Fixed ShellCheck linting for BATS tests ([#3347](https://github.com/docker-mailserver/docker-mailserver/pull/3347)) ## [v12.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v12.1.0) @@ -38,7 +139,7 @@ All notable changes to this project will be documented in this file. The format - add option to re-enable `reject_unknown_client_hostname` after #3248 ([#3255](https://github.com/docker-mailserver/docker-mailserver/pull/3255)) - add DKIM helper script ([#3286](https://github.com/docker-mailserver/docker-mailserver/pull/3286)) - make `policyd-spf` configurable ([#3246](https://github.com/docker-mailserver/docker-mailserver/pull/3246)) -- add 'log' command to setup for Fail2Ban ([#3299](https://github.com/docker-mailserver/docker-mailserver/pull/3299)) +- add 'log' command to set up for Fail2Ban ([#3299](https://github.com/docker-mailserver/docker-mailserver/pull/3299)) - `setup` command now expects accounts and aliases to be mutually exclusive ([#3270](https://github.com/docker-mailserver/docker-mailserver/pull/3270)) ### Updated diff --git a/VERSION b/VERSION index 77903b35..02161ca8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -12.1.0 +13.0.0 From 2c602299136fcac8a8fd02ada86039687e6ae18c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 26 Nov 2023 20:15:14 +0100 Subject: [PATCH 068/125] docs: updated `CONTRIBUTORS.md` (#3656) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 115 +++++++++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 893b9572..46e0523f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1180,6 +1180,13 @@ Thanks goes to these wonderful people ✨ jcalfee + + + mivek +
+ mivek + + init-js @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
JiLleON - + + jirislav
jirislav - - + jmccl @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
akkumar - + + KCrawley
KCrawley - - + khuedoan @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
luke- - + + LucidityCrash
LucidityCrash - - + MadsRC @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
michaeljensen - + + exhuma
exhuma - - + milas @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
naveensrinivasan - + + neuralp
neuralp - - + radicand @@ -1415,15 +1422,15 @@ Thanks goes to these wonderful people ✨
OrvilleQ - + + ovidiucp
ovidiucp - - + mrPjer @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
romansey - + + MightySCollins
MightySCollins - - + 501st-alpha1 @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
shyim - + + sjmudd
sjmudd - - + simonsystem @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
syl20bnr - + + sylvaindumont
sylvaindumont - - + TechnicLab @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
torus - + + VictorKoenders
VictorKoenders - - + Twist235 @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
42wim - + + ShiriNmi1520
ShiriNmi1520 - - + Zepmann @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
brainkiller - + + cternes
cternes - - + dborowy @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
helmutundarnold - + + hnws
hnws - - + i-C-o-d-e-r @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
paralax - + + jpduyx
jpduyx - - + landergate @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
mchamplain - + + millerjason
millerjason - - + mplx @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
ontheair81 - + + pravynandas
pravynandas - - + presocratics @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
schnippl0r - + + smargold476
smargold476 - - + sportshead @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
vivacarvajalito - + + wligtenberg
wligtenberg - - + wolkenschieber From 68a43eb4970f2ab7680ffa470a1d26e37fa375f0 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:44:47 +0100 Subject: [PATCH 069/125] ci: push `:edge` when `VERSION` is updated (#3662) Previously, we did not run the workflow on push on `master` when a release happened because the push on master is guarded by a check on which files were changed. With this change, I added `VERSION` to the list of files to consider when updating `:edge`. --- .github/workflows/default_on_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/default_on_push.yml b/.github/workflows/default_on_push.yml index a5983989..20921984 100644 --- a/.github/workflows/default_on_push.yml +++ b/.github/workflows/default_on_push.yml @@ -11,6 +11,7 @@ on: - .gitmodules - Dockerfile - setup.sh + - VERSION # also update :edge when a release happens tags: - '*.*.*' From b037288e5ad20b11fab4f375eaf0fe51f2783cd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:22:17 +0100 Subject: [PATCH 070/125] chore(deps): Bump anchore/scan-action from 3.3.6 to 3.3.7 (#3667) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b39cced8..cad2ac41 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.6 + uses: anchore/scan-action@v3.3.7 id: scan with: image: mailserver-testing:ci From a11951e39801dac78628a5ad15d2bc15d4f24e7e Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:33:29 +0100 Subject: [PATCH 071/125] hotfix: solve #3665 (#3669) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 14 +++++++++++++- docs/content/config/environment.md | 4 ++++ mailserver.env | 2 ++ target/rspamd/local.d/settings.conf | 2 +- target/scripts/startup/setup.d/security/rspamd.sh | 2 +- .../parallel/set1/spam_virus/rspamd_full.bats | 2 +- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c9de3f..a8544b80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) + +This patch release fixes two bugs that Rspamd users encounter on `v13.0.0`. Big thanks to the those that helped to identify these issues! + +### Fixed + +- **Rspamd:** + - The check for correct permission on the private key when signing e-mails with DKIM was flawed. The result was that a false warning was emitted ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)) + - When [`RSPAMD_CHECK_AUTHENTICATED=0`][docs::env-rspamd-check-auth], DKIM signing for outbound e-mail was disabled, which is undesirable ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)). **Make sure to check the documentation of [`RSPAMD_CHECK_AUTHENTICATED`][docs::env-rspamd-check-auth]**! + +[docs::env-rspamd-check-auth]: https://docker-mailserver.github.io/docker-mailserver/v13.0/config/environment/#rspamd_check_authenticated + ## [v13.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.0) ### Breaking @@ -78,7 +90,7 @@ All notable changes to this project will be documented in this file. The format - `logrotate` setup + Rspamd log path + tests log helper fallback path ([#3576](https://github.com/docker-mailserver/docker-mailserver/pull/3576)) - Setup during container startup is now more resilient ([#3578](https://github.com/docker-mailserver/docker-mailserver/pull/3578)) - Changed DKIM default config location ([#3597](https://github.com/docker-mailserver/docker-mailserver/pull/3597)) - - Removed the symlink for the `override.d/` directory in favor of using `cp`, integrated into the changedetector service, , added a `--force` option for the Rspamd DKIM management, and provided a dedicated helper script for common ENV variables ([#3599](https://github.com/docker-mailserver/docker-mailserver/pull/3599)) + - Removed the symlink for the `override.d/` directory in favor of using `cp`, integrated into the changedetector service, added a `--force` option for the Rspamd DKIM management, and provided a dedicated helper script for common ENV variables ([#3599](https://github.com/docker-mailserver/docker-mailserver/pull/3599)) - Required permissions are now verified for DKIM private key files ([#3627](https://github.com/docker-mailserver/docker-mailserver/pull/3627)) - **Documentation:** - Documentation aligned to Compose v2 conventions, `docker-compose` command changed to `docker compose`, `docker-compose.yaml` to `compose.yaml` ([#3295](https://github.com/docker-mailserver/docker-mailserver/pull/3295)) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 284549f1..b8e257cc 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -366,6 +366,10 @@ The purpose of this setting is to opt-out of starting an internal Redis instance This settings controls whether checks should be performed on emails coming from authenticated users (i.e. most likely outgoing emails). The default value is `0` in order to align better with SpamAssassin. **We recommend** reading through [the Rspamd documentation on scanning outbound emails][rspamd-scanning-outbound] though to decide for yourself whether you need and want this feature. +!!! note "Not all checks and actions are disabled" + + DKIM signing of e-mails will still happen. + - **0** => No checks will be performed for authenticated users - 1 => All default checks will be performed for authenticated users diff --git a/mailserver.env b/mailserver.env index 957a632e..68786224 100644 --- a/mailserver.env +++ b/mailserver.env @@ -153,6 +153,8 @@ RSPAMD_LEARN=0 # is `0` in order to align better with SpamAssassin. We recommend reading # through https://rspamd.com/doc/tutorials/scanning_outbound.html though to # decide for yourself whether you need and want this feature. +# +# Note that DKIM signing of e-mails will still happen. RSPAMD_CHECK_AUTHENTICATED=0 # Controls whether the Rspamd Greylisting module is enabled. diff --git a/target/rspamd/local.d/settings.conf b/target/rspamd/local.d/settings.conf index 4f635e74..10c4de88 100644 --- a/target/rspamd/local.d/settings.conf +++ b/target/rspamd/local.d/settings.conf @@ -6,7 +6,7 @@ authenticated { priority = high; authenticated = yes; apply { - groups_enabled = []; + groups_enabled = [dkim]; } } # DMS::SED_TAG::1::END diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 239397e5..86786932 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -325,7 +325,7 @@ function __rspamd__check_dkim_permissions() { __rspamd__log 'trace' "Checking DKIM file '${FILE}'" # See https://serverfault.com/a/829314 for an explanation on `-exec false {} +` # We additionally resolve symbolic links to check the permissions of the actual files - if find "$(realpath -eL "${FILE}")" -user _rspamd -or -group _rspamd -or -perm -o=r -exec false {} +; then + if find "$(realpath -eL "${FILE}")" \( -user _rspamd -or -group _rspamd -or -perm -o=r \) -exec false {} +; then __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' does not appear to have correct permissions/ownership for Rspamd to use it" else __rspamd__log 'trace' "DKIM file '${FILE}' permissions and ownership appear correct" diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 09d42d46..ba8a23f5 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -307,5 +307,5 @@ function teardown_file() { _default_teardown ; } _run_in_container grep -E -A 6 'authenticated \{' "${MODULE_FILE}" assert_success assert_output --partial 'authenticated = yes;' - assert_output --partial 'groups_enabled = [];' + assert_output --partial 'groups_enabled = [dkim];' } From 19e96b5131ba935a0e54e554c3f3a0e6fc66f3b4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 30 Nov 2023 10:21:26 +1300 Subject: [PATCH 072/125] fix: `update-check.sh` should query GH Releases (#3666) * fix: Source `VERSION` from image ENV Now CI builds triggered from tagged releases will always have the correct version. No need for manually updating a separate file. * fix: Query latest GH release tag Compare to the remote GH release tag published, rather than contents of a `VERSION` file. `VERSION` file remains in source for now as prior releases still rely on it for an update notification. * chore: Switch from `yq` to `jaq` - Can more easily express a string subslice. - Lighter weight: 9.3M vs 1.7M. - Drawback, no YAML input/output support. If `yq` is preferred, the `v` prefix could be removed via BASH easily enough. * chore: Add entry to `CHANGELOG.md` * ci: `VERSION` has no relevance to `:edge` * docs: Update build guide + simplify `make build` --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- .github/workflows/default_on_push.yml | 1 - .github/workflows/generic_publish.yml | 7 +---- CHANGELOG.md | 2 ++ Dockerfile | 5 ++-- Makefile | 6 +---- .../examples/tutorials/docker-build.md | 27 ++++++++++++------- target/scripts/build/packages.sh | 6 +++++ target/scripts/start-mailserver.sh | 2 +- target/scripts/update-check.sh | 7 ++--- 9 files changed, 35 insertions(+), 28 deletions(-) diff --git a/.github/workflows/default_on_push.yml b/.github/workflows/default_on_push.yml index 20921984..a5983989 100644 --- a/.github/workflows/default_on_push.yml +++ b/.github/workflows/default_on_push.yml @@ -11,7 +11,6 @@ on: - .gitmodules - Dockerfile - setup.sh - - VERSION # also update :edge when a release happens tags: - '*.*.*' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 0ed2fd3e..6df534ef 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -66,18 +66,13 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: 'Acquire the image version' - id: get-version - shell: bash - run: echo "version=$(>"${GITHUB_OUTPUT}" - - name: 'Build and publish images' uses: docker/build-push-action@v5.1.0 with: context: . build-args: | + DMS_RELEASE=${{ github.ref_type == 'tag' && github.ref_name || 'edge' }} VCS_REVISION=${{ github.sha }} - VCS_VERSION=${{ steps.get-version.outputs.version }} platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.prep.outputs.tags }} diff --git a/CHANGELOG.md b/CHANGELOG.md index a8544b80..67aa3ec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ This patch release fixes two bugs that Rspamd users encounter on `v13.0.0`. Big ### Fixed +- **Internal:** + - The update check service now queries the latest GH release for a version tag instead of a `VERSION` file from the repo. - **Rspamd:** - The check for correct permission on the private key when signing e-mails with DKIM was flawed. The result was that a false warning was emitted ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)) - When [`RSPAMD_CHECK_AUTHENTICATED=0`][docs::env-rspamd-check-auth], DKIM signing for outbound e-mail was disabled, which is undesirable ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)). **Make sure to check the documentation of [`RSPAMD_CHECK_AUTHENTICATED`][docs::env-rspamd-check-auth]**! diff --git a/Dockerfile b/Dockerfile index 5e12689d..0f19521a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -295,8 +295,8 @@ COPY target/scripts/startup/setup.d /usr/local/bin/setup.d # FROM stage-main AS stage-final +ARG DMS_RELEASE=edge ARG VCS_REVISION=unknown -ARG VCS_VERSION=edge WORKDIR / EXPOSE 25 587 143 465 993 110 995 4190 @@ -327,4 +327,5 @@ LABEL org.opencontainers.image.source="https://github.com/docker-mailserver/dock # ARG invalidates cache when it is used by a layer (implicitly affects RUN) # Thus to maximize cache, keep these lines last: LABEL org.opencontainers.image.revision=${VCS_REVISION} -LABEL org.opencontainers.image.version=${VCS_VERSION} +LABEL org.opencontainers.image.version=${DMS_RELEASE} +ENV DMS_RELEASE=${DMS_RELEASE} diff --git a/Makefile b/Makefile index 5732cc07..0962c11a 100644 --- a/Makefile +++ b/Makefile @@ -18,11 +18,7 @@ BATS_PARALLEL_JOBS ?= 2 all: lint build generate-accounts tests clean build: ALWAYS_RUN - @ DOCKER_BUILDKIT=1 docker build \ - --tag $(IMAGE_NAME) \ - --build-arg VCS_VERSION=$(shell git rev-parse --short HEAD) \ - --build-arg VCS_REVISION=$(shell cat VERSION) \ - . + @ docker build --tag $(IMAGE_NAME) . generate-accounts: ALWAYS_RUN @ cp test/config/templates/postfix-accounts.cf test/config/postfix-accounts.cf diff --git a/docs/content/examples/tutorials/docker-build.md b/docs/content/examples/tutorials/docker-build.md index fc6d5c37..538da822 100644 --- a/docs/content/examples/tutorials/docker-build.md +++ b/docs/content/examples/tutorials/docker-build.md @@ -10,7 +10,7 @@ You'll need to retrieve the git submodules prior to building your own Docker ima ```sh git submodule update --init --recursive -docker build -t . +docker build --tag . ``` Or, you can clone and retrieve the submodules in one command: @@ -21,19 +21,26 @@ git clone --recurse-submodules https://github.com/docker-mailserver/docker-mails ### About Docker -#### Version +#### Minimum supported version -We make use of build-features that require a recent version of Docker. Depending on your distribution, please have a look at [the official installation documentation for Docker](https://docs.docker.com/engine/install/) to get the latest version. Otherwise, you may encounter issues, for example with the `--link` flag for a [`#!dockerfile COPY`](https://docs.docker.com/engine/reference/builder/#copy) command. +We make use of build features that require a recent version of Docker. v23.0 or newer is advised, but earlier releases may work. -#### Environment +- To get the latest version for your distribution, please have a look at [the official installation documentation for Docker](https://docs.docker.com/engine/install/). +- If you are using a version of Docker prior to v23.0, you will need to enable BuildKit via the ENV [`DOCKER_BUILDKIT=1`](https://docs.docker.com/build/buildkit/#getting-started). -If you are not using `make` to build the image, note that you will need to provide `DOCKER_BUILDKIT=1` to the `docker build` command for the build to succeed. +#### Build Arguments (Optional) -#### Build Arguments +The `Dockerfile` includes several build [`ARG`][docker-docs::builder-arg] instructions that can be configured: -The `Dockerfile` takes additional, so-called build arguments. These are +- `DOVECOT_COMMUNITY_REPO`: Install Dovecot from the community repo instead of from Debian (default = 1) +- `DMS_RELEASE`: The image version (default = edge) +- `VCS_REVISION`: The git commit hash used for the build (default = unknown) -1. `VCS_VERSION`: the image version (default = edge) -2. `VCS_REVISION`: the image revision (default = unknown) +!!! note -When using `make` to build the image, these are filled with proper values. You can build the image without supplying these arguments just fine though. + - `DMS_RELEASE` (_when not `edge`_) will be used to check for updates from our GH releases page at runtime due to the default feature [`ENABLE_UPDATE_CHECK=1`][docs::env-update-check]. + - Both `DMS_RELEASE` and `VCS_REVISION` are also used with `opencontainers` metadata [`LABEL`][docker-docs::builder-label] instructions. + +[docs::env-update-check]: https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#enable_update_check +[docker-docs::builder-arg]: https://docs.docker.com/engine/reference/builder/#using-arg-variables +[docker-docs::builder-label]: https://docs.docker.com/engine/reference/builder/#label diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index a025c3b4..97ebae04 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -205,6 +205,11 @@ function _install_getmail() { apt-get "${QUIET}" autoremove } +function _install_utils() { + _log 'debug' 'Installing utils sourced from Github' + curl -sL https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-x86_64-unknown-linux-musl -o /usr/bin/jaq && chmod +x /usr/bin/jaq +} + function _remove_data_after_package_installations() { _log 'debug' 'Deleting sensitive files (secrets)' rm /etc/postsrsd.secret @@ -228,5 +233,6 @@ _install_dovecot _install_rspamd _install_fail2ban _install_getmail +_install_utils _remove_data_after_package_installations _post_installation_steps diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index f0f385f3..cc6c2244 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -120,7 +120,7 @@ function _register_functions() { [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' - [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check' + [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && [[ ${DMS_RELEASE} != 'edge' ]] && _register_start_daemon '_start_daemon_update_check' # The order here matters: Since Rspamd is using Redis, Redis should be started before Rspamd. [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd_redis' diff --git a/target/scripts/update-check.sh b/target/scripts/update-check.sh index 9010371f..c30594f4 100755 --- a/target/scripts/update-check.sh +++ b/target/scripts/update-check.sh @@ -3,8 +3,8 @@ # shellcheck source=./helpers/log.sh source /usr/local/bin/helpers/log.sh -VERSION=$( Date: Thu, 30 Nov 2023 14:47:31 +1300 Subject: [PATCH 073/125] fix: Logging - Welcome should use `DMS_RELEASE` ENV (#3676) --- CHANGELOG.md | 10 ++++++++-- VERSION | 2 +- target/scripts/start-mailserver.sh | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67aa3ec0..f7fdb981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,20 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Fixed + +- **Internal**: + - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + ## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) -This patch release fixes two bugs that Rspamd users encounter on `v13.0.0`. Big thanks to the those that helped to identify these issues! +This patch release fixes two bugs that Rspamd users encountered with the `v13.0.0` release. Big thanks to the those that helped to identify these issues! ❤️ ### Fixed - **Internal:** - - The update check service now queries the latest GH release for a version tag instead of a `VERSION` file from the repo. + - The update check service now queries the latest GH release for a version tag (_instead of from a `VERSION` file at the GH repo_). This should provide more reliable update notifications ([#3666](https://github.com/docker-mailserver/docker-mailserver/pull/3666)) - **Rspamd:** - The check for correct permission on the private key when signing e-mails with DKIM was flawed. The result was that a false warning was emitted ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)) - When [`RSPAMD_CHECK_AUTHENTICATED=0`][docs::env-rspamd-check-auth], DKIM signing for outbound e-mail was disabled, which is undesirable ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)). **Make sure to check the documentation of [`RSPAMD_CHECK_AUTHENTICATED`][docs::env-rspamd-check-auth]**! diff --git a/VERSION b/VERSION index 02161ca8..5cb7d856 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.0.0 +13.0.1 diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index cc6c2244..fa8214e1 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -153,7 +153,7 @@ function _register_functions() { _early_supervisor_setup _early_variables_setup -_log 'info' "Welcome to docker-mailserver $( Date: Sun, 3 Dec 2023 22:28:40 +0100 Subject: [PATCH 074/125] ci: add `run-local-instance` target to `Makefile` (#3663) --- CHANGELOG.md | 4 ++++ Makefile | 16 ++++++++++++++++ docs/content/contributing/tests.md | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7fdb981..e6faa74a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Added + +- command (`run-local-instance`) to test a version of DMS that was built locally to test changes + ### Fixed - **Internal**: diff --git a/Makefile b/Makefile index 0962c11a..37267d2c 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,22 @@ clean: ALWAYS_RUN -@ while read -r LINE; do [[ $${LINE} =~ test/.+ ]] && FILES+=("/mnt$${LINE#test}"); done < .gitignore ; \ docker run --rm -v "$(REPOSITORY_ROOT)/test/:/mnt" alpine ash -c "rm -rf $${FILES[@]}" +run-local-instance: ALWAYS_RUN + bash -c 'sleep 8 ; ./setup.sh email add postmaster@example.test 123' & + docker run --rm --interactive --tty --name dms-test_example \ + --env OVERRIDE_HOSTNAME=mail.example.test \ + --env POSTFIX_INET_PROTOCOLS=ipv4 \ + --env DOVECOT_INET_PROTOCOLS=ipv4 \ + --env ENABLE_CLAMAV=0 \ + --env ENABLE_AMAVIS=0 \ + --env ENABLE_RSPAMD=0 \ + --env ENABLE_OPENDKIM=0 \ + --env ENABLE_OPENDMARC=0 \ + --env ENABLE_POLICYD_SPF=0 \ + --env ENABLE_SPAMASSASSIN=0 \ + --env LOG_LEVEL=trace \ + $(IMAGE_NAME) + # ----------------------------------------------- # --- Tests ------------------------------------ # ----------------------------------------------- diff --git a/docs/content/contributing/tests.md b/docs/content/contributing/tests.md index 6f649529..8816a228 100644 --- a/docs/content/contributing/tests.md +++ b/docs/content/contributing/tests.md @@ -78,6 +78,10 @@ We use `make` to run commands. When writing tests, ensure that parallel set tests still pass when run in parallel. You need to account for other tests running in parallel that may interfere with your own tests logic. +!!! tip + + You may use `make run-local-instance` to run a version of the image built locally to test and edit your changes in a running DMS instance. + ### An Example In this example, you've made a change to the Rspamd feature support (_or adjusted it's tests_). First verify no regressions have been introduced by running it's specific test file: From 01689ab788022097e79d80979303e206a5d48f24 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:22:43 +1300 Subject: [PATCH 075/125] docs: Troubleshooting - Bare domain misconfiguration (#3680) --- CHANGELOG.md | 8 +++++++- docs/content/config/debugging.md | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6faa74a..8ecfbe55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,13 @@ All notable changes to this project will be documented in this file. The format ### Added -- command (`run-local-instance`) to test a version of DMS that was built locally to test changes +- **Tests:** + - You can now use `make run-local-instance` to run a DMS image that was built locally to test changes ([#3663](https://github.com/docker-mailserver/docker-mailserver/pull/3663)) + +### Updates + +- **Documentation:** + - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) ### Fixed diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index d1e29376..9c3bebb5 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -46,6 +46,15 @@ Some service providers block outbound traffic on port 25. Common hosting provide These links may advise how the provider can unblock the port through additional services offered, or via a support ticket request. +### Mail sent to DMS does not get delivered to user + +Common logs related to this are: + +- `warning: do not list domain domain.fr in BOTH mydestination and virtual_mailbox_domains` +- `Recipient address rejected: User unknown in local recipient table` + +If your logs look like this, you likely have [assigned the same FQDN to the DMS `hostname` and your mail accounts][gh-issues::dms-fqdn-misconfigured] which is not supported by default. You can either adjust your DMS `hostname` or follow [this FAQ advice][docs::faq-bare-domain] + ## Steps for Debugging DMS 1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. @@ -109,12 +118,14 @@ This could be from outdated software, or running a system that isn't able to pro [docs-environment-log-level]: ./environment.md#log_level [docs-faq]: ../faq.md +[docs::faq-bare-domain]: ../faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname [docs-ipv6]: ./advanced/ipv6.md [docs-introduction]: ../introduction.md [docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container [docs-usage]: ../usage.md [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues +[gh-issues::dms-fqdn-misconfigured]: https://github.com/docker-mailserver/docker-mailserver/issues/3679#issuecomment-1837609043 [gh-macos-support]: https://github.com/docker-mailserver/docker-mailserver/issues/3648#issuecomment-1822774080 [gh-discuss-roundcube-fail2ban]: https://github.com/orgs/docker-mailserver/discussions/3273#discussioncomment-5654603 From 7ce745a82dfcb92c1527cc21f4bb4d78d412a292 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:32:49 +0100 Subject: [PATCH 076/125] chore(deps): Bump docker/metadata-action from 5.0.0 to 5.3.0 (#3683) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 6df534ef..36b44770 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.0.0 + uses: docker/metadata-action@v5.3.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 1ff8d57ea1bd2d318240be71d7da760784ea9969 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:34:20 +0000 Subject: [PATCH 077/125] chore(deps): Bump anchore/scan-action from 3.3.7 to 3.3.8 (#3682) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index cad2ac41..95245557 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.7 + uses: anchore/scan-action@v3.3.8 id: scan with: image: mailserver-testing:ci From c75975d59e7cb4d8b25b67792c4b74b60b35545d Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:16:39 +1300 Subject: [PATCH 078/125] chore: Postfix should integrate Dovecot at runtime (#3681) * chore: Better establish startup scope * chore: Configure `main.cf` for Dovecot at runtime --- CHANGELOG.md | 2 ++ target/postfix/main.cf | 6 ++-- target/scripts/helpers/aliases.sh | 1 + target/scripts/start-mailserver.sh | 12 ++++---- target/scripts/startup/setup.d/postfix.sh | 30 +++++++++++++++---- .../startup/setup.d/security/spoofing.sh | 3 ++ 6 files changed, 40 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ecfbe55..5d539271 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ All notable changes to this project will be documented in this file. The format - **Documentation:** - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) +- **Internal:** + - Postfix configures `virtual_mailbox_maps` and `virtual_transport` during startup instead of using defaults (configured for Dovecot) via our `main.cf` ([#3681](https://github.com/docker-mailserver/docker-mailserver/pull/3681)) ### Fixed diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 405dc0fb..8c329c94 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -88,10 +88,10 @@ smtpd_sasl_security_options = noanonymous smtpd_sasl_local_domain = $mydomain broken_sasl_auth_clients = yes -# Mail directory -virtual_transport = lmtp:unix:/var/run/dovecot/lmtp +# Postfix lookup tables for verifying valid users and managed mail domains: +# Populated during startup in: scripts/helpers/postfix.sh virtual_mailbox_domains = /etc/postfix/vhost -virtual_mailbox_maps = texthash:/etc/postfix/vmailbox +# Populated during startup in: scripts/helpers/aliases.sh virtual_alias_maps = texthash:/etc/postfix/virtual # Milters used by DKIM diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 0890d994..04a56da3 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -23,6 +23,7 @@ function _handle_postfix_virtual_config() { fi } +# TODO: Investigate why this file is always created, nothing seems to append only the cp below? function _handle_postfix_regexp_config() { : >/etc/postfix/regexp diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index fa8214e1..aadac2b5 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -91,20 +91,22 @@ function _register_functions() { _register_setup_function '_setup_dovecot_hostname' _register_setup_function '_setup_postfix_early' - _register_setup_function '_setup_fetchmail' - _register_setup_function '_setup_fetchmail_parallel' - # needs to come after _setup_postfix_early + # Dependent upon _setup_postfix_early first calling _create_aliases + # Due to conditional check for /etc/postfix/regexp _register_setup_function '_setup_spoof_protection' - _register_setup_function '_setup_getmail' + _register_setup_function '_setup_postfix_late' if [[ ${ENABLE_SRS} -eq 1 ]]; then _register_setup_function '_setup_SRS' _register_start_daemon '_start_daemon_postsrsd' fi - _register_setup_function '_setup_postfix_late' + _register_setup_function '_setup_fetchmail' + _register_setup_function '_setup_fetchmail_parallel' + _register_setup_function '_setup_getmail' + _register_setup_function '_setup_logrotate' _register_setup_function '_setup_mail_summary' _register_setup_function '_setup_logwatch' diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 0d7cb1ae..126a195c 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -30,18 +30,25 @@ mech_list: plain login EOF fi + # User has explicitly requested to disable SASL auth: + # TODO: Additive config by feature would be better. Should only enable SASL auth + # on submission(s) services in master.cf when SASLAuthd or Dovecot is enabled. if [[ ${ENABLE_SASLAUTHD} -eq 0 ]] && [[ ${SMTP_ONLY} -eq 1 ]]; then + # Default for services (eg: Port 25); NOTE: This has since become the default: sed -i -E \ 's|^smtpd_sasl_auth_enable =.*|smtpd_sasl_auth_enable = no|g' \ /etc/postfix/main.cf + # Submission services that are explicitly enabled by default: sed -i -E \ 's|^ -o smtpd_sasl_auth_enable=.*| -o smtpd_sasl_auth_enable=no|g' \ /etc/postfix/master.cf fi + # scripts/helpers/aliases.sh:_create_aliases() __postfix__log 'trace' 'Setting up aliases' _create_aliases + # scripts/helpers/postfix.sh:_create_postfix_vhost() __postfix__log 'trace' 'Setting up Postfix vhost' _create_postfix_vhost @@ -63,6 +70,23 @@ EOF 's|^(dms_smtpd_sender_restrictions = .*)|\1, reject_unknown_client_hostname|' \ /etc/postfix/main.cf fi + + # Dovecot feature integration + # TODO: Alias SMTP_ONLY=0 to DOVECOT_ENABLED=1? + if [[ ${SMTP_ONLY} -ne 1 ]]; then + __postfix__log 'trace' 'Configuring Postfix with Dovecot integration' + + # /etc/postfix/vmailbox is created by: scripts/helpers/accounts.sh:_create_accounts() + # This file config is for Postfix to verify a mail account exists before accepting + # mail arriving and delivering it to Dovecot over LMTP. + postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox' + postconf 'virtual_transport = lmtp:unix:/var/run/dovecot/lmtp' + fi + + if [[ -n ${POSTFIX_DAGENT} ]]; then + __postfix__log 'trace' "Changing virtual transport to '${POSTFIX_DAGENT}'" + postconf "virtual_transport = ${POSTFIX_DAGENT}" + fi } function _setup_postfix_late() { @@ -80,12 +104,6 @@ function _setup_postfix_late() { __postfix__log 'trace' 'Configuring relay host' _setup_relayhost - if [[ -n ${POSTFIX_DAGENT} ]]; then - __postfix__log 'trace' "Changing virtual transport to '${POSTFIX_DAGENT}'" - # Default value in main.cf should be 'lmtp:unix:/var/run/dovecot/lmtp' - postconf "virtual_transport = ${POSTFIX_DAGENT}" - fi - __postfix__setup_override_configuration } diff --git a/target/scripts/startup/setup.d/security/spoofing.sh b/target/scripts/startup/setup.d/security/spoofing.sh index 7c38821d..ffefb279 100644 --- a/target/scripts/startup/setup.d/security/spoofing.sh +++ b/target/scripts/startup/setup.d/security/spoofing.sh @@ -11,6 +11,9 @@ function _setup_spoof_protection() { postconf 'smtpd_sender_login_maps = ldap:/etc/postfix/ldap-senders.cf' fi else + # NOTE: This file is always created at startup, it potentially has content added. + # TODO: From section: "SPOOF_PROTECTION=1 handling for smtpd_sender_login_maps" + # https://github.com/docker-mailserver/docker-mailserver/issues/2819#issue-1402114383 if [[ -f /etc/postfix/regexp ]]; then postconf 'smtpd_sender_login_maps = unionmap:{ texthash:/etc/postfix/virtual, hash:/etc/aliases, pcre:/etc/postfix/maps/sender_login_maps.pcre, pcre:/etc/postfix/regexp }' else From 908d38047ce4be6dd1e8ca5021d0bce86625a3c3 Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 5 Dec 2023 21:42:30 +0100 Subject: [PATCH 079/125] scripts: add warning when update-check is enabled, but no stable release image is used (#3684) --- CHANGELOG.md | 1 + target/scripts/start-mailserver.sh | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d539271..18ef5057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. The format - **Tests:** - You can now use `make run-local-instance` to run a DMS image that was built locally to test changes ([#3663](https://github.com/docker-mailserver/docker-mailserver/pull/3663)) +- Log a warning when update-check is enabled, but no stable release image is used ([#3684](https://github.com/docker-mailserver/docker-mailserver/pull/3684)) ### Updates diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index aadac2b5..2129b74a 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -122,7 +122,13 @@ function _register_functions() { [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' - [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && [[ ${DMS_RELEASE} != 'edge' ]] && _register_start_daemon '_start_daemon_update_check' + if [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]]; then + if [[ ${DMS_RELEASE} != 'edge' ]]; then + _register_start_daemon '_start_daemon_update_check' + else + _log 'warn' "ENABLE_UPDATE_CHECK=1 is configured, but image is not a stable release. Update-Check is disabled." + fi + fi # The order here matters: Since Rspamd is using Redis, Redis should be started before Rspamd. [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd_redis' From 77917f5cc64b444db2a76def92161186db66670b Mon Sep 17 00:00:00 2001 From: Peter Adam Date: Thu, 7 Dec 2023 23:45:02 +0100 Subject: [PATCH 080/125] scripts: Install arm64 rspamd from official repository (#3686) * scripts: Install rspamd from official repository instead of debian backports on arm64 architecture * Remove unnecessary deb-src repository for rspamd * Remove note about ARM64 rspamd version, update CHANGELOG.md --------- Co-authored-by: Peter Adam --- CHANGELOG.md | 2 ++ docs/content/config/security/rspamd.md | 4 ---- target/scripts/build/packages.sh | 23 ++++------------------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18ef5057..10d2bb25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ All notable changes to this project will be documented in this file. The format - **Internal**: - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) +- **Rspamd:** + - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) ## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 442e5e78..fe9bd5ea 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -8,10 +8,6 @@ Rspamd is a ["fast, free and open-source spam filtering system"][rspamd-homepage If you want to have a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][dms-default-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. -!!! note "AMD64 vs ARM64" - - We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 23rd Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. - [rspamd-homepage]: https://rspamd.com/ [dms-default-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/master/target/rspamd diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 97ebae04..ec468d41 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -130,29 +130,14 @@ function _install_dovecot() { function _install_rspamd() { _log 'trace' 'Adding Rspamd package signatures' local DEB_FILE='/etc/apt/sources.list.d/rspamd.list' - local RSPAMD_PACKAGE_NAME - # We try getting the most recent version of Rspamd for aarch64 (from an official source, which - # is the backports repository). The version for aarch64 is 3.2; the most recent version for amd64 - # that we get with the official PPA is 3.4. - # - # Not removing it later is fine as you have to explicitly opt into installing a backports package - # which is not something you could be doing by accident. - if [[ $(uname --machine) == 'aarch64' ]]; then - echo '# Official Rspamd PPA does not support aarch64, so we use the Bullseye backports' >"${DEB_FILE}" - echo 'deb [arch=arm64] http://deb.debian.org/debian bullseye-backports main' >>"${DEB_FILE}" - RSPAMD_PACKAGE_NAME='rspamd/bullseye-backports' - else - curl -sSfL https://rspamd.com/apt-stable/gpg.key | gpg --dearmor >/etc/apt/trusted.gpg.d/rspamd.gpg - local URL='[arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rspamd.gpg] http://rspamd.com/apt-stable/ bullseye main' - echo "deb ${URL}" >"${DEB_FILE}" - echo "deb-src ${URL}" >>"${DEB_FILE}" - RSPAMD_PACKAGE_NAME='rspamd' - fi + curl -sSfL https://rspamd.com/apt-stable/gpg.key | gpg --dearmor >/etc/apt/trusted.gpg.d/rspamd.gpg + local URL='[signed-by=/etc/apt/trusted.gpg.d/rspamd.gpg] http://rspamd.com/apt-stable/ bullseye main' + echo "deb ${URL}" >"${DEB_FILE}" _log 'debug' 'Installing Rspamd' apt-get "${QUIET}" update - apt-get "${QUIET}" --no-install-recommends install "${RSPAMD_PACKAGE_NAME}" 'redis-server' + apt-get "${QUIET}" --no-install-recommends install 'rspamd' 'redis-server' } function _install_fail2ban() { From d3b4e94d0681b98ea6a7a955051c397cf19c0b88 Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 8 Dec 2023 01:20:17 +0100 Subject: [PATCH 081/125] update-check: fix 'read' exit status (#3688) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 1 + target/scripts/update-check.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d2bb25..3553804f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file. The format - **Internal**: - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) diff --git a/target/scripts/update-check.sh b/target/scripts/update-check.sh index c30594f4..257fc37d 100755 --- a/target/scripts/update-check.sh +++ b/target/scripts/update-check.sh @@ -27,7 +27,7 @@ while true; do # compare versions if dpkg --compare-versions "${VERSION}" lt "${LATEST}"; then # send mail notification to postmaster - read -r -d '' MAIL << EOF + read -r -d '#' MAIL << EOF Hello ${POSTMASTER_ADDRESS}! There is a docker-mailserver update available on your host: $(hostname -f) @@ -35,7 +35,7 @@ There is a docker-mailserver update available on your host: $(hostname -f) Current version: ${VERSION} Latest version: ${LATEST} -Changelog: ${CHANGELOG_URL} +Changelog: ${CHANGELOG_URL}#END EOF _log_with_date 'info' "Update available [ ${VERSION} --> ${LATEST} ]" From bc9172fa74f7307b64b57e5c05ed740a3109ef86 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 10 Dec 2023 21:08:19 +0100 Subject: [PATCH 082/125] docs: updated `CONTRIBUTORS.md` (#3691) --- CONTRIBUTORS.md | 79 +++++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 46e0523f..860f5df1 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1438,6 +1438,13 @@ Thanks goes to these wonderful people ✨ mrPjer + + + p3dda +
+ p3dda + + peter-hartmann @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
remoe - + + romansey
romansey - - + MightySCollins @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
3ap - + + shyim
shyim - - + sjmudd @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
5ven - + + syl20bnr
syl20bnr - - + sylvaindumont @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
tweibert - + + torus
torus - - + VictorKoenders @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
vilisas - + + 42wim
42wim - - + ShiriNmi1520 @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
awb99 - + + brainkiller
brainkiller - - + cternes @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
ghnp5 - + + helmutundarnold
helmutundarnold - - + hnws @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
jjtt - + + paralax
paralax - - + jpduyx @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
matrixes - + + mchamplain
mchamplain - - + millerjason @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
olaf-mandel - + + ontheair81
ontheair81 - - + pravynandas @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
rriski - + + schnippl0r
schnippl0r - - + smargold476 @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
tamueller - + + vivacarvajalito
vivacarvajalito - - + wligtenberg From 2f5dfed7261be9338cce74b4e5ed64763b7ef038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Pl=C3=B6tz?= Date: Sun, 10 Dec 2023 22:22:31 +0100 Subject: [PATCH 083/125] fix: Only set `virtual_mailbox_maps` to `texthash` when using the `FILE` account provisioner (#3693) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: René Plötz --- target/scripts/startup/setup.d/postfix.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 126a195c..58998376 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -79,7 +79,9 @@ EOF # /etc/postfix/vmailbox is created by: scripts/helpers/accounts.sh:_create_accounts() # This file config is for Postfix to verify a mail account exists before accepting # mail arriving and delivering it to Dovecot over LMTP. - postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox' + if [[ ${ACCOUNT_PROVISIONER} == 'FILE' ]]; then + postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox' + fi postconf 'virtual_transport = lmtp:unix:/var/run/dovecot/lmtp' fi From bbed3f66081a838275c54ac32de5d3ceac4f49b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 21:36:30 +0100 Subject: [PATCH 084/125] chore(deps): Bump actions/stale from 8 to 9 (#3696) --- .github/workflows/handle_stalled.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/handle_stalled.yml b/.github/workflows/handle_stalled.yml index 236dcec9..672286fe 100644 --- a/.github/workflows/handle_stalled.yml +++ b/.github/workflows/handle_stalled.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Close stale issues - uses: actions/stale@v8 + uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 20 From 98a4c13ca9c8340c38a2d9ae6aef5607d695a9d5 Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 18 Dec 2023 12:26:28 +0100 Subject: [PATCH 085/125] Add ENV `ENABLE_IMAP` (#3703) --- CHANGELOG.md | 5 ++- docs/content/config/environment.md | 7 ++++- mailserver.env | 10 ++++-- target/scripts/startup/setup.d/dovecot.sh | 38 ++++++++++++++--------- target/scripts/startup/variables-stack.sh | 1 + 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3553804f..339c92e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,12 @@ All notable changes to this project will be documented in this file. The format ### Added +- **Dovecot:** + - ENV `ENABLE_IMAP` ([#3703](https://github.com/docker-mailserver/docker-mailserver/pull/3703)) - **Tests:** - You can now use `make run-local-instance` to run a DMS image that was built locally to test changes ([#3663](https://github.com/docker-mailserver/docker-mailserver/pull/3663)) -- Log a warning when update-check is enabled, but no stable release image is used ([#3684](https://github.com/docker-mailserver/docker-mailserver/pull/3684)) +- **Internal**: + - Log a warning when update-check is enabled, but no stable release image is used ([#3684](https://github.com/docker-mailserver/docker-mailserver/pull/3684)) ### Updates diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index b8e257cc..5ef3b968 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -131,9 +131,14 @@ Enabled `policyd-spf` in Postfix's configuration. You will likely want to set th ##### ENABLE_POP3 -- **empty** => POP3 service disabled +- **0** => POP3 service disabled - 1 => Enables POP3 service +##### ENABLE_IMAP + +- 0 => Disabled +- **1** => Enabled + ##### ENABLE_CLAMAV - **0** => ClamAV is disabled diff --git a/mailserver.env b/mailserver.env index 68786224..48b537bc 100644 --- a/mailserver.env +++ b/mailserver.env @@ -119,10 +119,16 @@ ENABLE_OPENDMARC=1 # - **1** => Enabled ENABLE_POLICYD_SPF=1 -# 1 => Enables POP3 service -# empty => disables POP3 +# Enables POP3 service +# - **0** => Disabled +# - 1 => Enabled ENABLE_POP3= +# Enables IMAP service +# - 0 => Disabled +# - **1** => Enabled +ENABLE_IMAP=1 + # Enables ClamAV, and anti-virus scanner. # 1 => Enabled # **0** => Disabled diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index 3eeda286..e46aca21 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -6,12 +6,10 @@ function _setup_dovecot() { cp -a /usr/share/dovecot/protocols.d /etc/dovecot/ # disable pop3 (it will be eventually enabled later in the script, if requested) mv /etc/dovecot/protocols.d/pop3d.protocol /etc/dovecot/protocols.d/pop3d.protocol.disab + # disable imap (it will be eventually enabled later in the script, if requested) + mv /etc/dovecot/protocols.d/imapd.protocol /etc/dovecot/protocols.d/imapd.protocol.disab mv /etc/dovecot/protocols.d/managesieved.protocol /etc/dovecot/protocols.d/managesieved.protocol.disab - sed -i -e 's|#ssl = yes|ssl = yes|g' /etc/dovecot/conf.d/10-master.conf - sed -i -e 's|#port = 993|port = 993|g' /etc/dovecot/conf.d/10-master.conf - sed -i -e 's|#port = 995|port = 995|g' /etc/dovecot/conf.d/10-master.conf - sed -i -e 's|#ssl = yes|ssl = required|g' /etc/dovecot/conf.d/10-ssl.conf - sed -i 's|^postmaster_address = .*$|postmaster_address = '"${POSTMASTER_ADDRESS}"'|g' /etc/dovecot/conf.d/15-lda.conf + sedfile -i 's|^postmaster_address = .*$|postmaster_address = '"${POSTMASTER_ADDRESS}"'|g' /etc/dovecot/conf.d/15-lda.conf if ! grep -q -E '^stats_writer_socket_path=' /etc/dovecot/dovecot.conf; then printf '\n%s\n' 'stats_writer_socket_path=' >>/etc/dovecot/dovecot.conf @@ -37,9 +35,21 @@ function _setup_dovecot() { esac + if [[ ${ENABLE_POP3} -eq 1 || ${ENABLE_IMAP} -eq 1 ]]; then + sedfile -i -e 's|#ssl = yes|ssl = yes|g' /etc/dovecot/conf.d/10-master.conf + sedfile -i -e 's|#ssl = yes|ssl = required|g' /etc/dovecot/conf.d/10-ssl.conf + fi + if [[ ${ENABLE_POP3} -eq 1 ]]; then _log 'debug' 'Enabling POP3 services' mv /etc/dovecot/protocols.d/pop3d.protocol.disab /etc/dovecot/protocols.d/pop3d.protocol + sedfile -i -e 's|#port = 995|port = 995|g' /etc/dovecot/conf.d/10-master.conf + fi + + if [[ ${ENABLE_IMAP} -eq 1 ]]; then + _log 'debug' 'Enabling IMAP services' + mv /etc/dovecot/protocols.d/imapd.protocol.disab /etc/dovecot/protocols.d/imapd.protocol + sedfile -i -e 's|#port = 993|port = 993|g' /etc/dovecot/conf.d/10-master.conf fi [[ -f /tmp/docker-mailserver/dovecot.cf ]] && cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf @@ -89,23 +99,23 @@ function _setup_dovecot_quota() { # disable dovecot quota in docevot confs if [[ -f /etc/dovecot/conf.d/90-quota.conf ]]; then mv /etc/dovecot/conf.d/90-quota.conf /etc/dovecot/conf.d/90-quota.conf.disab - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins quota|mail_plugins = \$mail_plugins|g" \ /etc/dovecot/conf.d/10-mail.conf - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins imap_quota|mail_plugins = \$mail_plugins|g" \ /etc/dovecot/conf.d/20-imap.conf fi # disable quota policy check in postfix - sed -i "s|check_policy_service inet:localhost:65265||g" /etc/postfix/main.cf + sedfile -i "s|check_policy_service inet:localhost:65265||g" /etc/postfix/main.cf else if [[ -f /etc/dovecot/conf.d/90-quota.conf.disab ]]; then mv /etc/dovecot/conf.d/90-quota.conf.disab /etc/dovecot/conf.d/90-quota.conf - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins|mail_plugins = \$mail_plugins quota|g" \ /etc/dovecot/conf.d/10-mail.conf - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins|mail_plugins = \$mail_plugins imap_quota|g" \ /etc/dovecot/conf.d/20-imap.conf fi @@ -113,11 +123,11 @@ function _setup_dovecot_quota() { local MESSAGE_SIZE_LIMIT_MB=$((POSTFIX_MESSAGE_SIZE_LIMIT / 1000000)) local MAILBOX_LIMIT_MB=$((POSTFIX_MAILBOX_SIZE_LIMIT / 1000000)) - sed -i \ + sedfile -i \ "s|quota_max_mail_size =.*|quota_max_mail_size = ${MESSAGE_SIZE_LIMIT_MB}$([[ ${MESSAGE_SIZE_LIMIT_MB} -eq 0 ]] && echo "" || echo "M")|g" \ /etc/dovecot/conf.d/90-quota.conf - sed -i \ + sedfile -i \ "s|quota_rule = \*:storage=.*|quota_rule = *:storage=${MAILBOX_LIMIT_MB}$([[ ${MAILBOX_LIMIT_MB} -eq 0 ]] && echo "" || echo "M")|g" \ /etc/dovecot/conf.d/90-quota.conf @@ -127,7 +137,7 @@ function _setup_dovecot_quota() { fi # enable quota policy check in postfix - sed -i -E \ + sedfile -i -E \ "s|(reject_unknown_recipient_domain)|\1, check_policy_service inet:localhost:65265|g" \ /etc/postfix/main.cf fi @@ -188,5 +198,5 @@ function _setup_dovecot_dhparam() { function _setup_dovecot_hostname() { _log 'debug' 'Applying hostname to Dovecot' - sed -i "s|^#hostname =.*$|hostname = '${HOSTNAME}'|g" /etc/dovecot/conf.d/15-lda.conf + sedfile -i "s|^#hostname =.*$|hostname = '${HOSTNAME}'|g" /etc/dovecot/conf.d/15-lda.conf } diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 3b575f50..2660ce89 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -87,6 +87,7 @@ function __environment_variables_general_setup() { VARS[ENABLE_OPENDMARC]="${ENABLE_OPENDMARC:=1}" VARS[ENABLE_POLICYD_SPF]="${ENABLE_POLICYD_SPF:=1}" VARS[ENABLE_POP3]="${ENABLE_POP3:=0}" + VARS[ENABLE_IMAP]="${ENABLE_IMAP:=1}" VARS[ENABLE_POSTGREY]="${ENABLE_POSTGREY:=0}" VARS[ENABLE_QUOTAS]="${ENABLE_QUOTAS:=1}" VARS[ENABLE_RSPAMD]="${ENABLE_RSPAMD:=0}" From 083e46408413c11e71f7e9ae299ae89874d8c35c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:05:37 +0100 Subject: [PATCH 086/125] chore(deps): Bump github/codeql-action from 2 to 3 (#3709) --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 95245557..08b4fd29 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -62,6 +62,6 @@ jobs: fail-build: false - name: 'Upload vulnerability report' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: ${{ steps.scan.outputs.sarif }} From b9f7ff86dea6709f785be0bf900c2845f9e321ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:50:56 +0100 Subject: [PATCH 087/125] chore(deps): Bump docker/metadata-action from 5.3.0 to 5.4.0 (#3710) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 5.3.0 to 5.4.0. - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/v5.3.0...v5.4.0) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 36b44770..b0a32b93 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.3.0 + uses: docker/metadata-action@v5.4.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 5015dc80b16f61e29cb9b496ceb6c6210de635a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 00:16:38 +0100 Subject: [PATCH 088/125] chore(deps): Bump actions/upload-artifact from 3 to 4 (#3708) --- .github/workflows/docs-preview-prepare.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-prepare.yml b/.github/workflows/docs-preview-prepare.yml index befd4008..cee5562a 100644 --- a/.github/workflows/docs-preview-prepare.yml +++ b/.github/workflows/docs-preview-prepare.yml @@ -73,7 +73,7 @@ jobs: tar --zstd -cf artifact.tar.zst pr.env ${{ env.BUILD_DIR }} - name: 'Upload artifact for workflow transfer' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: preview-build path: artifact.tar.zst From ee87291225f55feb4a4c96b2383c55827db54736 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 01:29:15 +0100 Subject: [PATCH 089/125] chore(deps): Bump dawidd6/action-download-artifact from 2 to 3 (#3707) --- .github/workflows/docs-preview-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index cac2ac64..93819efa 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -25,7 +25,7 @@ jobs: # The official Github Action for downloading artifacts does not support multi-workflow - name: 'Download build artifact' - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} run_id: ${{ github.event.workflow_run.id }} From 5908d9f060b1c74d7f168a578d11869ff15f7ad4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:33:38 +1300 Subject: [PATCH 090/125] tests(refactor): Dovecot quotas (#3068) * chore: Extract out Dovecot Quota test cases into new test file Test cases are just cut + paste, no logic changed there yet. * chore: Rename test case descriptions * chore: Use `setup ...` methods instead of direct calls * chore: Adjust `_run_in_container_bash` to `_run_in_container` Plus some additional bug fixes in the disabled test case * tests(refactor): Revise ENV test cases for max mailbox and message sizes * tests(refactor): Revise ENV test cases for mailbox and message limits v2 Removes the extra variables and filtering in favour of explicit values instead of matching for comparison. - Easier at a glance to know what is actually expected. - Additionally reworks the quota limit checks in other test cases. Using a different formatter for `doveadm` is easier to match the desired value (`Limit`). * chore: Sync improvement from `tests.bats` master --- NOTE: This PR has been merged to avoid additional maintenance burden without losing the improvements. It was not considered complete, but remaining tasks were not documented in the PR. --- docs/content/config/environment.md | 8 +- mailserver.env | 4 +- target/bin/setquota | 6 +- .../parallel/set1/dovecot/dovecot_quotas.bats | 246 ++++++++++++++++++ test/tests/serial/tests.bats | 193 -------------- 5 files changed, 257 insertions(+), 200 deletions(-) create mode 100644 test/tests/parallel/set1/dovecot/dovecot_quotas.bats diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 5ef3b968..71807c87 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -228,9 +228,9 @@ Provide any valid URI. Examples: - `lmtps:inet::` (secure lmtp with starttls) - `lmtp::2003` (use kopano as mailstore) -##### POSTFIX\_MAILBOX\_SIZE\_LIMIT +##### POSTFIX_MAILBOX_SIZE_LIMIT -Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). +Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). Size is in bytes. - **empty** => 0 (no limit) @@ -241,9 +241,9 @@ Set the mailbox size limit for all users. If set to zero, the size will be unlim See [mailbox quota][docs-accounts-quota]. -##### POSTFIX\_MESSAGE\_SIZE\_LIMIT +##### POSTFIX_MESSAGE_SIZE_LIMIT -Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!) +Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!). Size is in bytes. - **empty** => 10240000 (~10 MB) diff --git a/mailserver.env b/mailserver.env index 48b537bc..e84157a2 100644 --- a/mailserver.env +++ b/mailserver.env @@ -254,7 +254,7 @@ VIRUSMAILS_DELETE_DELAY= # `lmtp::2003` (use kopano as mailstore) POSTFIX_DAGENT= -# Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). +# Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). Size is in bytes. # # empty => 0 POSTFIX_MAILBOX_SIZE_LIMIT= @@ -264,7 +264,7 @@ POSTFIX_MAILBOX_SIZE_LIMIT= # 1 => Dovecot quota is enabled ENABLE_QUOTAS=1 -# Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!) +# Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!). Size is in bytes. # # empty => 10240000 (~10 MB) POSTFIX_MESSAGE_SIZE_LIMIT= diff --git a/target/bin/setquota b/target/bin/setquota index 5b2bba41..039421b2 100755 --- a/target/bin/setquota +++ b/target/bin/setquota @@ -59,10 +59,14 @@ function _quota_request_if_missing() { fi } + +# Dovecot docs incorrectly refer to these units with names for SI types (base 10), +# But then mentions they're actually treated as IEC type (base 2): +# https://doc.dovecot.org/settings/types/#size function _quota_unit_is_valid() { if ! grep -qE "^([0-9]+(B|k|M|G|T)|0)\$" <<< "${QUOTA}"; then __usage - _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kilobyte), M (megabyte), G (gigabyte) or T (terabyte))' + _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kibibyte), M (mebibyte), G (gibibyte) or T (tebibyte))' fi } diff --git a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats new file mode 100644 index 00000000..2c176235 --- /dev/null +++ b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats @@ -0,0 +1,246 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +# upstream default: 10 240 000 +# https://www.postfix.org/postconf.5.html#message_size_limit +# > The maximal size in bytes of a message, including envelope information. +# > The value cannot exceed LONG_MAX (typically, a 32-bit or 64-bit signed integer). +# > Note: Be careful when making changes. Excessively small values will result in the loss of non-delivery notifications, when a bounce message size exceeds the local or remote MTA's message size limit. + +# upstream default: 51 200 000 +# https://www.postfix.org/postconf.5.html#mailbox_size_limit +# > The maximal size of any local(8) individual mailbox or maildir file, or zero (no limit). +# > In fact, this limits the size of any file that is written to upon local delivery, including files written by external commands that are executed by the local(8) delivery agent. +# > The value cannot exceed LONG_MAX (typically, a 32-bit or 64-bit signed integer). +# > This limit must not be smaller than the message size limit. + +# upstream default: 51 200 000 +# https://www.postfix.org/postconf.5.html#virtual_mailbox_limit +# > The maximal size in bytes of an individual virtual(8) mailbox or maildir file, or zero (no limit). +# > This parameter is specific to the virtual(8) delivery agent. +# > It does not apply when mail is delivered with a different mail delivery program. + +BATS_TEST_NAME_PREFIX='[Dovecot Quotas] ' +CONTAINER_NAME='dms-test_dovecot-quotas' + +function setup_file() { + _init_with_defaults + + local CONTAINER_ARGS_ENV_CUSTOM=( + --env ENABLE_QUOTAS=1 + --env POSTFIX_MAILBOX_SIZE_LIMIT=4096000 + --env POSTFIX_MESSAGE_SIZE_LIMIT=2048000 + --env PERMIT_DOCKER=container + ) + _common_container_setup 'CONTAINER_ARGS_ENV_CUSTOM' +} + +function teardown_file() { _default_teardown ; } + +@test 'should only support setting quota for a valid account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota set quota_user 50M + assert_failure + + _run_in_container setup quota set username@fulldomain 50M + assert_failure + + _run_in_container setup quota set quota_user@domain.tld 50M + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should only allow valid units as quota size' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota set quota_user@domain.tld 26GIGOTS + assert_failure + _run_in_container setup quota set quota_user@domain.tld 123 + assert_failure + _run_in_container setup quota set quota_user@domain.tld M + assert_failure + _run_in_container setup quota set quota_user@domain.tld -60M + assert_failure + + + _run_in_container setup quota set quota_user@domain.tld 10B + assert_success + _run_in_container setup quota set quota_user@domain.tld 10k + assert_success + _run_in_container setup quota set quota_user@domain.tld 10M + assert_success + _run_in_container setup quota set quota_user@domain.tld 10G + assert_success + _run_in_container setup quota set quota_user@domain.tld 10T + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should only support removing quota from a valid account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota del uota_user@domain.tld + assert_failure + _run_in_container setup quota del quota_user + assert_failure + _run_in_container setup quota del dontknowyou@domain.tld + assert_failure + + _run_in_container setup quota set quota_user@domain.tld 10T + assert_success + _run_in_container setup quota del quota_user@domain.tld + assert_success + _run_in_container grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf + assert_failure + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should not error when there is no quota to remove for an account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf + assert_failure + + _run_in_container setup quota del quota_user@domain.tld + assert_success + _run_in_container setup quota del quota_user@domain.tld + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should have configured Postfix to use the Dovecot quota-status service' { + _run_in_container postconf + assert_success + assert_output --partial 'check_policy_service inet:localhost:65265' +} + +@test '(ENV POSTFIX_MAILBOX_SIZE_LIMIT) should be configured for both Postfix and Dovecot' { + _run_in_container postconf -h mailbox_size_limit + assert_output 4096000 + + # Dovecot mailbox is sized by `virtual_mailbox_size` from Postfix: + _run_in_container postconf -h virtual_mailbox_limit + assert_output 4096000 + + # Quota support: + _run_in_container doveconf -h plugin/quota_rule + # Global default storage limit quota for each mailbox 4 MiB: + assert_output '*:storage=4M' + + # Sizes are equivalent - Bytes to MiB (rounded): + run numfmt --to=iec --format '%.0f' 4096000 + assert_output '4M' +} + +@test '(ENV POSTFIX_MESSAGE_SIZE_LIMIT) should be configured for both Postfix and Dovecot' { + _run_in_container postconf -h message_size_limit + assert_output 2048000 + + _run_in_container doveconf -h plugin/quota_max_mail_size + assert_output '2M' + + # Sizes are equivalent - Bytes to MiB (rounded): + run numfmt --to=iec --format '%.0f' 2048000 + assert_output '2M' +} + +@test 'Deleting an mailbox account should also remove that account from dovecot-quotas.cf' { + _add_mail_account_then_wait_until_ready 'quserremoved@domain.tld' + + _run_in_container setup quota set quserremoved@domain.tld 12M + assert_success + + _run_in_container cat '/tmp/docker-mailserver/dovecot-quotas.cf' + assert_success + assert_output 'quserremoved@domain.tld:12M' + + _run_in_container setup email del -y quserremoved@domain.tld + assert_success + + _run_in_container cat /tmp/docker-mailserver/dovecot-quotas.cf + assert_success + refute_output --partial 'quserremoved@domain.tld:12M' +} + +@test 'Dovecot should acknowledge quota configured for accounts' { + # sed -nE 's/.*STORAGE.*Limit=([0-9]+).*/\1/p' | numfmt --from-unit=1024 --to=iec --format '%.0f' + local CMD_GET_QUOTA="doveadm -f flow quota get -u 'user1@localhost.localdomain'" + + # 4M == 4096 kiB (numfmt --to-unit=1024 --from=iec 4M) + _run_in_container_bash "${CMD_GET_QUOTA}" + assert_line --partial 'Type=STORAGE Value=0 Limit=4096' + + # Setting a new limit for the user: + _run_in_container setup quota set 'user1@localhost.localdomain' 50M + assert_success + # 50M (50 * 1024^2) == 51200 kiB (numfmt --to-unit=1024 --from=iec 52428800) + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "${CMD_GET_QUOTA} | grep -o 'Type=STORAGE Value=0 Limit=51200'" + assert_success + + # Deleting quota resets it to default global quota limit (`plugin/quota_rule`): + _run_in_container setup quota del 'user1@localhost.localdomain' + assert_success + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "${CMD_GET_QUOTA} | grep -o 'Type=STORAGE Value=0 Limit=4096'" + assert_success +} + +@test 'should receive a warning mail from Dovecot when quota is exceeded' { + # skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2511' + + # Prepare + _add_mail_account_then_wait_until_ready 'quotauser@otherdomain.tld' + + # Actual tests + _run_in_container setup quota set quotauser@otherdomain.tld 10k + assert_success + + # wait until quota has been updated + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "doveadm -f flow quota get -u 'quotauser@otherdomain.tld' | grep -o 'Type=STORAGE Value=0 Limit=10'" + assert_success + + # dovecot and postfix has been restarted + _wait_for_service postfix + _wait_for_service dovecot + sleep 10 + + # send some big emails + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + # check for quota warn message existence + run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/ + assert_success + + run _repeat_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER_NAME} | grep 'Quota exceeded (mailbox for user is full)'" + assert_success + + # ensure only the first big message and the warn message are present (other messages are rejected: mailbox is full) + _run_in_container sh -c 'ls /var/mail/otherdomain.tld/quotauser/new/ | wc -l' + assert_success + assert_output "2" + + # Cleanup + _run_in_container setup email del -y quotauser@otherdomain.tld + assert_success +} diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 20ee0dd1..094454f0 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -17,7 +17,6 @@ function setup_file() { local CONTAINER_ARGS_ENV_CUSTOM=( --env ENABLE_AMAVIS=1 --env AMAVIS_LOGLEVEL=2 - --env ENABLE_QUOTAS=1 --env ENABLE_SRS=1 --env PERMIT_DOCKER=host --env PFLOGSUMM_TRIGGER=logrotate @@ -244,198 +243,6 @@ zip EOF } -@test "quota: setquota user must be existing" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "setquota quota_user 50M" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld 50M" - assert_success - - _run_in_container_bash "setquota username@fulldomain 50M" - assert_failure - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: setquota must be well formatted" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "setquota quota_user@domain.tld 26GIGOTS" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld 123" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld M" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld -60M" - assert_failure - - - _run_in_container_bash "setquota quota_user@domain.tld 10B" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10k" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10M" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10G" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10T" - assert_success - - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: delquota user must be existing" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "delquota uota_user@domain.tld" - assert_failure - _run_in_container_bash "delquota quota_user" - assert_failure - _run_in_container_bash "delquota dontknowyou@domain.tld" - assert_failure - - _run_in_container_bash "setquota quota_user@domain.tld 10T" - assert_success - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - _run_in_container_bash "grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf" - assert_failure - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: delquota allow when no quota for existing user" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf" - assert_failure - - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: dovecot quota present in postconf" { - _run_in_container_bash "postconf | grep 'check_policy_service inet:localhost:65265'" - assert_success -} - - -@test "quota: dovecot mailbox max size must be equal to postfix mailbox max size" { - postfix_mailbox_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=mailbox_size_limit = )[0-9]+'") - run echo "${postfix_mailbox_size}" - refute_output "" - - # dovecot relies on virtual_mailbox_size by default - postfix_virtual_mailbox_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=virtual_mailbox_limit = )[0-9]+'") - assert_equal "${postfix_virtual_mailbox_size}" "${postfix_mailbox_size}" - - postfix_mailbox_size_mb=$(( postfix_mailbox_size / 1000000)) - - dovecot_mailbox_size_mb=$(_exec_in_container_bash "doveconf | grep -oP '(?<=quota_rule \= \*\:storage=)[0-9]+'") - run echo "${dovecot_mailbox_size_mb}" - refute_output "" - - assert_equal "${postfix_mailbox_size_mb}" "${dovecot_mailbox_size_mb}" -} - - -@test "quota: dovecot message max size must be equal to postfix messsage max size" { - postfix_message_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=message_size_limit = )[0-9]+'") - run echo "${postfix_message_size}" - refute_output "" - - postfix_message_size_mb=$(( postfix_message_size / 1000000)) - - dovecot_message_size_mb=$(_exec_in_container_bash "doveconf | grep -oP '(?<=quota_max_mail_size = )[0-9]+'") - run echo "${dovecot_message_size_mb}" - refute_output "" - - assert_equal "${postfix_message_size_mb}" "${dovecot_message_size_mb}" -} - -@test "quota: quota directive is removed when mailbox is removed" { - _add_mail_account_then_wait_until_ready 'quserremoved@domain.tld' - - _run_in_container_bash "setquota quserremoved@domain.tld 12M" - assert_success - - _run_in_container_bash 'cat /tmp/docker-mailserver/dovecot-quotas.cf | grep -E "^quserremoved@domain.tld\:12M\$" | wc -l | grep 1' - assert_success - - _run_in_container_bash "delmailuser -y quserremoved@domain.tld" - assert_success - - _run_in_container_bash 'cat /tmp/docker-mailserver/dovecot-quotas.cf | grep -E "^quserremoved@domain.tld\:12M\$"' - assert_failure -} - -@test "quota: dovecot applies user quota" { - _run_in_container_bash "doveadm quota get -u 'user1@localhost.localdomain' | grep 'User quota STORAGE'" - assert_output --partial "- 0" - - _run_in_container_bash "setquota user1@localhost.localdomain 50M" - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u user1@localhost.localdomain | grep -oP "(User quota STORAGE\s+[0-9]+\s+)51200(.*)"' - assert_success - - _run_in_container_bash "delquota user1@localhost.localdomain" - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u user1@localhost.localdomain | grep -oP "(User quota STORAGE\s+[0-9]+\s+)-(.*)"' - assert_success -} - -@test "quota: warn message received when quota exceeded" { - skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2511' - - # create user - _add_mail_account_then_wait_until_ready 'quotauser@otherdomain.tld' - _run_in_container_bash 'setquota quotauser@otherdomain.tld 10k' - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u quotauser@otherdomain.tld | grep -oP \"(User quota STORAGE\s+[0-9]+\s+)10(.*)\"' - assert_success - - # dovecot and postfix has been restarted - _wait_for_service postfix - _wait_for_service dovecot - sleep 10 - - # send some big emails - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - - # check for quota warn message existence - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'grep \"Subject: quota warning\" /var/mail/otherdomain.tld/quotauser/new/ -R' - assert_success - - run _repeat_until_success_or_timeout 20 sh -c "docker logs mail | grep 'Quota exceeded (mailbox for user is full)'" - assert_success - - # ensure only the first big message and the warn message are present (other messages are rejected: mailbox is full) - _run_in_container sh -c 'ls /var/mail/otherdomain.tld/quotauser/new/ | wc -l' - assert_success - assert_output "2" - - _run_in_container_bash "delmailuser -y quotauser@otherdomain.tld" - assert_success -} - # # PERMIT_DOCKER mynetworks # From ca2c53dde714a5c4d64aebb6724cfb6d87352a54 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:41:07 +1300 Subject: [PATCH 091/125] ci: Avoiding linting `CONTRIBUTORS.yml` (#3705) The file is managed by the `contributors.yml` workflow, no need for linting to be triggered on PRs for that change. This should ideally skip the required check status for the lint workflow which cannot trigger implicitly for automated PRs. If this doesn't work the change should be reverted. --- .github/workflows/linting.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index bdde2d0f..1e93377f 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -2,6 +2,9 @@ name: Lint on: pull_request: + paths-ignore: + # Managed by workflow: contributors.yml + - CONTRIBUTORS.md push: branches: [ master ] From 03052a65b87542ce714d1e58dca579907b534d39 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:59:59 +1300 Subject: [PATCH 092/125] ci: Allow lint workflow to be manually triggered (#3714) * ci: Allow lint workflow to be manually triggered Without this a different event must occur to trigger the workflow, which is inconvenient for automated PRs. --- .github/workflows/linting.yml | 6 +++--- CHANGELOG.md | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 1e93377f..d419e5ec 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,10 +1,10 @@ name: Lint on: + # A workflow that creates a PR will not trigger this workflow, + # Providing a manual trigger as a workaround + workflow_dispatch: pull_request: - paths-ignore: - # Managed by workflow: contributors.yml - - CONTRIBUTORS.md push: branches: [ master ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 339c92e0..8b1331f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ All notable changes to this project will be documented in this file. The format - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) +- **CI / Automation:** + - The lint workflow can now be manually triggered by maintainers ([#3714]https://github.com/docker-mailserver/docker-mailserver/pull/3714) ## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) From 6a56c7e74936488626db973f1c586d888894fc82 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 06:01:25 +0000 Subject: [PATCH 093/125] docs: update `CONTRIBUTORS.md` (#3704) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 860f5df1..4103ad5f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -442,6 +442,13 @@ Thanks goes to these wonderful people ✨ pbek + + + reneploetz +
+ reneploetz + + Rubytastic2 @@ -469,15 +476,15 @@ Thanks goes to these wonderful people ✨
Zehir - + + guardiande
guardiande - - + kamuri @@ -512,15 +519,15 @@ Thanks goes to these wonderful people ✨
m-schmoock - + + VanVan
VanVan - - + elbracht @@ -555,15 +562,15 @@ Thanks goes to these wonderful people ✨
ubenmackin - + + craue
craue - - + danielpanteleit @@ -598,15 +605,15 @@ Thanks goes to these wonderful people ✨
emazzotta - + + fl42
fl42 - - + ipernet @@ -641,15 +648,15 @@ Thanks goes to these wonderful people ✨
millaguie - + + jedateach
jedateach - - + spacecowboy @@ -684,15 +691,15 @@ Thanks goes to these wonderful people ✨
keslerm - + + castorinop
castorinop - - + p-fruck @@ -707,13 +714,6 @@ Thanks goes to these wonderful people ✨ Rillke - - - reneploetz -
- reneploetz - - bobbravo2 From 226ec847a4875e80c4ceff4c486f71a88176b897 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:35:16 +1300 Subject: [PATCH 094/125] ci: Remove `VERSION` from `Dockerfile` (#3711) * ci: Remove `VERSION` from `Dockerfile` This line was meant to be dropped with the switch to using the `DMS_RELEASE` ENV. --- CHANGELOG.md | 1 + Dockerfile | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b1331f1..37a7a123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ All notable changes to this project will be documented in this file. The format - **Internal**: - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + - `VERSION` is no longer included in the image ([#3711](https://github.com/docker-mailserver/docker-mailserver/pull/3711)) - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) diff --git a/Dockerfile b/Dockerfile index 0f19521a..4d0e3568 100644 --- a/Dockerfile +++ b/Dockerfile @@ -277,8 +277,6 @@ RUN < Date: Wed, 20 Dec 2023 01:43:32 +0100 Subject: [PATCH 095/125] Remove sed statement (#3715) --- CHANGELOG.md | 1 + target/scripts/startup/setup.d/dovecot.sh | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a7a123..3fc60c05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes to this project will be documented in this file. The format - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` is no longer included in the image ([#3711](https://github.com/docker-mailserver/docker-mailserver/pull/3711)) - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) + - `ENABLE_QUOTAS=0` no longer tries to remove non-existent config ([#3715](https://github.com/docker-mailserver/docker-mailserver/pull/3715)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) - **CI / Automation:** diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index e46aca21..8e7dcfe7 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -106,9 +106,6 @@ function _setup_dovecot_quota() { "s|mail_plugins = \$mail_plugins imap_quota|mail_plugins = \$mail_plugins|g" \ /etc/dovecot/conf.d/20-imap.conf fi - - # disable quota policy check in postfix - sedfile -i "s|check_policy_service inet:localhost:65265||g" /etc/postfix/main.cf else if [[ -f /etc/dovecot/conf.d/90-quota.conf.disab ]]; then mv /etc/dovecot/conf.d/90-quota.conf.disab /etc/dovecot/conf.d/90-quota.conf From 72517d3f824859cb15a3ccc653ad8cc4bb1a4c32 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 26 Dec 2023 09:53:32 +1300 Subject: [PATCH 096/125] docs: Debugging - Delivery failure from service downtime (#3718) * docs: Debugging - Delivery failure from service downtime Services may be temporarily down, such as when restarted when certificates are updated due to the `check-for-changes.sh` service. This is another known source of intermittent delivery failures. --- CHANGELOG.md | 3 ++- docs/content/config/debugging.md | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fc60c05..7a260508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Documentation:** - - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) + - Debugging - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) + - Debugging - Raise awareness of temporary downtime during certificate renewal that can cause a failure to deliver local mail ([#3718](https://github.com/docker-mailserver/docker-mailserver/pull/3718)) - **Internal:** - Postfix configures `virtual_mailbox_maps` and `virtual_transport` during startup instead of using defaults (configured for Dovecot) via our `main.cf` ([#3681](https://github.com/docker-mailserver/docker-mailserver/pull/3681)) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 9c3bebb5..d58430e1 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -55,6 +55,8 @@ Common logs related to this are: If your logs look like this, you likely have [assigned the same FQDN to the DMS `hostname` and your mail accounts][gh-issues::dms-fqdn-misconfigured] which is not supported by default. You can either adjust your DMS `hostname` or follow [this FAQ advice][docs::faq-bare-domain] +It is also possible that [DMS services are temporarily unavailable][gh-issues::dms-services-unavailable] when configuration changes are detected, producing the 2nd error. Certificate updates may be a less obvious trigger. + ## Steps for Debugging DMS 1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. @@ -126,6 +128,7 @@ This could be from outdated software, or running a system that isn't able to pro [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues [gh-issues::dms-fqdn-misconfigured]: https://github.com/docker-mailserver/docker-mailserver/issues/3679#issuecomment-1837609043 +[gh-issues::dms-services-unavailable]: https://github.com/docker-mailserver/docker-mailserver/issues/3679#issuecomment-1848083358 [gh-macos-support]: https://github.com/docker-mailserver/docker-mailserver/issues/3648#issuecomment-1822774080 [gh-discuss-roundcube-fail2ban]: https://github.com/orgs/docker-mailserver/discussions/3273#discussioncomment-5654603 From 8392e3c1a8db660e6fd458b1bf426ab4736104c4 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 29 Dec 2023 13:58:54 +0100 Subject: [PATCH 097/125] release: v13.1.0 (#3720) Co-authored-by: Casper Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a260508..32aaf78d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.1.0) + ### Added - **Dovecot:** @@ -22,6 +24,8 @@ All notable changes to this project will be documented in this file. The format - Debugging - Raise awareness of temporary downtime during certificate renewal that can cause a failure to deliver local mail ([#3718](https://github.com/docker-mailserver/docker-mailserver/pull/3718)) - **Internal:** - Postfix configures `virtual_mailbox_maps` and `virtual_transport` during startup instead of using defaults (configured for Dovecot) via our `main.cf` ([#3681](https://github.com/docker-mailserver/docker-mailserver/pull/3681)) +- **Rspamd:** + - Upgraded to version `3.7.5`. This was previously inconsistent between our AMD64 (`3.5`) and ARM64 (`3.4`) images ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) ### Fixed diff --git a/VERSION b/VERSION index 5cb7d856..e6ba3513 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.0.1 +13.1.0 From 0889b0ff063a37b482113a684d934e4bd728a33c Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 30 Dec 2023 09:59:09 +1300 Subject: [PATCH 098/125] fix: `supervisor-app.conf` - Correct the log location for `postgrey` (#3724) * fix: `supervisor-app.conf` - Correct `postgrey` log location Looks like this should have been like every other service and reference a log file(s) based on program name in the supervisor log directory. * tests: Adjust log location for `postgrey_enabled.bats` --- CHANGELOG.md | 1 + target/supervisor/conf.d/supervisor-app.conf | 4 ++-- test/tests/parallel/set1/spam_virus/postgrey_enabled.bats | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32aaf78d..f37fded4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ All notable changes to this project will be documented in this file. The format - `VERSION` is no longer included in the image ([#3711](https://github.com/docker-mailserver/docker-mailserver/pull/3711)) - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - `ENABLE_QUOTAS=0` no longer tries to remove non-existent config ([#3715](https://github.com/docker-mailserver/docker-mailserver/pull/3715)) + - The `postgrey` service now writes logs to the supervisor directory like all other services. Previously this was `/var/log/mail/mail.log` ([#3724](https://github.com/docker-mailserver/docker-mailserver/pull/3724)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) - **CI / Automation:** diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 2dd8b917..431357d8 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -83,8 +83,8 @@ startsecs=0 stopwaitsecs=55 autostart=false autorestart=true -stdout_logfile=/var/log/mail/mail.log -stderr_logfile=/var/log/mail/mail.log +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/usr/sbin/postgrey --inet=127.0.0.1:10023 --syslog-facility=mail --delay="%(ENV_POSTGREY_DELAY)s" --max-age="%(ENV_POSTGREY_MAX_AGE)s" --auto-whitelist-clients="%(ENV_POSTGREY_AUTO_WHITELIST_CLIENTS)s" --greylist-text="%(ENV_POSTGREY_TEXT)s" [program:amavis] diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index d877a1ce..e32210ca 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -123,7 +123,7 @@ function _should_have_log_entry() { # Allow some extra time for logs to update to avoids a false-positive failure: _run_until_success_or_timeout 10 _exec_in_container grep \ "${ACTION}, ${REASON}," \ - /var/log/mail/mail.log + /var/log/supervisor/postgrey.log # Log entry matched should be for the expected triplet: assert_output --partial "${TRIPLET}" From 9e81517fe36d95597d3c8890f998bd7f9ea29aa7 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 3 Jan 2024 01:17:54 +0100 Subject: [PATCH 099/125] tests: Use `swaks` instead of `nc` for sending mail (#3732) See associated `CHANGELOG.md` entry for details. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .gitattributes | 6 +- CHANGELOG.md | 7 ++ target/scripts/build/packages.sh | 2 +- .../auth/added-imap-auth.txt | 0 .../auth/added-pop3-auth.txt | 0 test/{test-files => files}/auth/imap-auth.txt | 0 .../auth/imap-ldap-auth.txt | 0 test/{test-files => files}/auth/pop3-auth.txt | 0 .../emails/amavis/spam.txt} | 9 +- .../emails/amavis/virus.txt} | 9 +- .../auth/added-smtp-auth-spoofed-alias.txt | 9 -- .../emails}/auth/added-smtp-auth-spoofed.txt | 9 -- .../auth/ldap-smtp-auth-spoofed-alias.txt | 10 --- ...h-spoofed-sender-with-filter-exception.txt | 10 --- .../emails}/auth/ldap-smtp-auth-spoofed.txt | 10 --- .../emails/existing/added.txt} | 7 -- .../emails/existing/alias-external.txt} | 7 -- .../emails/existing/alias-local.txt} | 7 -- .../existing/alias-recipient-delimiter.txt} | 7 -- .../emails/existing/catchall-local.txt} | 7 -- .../existing/regexp-alias-external.txt} | 7 -- .../emails/existing/regexp-alias-local.txt} | 7 -- .../existing/user-and-cc-local-alias.txt} | 7 -- .../emails/existing/user1.txt} | 7 -- .../emails/nc_raw/dsn/authenticated.txt} | 0 .../emails/nc_raw/dsn/unauthenticated.txt} | 0 .../emails/nc_raw}/postscreen.txt | 0 .../emails/nc_raw}/smtp-only.txt | 0 .../emails}/non-existing-user.txt | 7 -- .../emails}/postgrey.txt | 7 -- test/files/emails/postscreen.txt | 5 ++ .../emails/privacy.txt} | 9 -- .../emails}/quota-exceeded.txt | 7 -- .../emails/rspamd/pass.txt} | 9 +- .../emails/rspamd/spam-header.txt} | 7 -- .../emails/rspamd/spam.txt} | 7 -- .../emails/rspamd/virus.txt} | 7 -- .../emails/sendmail}/root-email.txt | 0 .../emails/sieve/pipe.txt} | 7 -- .../emails/sieve/spam-folder.txt} | 7 -- .../emails}/test-email.txt | 0 .../nc}/imap_special_use_folders.txt | 0 .../nc}/postgrey_whitelist.txt | 0 .../nc}/postgrey_whitelist_recipients.txt | 0 .../nc}/rspamd_imap_move_to_inbox.txt | 0 .../nc}/rspamd_imap_move_to_junk.txt | 0 .../ssl/custom-dhe-params.pem | 0 .../ssl/example.test/README.md | 0 .../ssl/example.test/cert.ecdsa.pem | 0 .../ssl/example.test/cert.rsa.pem | 0 .../ssl/example.test/key.ecdsa.pem | 0 .../ssl/example.test/key.rsa.pem | 0 .../ssl/example.test/testssl.txt | 0 .../ssl/example.test/traefik.md | 0 .../with_ca/ecdsa/ca-cert.ecdsa.pem | 0 .../with_ca/ecdsa/ca-key.ecdsa.pem | 0 .../example.test/with_ca/ecdsa/cert.ecdsa.pem | 0 .../example.test/with_ca/ecdsa/cert.rsa.pem | 0 .../with_ca/ecdsa/ecdsa.acme.json | 0 .../example.test/with_ca/ecdsa/key.ecdsa.pem | 0 .../example.test/with_ca/ecdsa/key.rsa.pem | 0 .../example.test/with_ca/ecdsa/rsa.acme.json | 0 .../with_ca/ecdsa/wildcard/cert.ecdsa.pem | 0 .../with_ca/ecdsa/wildcard/ecdsa.acme.json | 0 .../with_ca/ecdsa/wildcard/key.ecdsa.pem | 0 .../example.test/with_ca/rsa/ca-cert.rsa.pem | 0 .../example.test/with_ca/rsa/ca-key.rsa.pem | 0 .../example.test/with_ca/rsa/cert.ecdsa.pem | 0 .../ssl/example.test/with_ca/rsa/cert.rsa.pem | 0 .../example.test/with_ca/rsa/ecdsa.acme.json | 0 .../example.test/with_ca/rsa/key.ecdsa.pem | 0 .../ssl/example.test/with_ca/rsa/key.rsa.pem | 0 .../example.test/with_ca/rsa/rsa.acme.json | 0 .../with_ca/rsa/wildcard/cert.rsa.pem | 0 .../with_ca/rsa/wildcard/key.rsa.pem | 0 .../with_ca/rsa/wildcard/rsa.acme.json | 0 test/helper/common.bash | 14 +++ test/helper/sending.bash | 65 +++++++++----- test/helper/setup.bash | 2 +- .../auth/added-smtp-auth-login-wrong.txt | 4 - .../test-files/auth/added-smtp-auth-login.txt | 4 - .../auth/added-smtp-auth-plain-wrong.txt | 3 - .../test-files/auth/added-smtp-auth-plain.txt | 3 - test/test-files/auth/sasl-ldap-smtp-auth.txt | 5 -- .../test-files/auth/smtp-auth-login-wrong.txt | 4 - test/test-files/auth/smtp-auth-login.txt | 4 - .../test-files/auth/smtp-auth-plain-wrong.txt | 3 - test/test-files/auth/smtp-auth-plain.txt | 3 - .../email-templates/existing-user2.txt | 12 --- .../email-templates/existing-user3.txt | 12 --- test/test-files/email-templates/smtp-ehlo.txt | 2 - .../parallel/set1/dovecot/dovecot_quotas.bats | 9 +- .../parallel/set1/dovecot/dovecot_sieve.bats | 6 +- .../set1/dovecot/mailbox_format_dbox.bats | 6 +- .../set1/dovecot/special_use_folders.bats | 5 +- .../parallel/set1/spam_virus/clamav.bats | 13 +-- .../disabled_clamav_spamassassin.bats | 4 +- .../parallel/set1/spam_virus/fail2ban.bats | 13 ++- .../set1/spam_virus/postgrey_enabled.bats | 37 +++----- .../parallel/set1/spam_virus/postscreen.bats | 55 +++++------- .../parallel/set1/spam_virus/rspamd_full.bats | 12 +-- .../set1/spam_virus/spam_junk_folder.bats | 2 +- test/tests/parallel/set1/tls/dhparams.bats | 2 +- test/tests/parallel/set1/tls/letsencrypt.bats | 4 +- test/tests/parallel/set1/tls/manual.bats | 6 +- test/tests/parallel/set2/tls_cipherlists.bats | 2 +- .../container_configuration/hostname.bats | 2 +- test/tests/parallel/set3/mta/dsn.bats | 20 +++-- test/tests/parallel/set3/mta/lmtp_ip.bats | 2 +- test/tests/parallel/set3/mta/privacy.bats | 6 +- .../parallel/set3/mta/smtp_delivery.bats | 88 +++++++++++-------- test/tests/parallel/set3/mta/smtponly.bats | 11 ++- test/tests/serial/mail_pop3.bats | 6 +- test/tests/serial/mail_with_imap.bats | 28 ++++-- test/tests/serial/mail_with_ldap.bats | 57 +++++++++--- test/tests/serial/permit_docker.bats | 10 +-- test/tests/serial/test_helper.bats | 8 +- test/tests/serial/tests.bats | 31 ++++++- test/tests/serial/vmail-id.bats | 2 +- 119 files changed, 355 insertions(+), 455 deletions(-) rename test/{test-files => files}/auth/added-imap-auth.txt (100%) rename test/{test-files => files}/auth/added-pop3-auth.txt (100%) rename test/{test-files => files}/auth/imap-auth.txt (100%) rename test/{test-files => files}/auth/imap-ldap-auth.txt (100%) rename test/{test-files => files}/auth/pop3-auth.txt (100%) rename test/{test-files/email-templates/amavis-spam.txt => files/emails/amavis/spam.txt} (63%) rename test/{test-files/email-templates/amavis-virus.txt => files/emails/amavis/virus.txt} (83%) rename test/{test-files => files/emails}/auth/added-smtp-auth-spoofed-alias.txt (52%) rename test/{test-files => files/emails}/auth/added-smtp-auth-spoofed.txt (53%) rename test/{test-files => files/emails}/auth/ldap-smtp-auth-spoofed-alias.txt (57%) rename test/{test-files => files/emails}/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt (58%) rename test/{test-files => files/emails}/auth/ldap-smtp-auth-spoofed.txt (53%) rename test/{test-files/email-templates/existing-added.txt => files/emails/existing/added.txt} (67%) rename test/{test-files/email-templates/existing-alias-external.txt => files/emails/existing/alias-external.txt} (68%) rename test/{test-files/email-templates/existing-alias-local.txt => files/emails/existing/alias-local.txt} (68%) rename test/{test-files/email-templates/existing-alias-recipient-delimiter.txt => files/emails/existing/alias-recipient-delimiter.txt} (70%) rename test/{test-files/email-templates/existing-catchall-local.txt => files/emails/existing/catchall-local.txt} (68%) rename test/{test-files/email-templates/existing-regexp-alias-external.txt => files/emails/existing/regexp-alias-external.txt} (68%) rename test/{test-files/email-templates/existing-regexp-alias-local.txt => files/emails/existing/regexp-alias-local.txt} (68%) rename test/{test-files/email-templates/existing-user-and-cc-local-alias.txt => files/emails/existing/user-and-cc-local-alias.txt} (73%) rename test/{test-files/email-templates/existing-user1.txt => files/emails/existing/user1.txt} (67%) rename test/{test-files/email-templates/dsn-authenticated.txt => files/emails/nc_raw/dsn/authenticated.txt} (100%) rename test/{test-files/email-templates/dsn-unauthenticated.txt => files/emails/nc_raw/dsn/unauthenticated.txt} (100%) rename test/{test-files/email-templates => files/emails/nc_raw}/postscreen.txt (100%) rename test/{test-files/email-templates => files/emails/nc_raw}/smtp-only.txt (100%) rename test/{test-files/email-templates => files/emails}/non-existing-user.txt (67%) rename test/{test-files/email-templates => files/emails}/postgrey.txt (66%) create mode 100644 test/files/emails/postscreen.txt rename test/{test-files/email-templates/send-privacy-email.txt => files/emails/privacy.txt} (61%) rename test/{test-files/email-templates => files/emails}/quota-exceeded.txt (98%) rename test/{test-files/email-templates/rspamd-pass.txt => files/emails/rspamd/pass.txt} (57%) rename test/{test-files/email-templates/rspamd-spam-header.txt => files/emails/rspamd/spam-header.txt} (70%) rename test/{test-files/email-templates/rspamd-spam.txt => files/emails/rspamd/spam.txt} (70%) rename test/{test-files/email-templates/rspamd-virus.txt => files/emails/rspamd/virus.txt} (70%) rename test/{test-files/email-templates => files/emails/sendmail}/root-email.txt (100%) rename test/{test-files/email-templates/sieve-pipe.txt => files/emails/sieve/pipe.txt} (67%) rename test/{test-files/email-templates/sieve-spam-folder.txt => files/emails/sieve/spam-folder.txt} (64%) rename test/{test-files/email-templates => files/emails}/test-email.txt (100%) rename test/{test-files/nc_templates => files/nc}/imap_special_use_folders.txt (100%) rename test/{test-files/nc_templates => files/nc}/postgrey_whitelist.txt (100%) rename test/{test-files/nc_templates => files/nc}/postgrey_whitelist_recipients.txt (100%) rename test/{test-files/nc_templates => files/nc}/rspamd_imap_move_to_inbox.txt (100%) rename test/{test-files/nc_templates => files/nc}/rspamd_imap_move_to_junk.txt (100%) rename test/{test-files => files}/ssl/custom-dhe-params.pem (100%) rename test/{test-files => files}/ssl/example.test/README.md (100%) rename test/{test-files => files}/ssl/example.test/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/testssl.txt (100%) rename test/{test-files => files}/ssl/example.test/traefik.md (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/rsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/ca-key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/ecdsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/rsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json (100%) delete mode 100644 test/test-files/auth/added-smtp-auth-login-wrong.txt delete mode 100644 test/test-files/auth/added-smtp-auth-login.txt delete mode 100644 test/test-files/auth/added-smtp-auth-plain-wrong.txt delete mode 100644 test/test-files/auth/added-smtp-auth-plain.txt delete mode 100644 test/test-files/auth/sasl-ldap-smtp-auth.txt delete mode 100644 test/test-files/auth/smtp-auth-login-wrong.txt delete mode 100644 test/test-files/auth/smtp-auth-login.txt delete mode 100644 test/test-files/auth/smtp-auth-plain-wrong.txt delete mode 100644 test/test-files/auth/smtp-auth-plain.txt delete mode 100644 test/test-files/email-templates/existing-user2.txt delete mode 100644 test/test-files/email-templates/existing-user3.txt delete mode 100644 test/test-files/email-templates/smtp-ehlo.txt diff --git a/.gitattributes b/.gitattributes index d3dba13d..869c153e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,7 +10,7 @@ *.yml text ### Documentation (Project, Tests, Docs site) *.md text -### TLS certs (test/test-files/) + DHE params (target/shared/) +### TLS certs (test/files/) + DHE params (target/shared/) *.pem text *.pem.sha512sum text @@ -90,9 +90,9 @@ TrustedHosts text whitelist_recipients text ## MISC -### test/config/ + test/test-files/ +### test/config/ + test/files/ *.txt text -### test/linting/ (.ecrc.json) + test/test-files/ (*.acme.json): +### test/linting/ (.ecrc.json) + test/files/ (*.acme.json): *.json text ################################################# diff --git a/CHANGELOG.md b/CHANGELOG.md index f37fded4..eeeb843d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Updates + +- The test suite now uses `swaks` instead of `nc`, which has multiple benefits ([#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732)): + - `swaks` handles pipelining correctly, hence we can now use `reject_unauth_pipelining` in Postfix's configuration. + - `swaks` provides better CLI options that make many files superflous. + - `swaks` can also replace `openssl s_client` and handles authentication on submission ports better. + ## [v13.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.1.0) ### Added diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index ec468d41..566d5441 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -80,7 +80,7 @@ function _install_packages() { # `bind9-dnsutils` provides the `dig` command # `iputils-ping` provides the `ping` command DEBUG_PACKAGES=( - bind9-dnsutils iputils-ping less nano + bind9-dnsutils iputils-ping less nano swaks ) apt-get "${QUIET}" --no-install-recommends install \ diff --git a/test/test-files/auth/added-imap-auth.txt b/test/files/auth/added-imap-auth.txt similarity index 100% rename from test/test-files/auth/added-imap-auth.txt rename to test/files/auth/added-imap-auth.txt diff --git a/test/test-files/auth/added-pop3-auth.txt b/test/files/auth/added-pop3-auth.txt similarity index 100% rename from test/test-files/auth/added-pop3-auth.txt rename to test/files/auth/added-pop3-auth.txt diff --git a/test/test-files/auth/imap-auth.txt b/test/files/auth/imap-auth.txt similarity index 100% rename from test/test-files/auth/imap-auth.txt rename to test/files/auth/imap-auth.txt diff --git a/test/test-files/auth/imap-ldap-auth.txt b/test/files/auth/imap-ldap-auth.txt similarity index 100% rename from test/test-files/auth/imap-ldap-auth.txt rename to test/files/auth/imap-ldap-auth.txt diff --git a/test/test-files/auth/pop3-auth.txt b/test/files/auth/pop3-auth.txt similarity index 100% rename from test/test-files/auth/pop3-auth.txt rename to test/files/auth/pop3-auth.txt diff --git a/test/test-files/email-templates/amavis-spam.txt b/test/files/emails/amavis/spam.txt similarity index 63% rename from test/test-files/email-templates/amavis-spam.txt rename to test/files/emails/amavis/spam.txt index 66be1df3..e8d26138 100644 --- a/test/test-files/email-templates/amavis-spam.txt +++ b/test/files/emails/amavis/spam.txt @@ -1,13 +1,6 @@ -HELO mail.external.tld -MAIL FROM: spam@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message amavis-spam.txt +Subject: Test Message amavis/spam.txt This is a test mail. XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X - -. -QUIT diff --git a/test/test-files/email-templates/amavis-virus.txt b/test/files/emails/amavis/virus.txt similarity index 83% rename from test/test-files/email-templates/amavis-virus.txt rename to test/files/emails/amavis/virus.txt index 1343a07c..2c47dcad 100644 --- a/test/test-files/email-templates/amavis-virus.txt +++ b/test/files/emails/amavis/virus.txt @@ -1,11 +1,7 @@ -HELO mail.external.tld -MAIL FROM: virus@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message amavis-virus.txt +Subject: Test Message amavis/virus.txt Content-type: multipart/mixed; boundary="emailboundary" MIME-version: 1.0 @@ -27,6 +23,3 @@ ACAA/4EAAAAAZWljYXIuY29tUEsFBgAAAAABAAEANwAAAGsAAAAAAA== --emailboundary-- - -. -QUIT diff --git a/test/test-files/auth/added-smtp-auth-spoofed-alias.txt b/test/files/emails/auth/added-smtp-auth-spoofed-alias.txt similarity index 52% rename from test/test-files/auth/added-smtp-auth-spoofed-alias.txt rename to test/files/emails/auth/added-smtp-auth-spoofed-alias.txt index 48145183..eeb68ac8 100644 --- a/test/test-files/auth/added-smtp-auth-spoofed-alias.txt +++ b/test/files/emails/auth/added-smtp-auth-spoofed-alias.txt @@ -1,14 +1,5 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -MAIL FROM: alias1@localhost.localdomain -RCPT TO: user1@localhost.localdomain -DATA From: user1_alias To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail. - -. -QUIT diff --git a/test/test-files/auth/added-smtp-auth-spoofed.txt b/test/files/emails/auth/added-smtp-auth-spoofed.txt similarity index 53% rename from test/test-files/auth/added-smtp-auth-spoofed.txt rename to test/files/emails/auth/added-smtp-auth-spoofed.txt index 279b6c0e..fd96d401 100644 --- a/test/test-files/auth/added-smtp-auth-spoofed.txt +++ b/test/files/emails/auth/added-smtp-auth-spoofed.txt @@ -1,14 +1,5 @@ -EHLO mail -AUTH LOGIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -MAIL FROM: user2@localhost.localdomain -RCPT TO: user1@localhost.localdomain -DATA From: Not_My_Business To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail. - -. -QUIT diff --git a/test/test-files/auth/ldap-smtp-auth-spoofed-alias.txt b/test/files/emails/auth/ldap-smtp-auth-spoofed-alias.txt similarity index 57% rename from test/test-files/auth/ldap-smtp-auth-spoofed-alias.txt rename to test/files/emails/auth/ldap-smtp-auth-spoofed-alias.txt index 007b0f99..7453675c 100644 --- a/test/test-files/auth/ldap-smtp-auth-spoofed-alias.txt +++ b/test/files/emails/auth/ldap-smtp-auth-spoofed-alias.txt @@ -1,15 +1,5 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyQGxvY2FsaG9zdC5sb2NhbGRvbWFpbg== -c2VjcmV0 -MAIL FROM: postmaster@localhost.localdomain -RCPT TO: some.user@localhost.localdomain -DATA From: alias_address To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail from ldap-smtp-auth-spoofed-alias.txt - -. -QUIT diff --git a/test/test-files/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt b/test/files/emails/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt similarity index 58% rename from test/test-files/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt rename to test/files/emails/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt index bc0447af..3b500bf6 100644 --- a/test/test-files/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt +++ b/test/files/emails/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt @@ -1,15 +1,5 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyLmVtYWlsQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgo= -c2VjcmV0 -MAIL FROM: randomspoofedaddress@localhost.localdomain -RCPT TO: some.user@localhost.localdomain -DATA From: spoofed_address To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail from ldap-smtp-auth-spoofed-sender-with-filter-exception.txt - -. -QUIT diff --git a/test/test-files/auth/ldap-smtp-auth-spoofed.txt b/test/files/emails/auth/ldap-smtp-auth-spoofed.txt similarity index 53% rename from test/test-files/auth/ldap-smtp-auth-spoofed.txt rename to test/files/emails/auth/ldap-smtp-auth-spoofed.txt index cc0b164d..83193e17 100644 --- a/test/test-files/auth/ldap-smtp-auth-spoofed.txt +++ b/test/files/emails/auth/ldap-smtp-auth-spoofed.txt @@ -1,15 +1,5 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyQGxvY2FsaG9zdC5sb2NhbGRvbWFpbg== -c2VjcmV0 -MAIL FROM: ldap@localhost.localdomain -RCPT TO: user1@localhost.localdomain -DATA From: forged_address To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-added.txt b/test/files/emails/existing/added.txt similarity index 67% rename from test/test-files/email-templates/existing-added.txt rename to test/files/emails/existing/added.txt index 320fa4d2..827b681f 100644 --- a/test/test-files/email-templates/existing-added.txt +++ b/test/files/emails/existing/added.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: added@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-added.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-alias-external.txt b/test/files/emails/existing/alias-external.txt similarity index 68% rename from test/test-files/email-templates/existing-alias-external.txt rename to test/files/emails/existing/alias-external.txt index 61b1df3c..03f1af6c 100644 --- a/test/test-files/email-templates/existing-alias-external.txt +++ b/test/files/emails/existing/alias-external.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: alias1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-alias-external.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-alias-local.txt b/test/files/emails/existing/alias-local.txt similarity index 68% rename from test/test-files/email-templates/existing-alias-local.txt rename to test/files/emails/existing/alias-local.txt index c1bbc890..9b481a98 100644 --- a/test/test-files/email-templates/existing-alias-local.txt +++ b/test/files/emails/existing/alias-local.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: alias2@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local Alias Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-alias-local.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-alias-recipient-delimiter.txt b/test/files/emails/existing/alias-recipient-delimiter.txt similarity index 70% rename from test/test-files/email-templates/existing-alias-recipient-delimiter.txt rename to test/files/emails/existing/alias-recipient-delimiter.txt index 47b01397..07cb8d40 100644 --- a/test/test-files/email-templates/existing-alias-recipient-delimiter.txt +++ b/test/files/emails/existing/alias-recipient-delimiter.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: alias1~test@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local Alias With Delimiter Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-alias-recipient-delimiter.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-catchall-local.txt b/test/files/emails/existing/catchall-local.txt similarity index 68% rename from test/test-files/email-templates/existing-catchall-local.txt rename to test/files/emails/existing/catchall-local.txt index c80db170..ab3e1988 100644 --- a/test/test-files/email-templates/existing-catchall-local.txt +++ b/test/files/emails/existing/catchall-local.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: wildcard@localdomain2.com -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-catchall-local.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-regexp-alias-external.txt b/test/files/emails/existing/regexp-alias-external.txt similarity index 68% rename from test/test-files/email-templates/existing-regexp-alias-external.txt rename to test/files/emails/existing/regexp-alias-external.txt index 0e214db4..b50ac90f 100644 --- a/test/test-files/email-templates/existing-regexp-alias-external.txt +++ b/test/files/emails/existing/regexp-alias-external.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: bounce-always@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-regexp-alias-external.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-regexp-alias-local.txt b/test/files/emails/existing/regexp-alias-local.txt similarity index 68% rename from test/test-files/email-templates/existing-regexp-alias-local.txt rename to test/files/emails/existing/regexp-alias-local.txt index 6af46e92..e45b7c6c 100644 --- a/test/test-files/email-templates/existing-regexp-alias-local.txt +++ b/test/files/emails/existing/regexp-alias-local.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: test123@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-regexp-alias-local.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-user-and-cc-local-alias.txt b/test/files/emails/existing/user-and-cc-local-alias.txt similarity index 73% rename from test/test-files/email-templates/existing-user-and-cc-local-alias.txt rename to test/files/emails/existing/user-and-cc-local-alias.txt index 5fcb333b..37814f91 100644 --- a/test/test-files/email-templates/existing-user-and-cc-local-alias.txt +++ b/test/files/emails/existing/user-and-cc-local-alias.txt @@ -1,13 +1,6 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Cc: Existing Local Alias Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-user-and-cc-local-alias.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-user1.txt b/test/files/emails/existing/user1.txt similarity index 67% rename from test/test-files/email-templates/existing-user1.txt rename to test/files/emails/existing/user1.txt index 5ab0333f..23d49dc9 100644 --- a/test/test-files/email-templates/existing-user1.txt +++ b/test/files/emails/existing/user1.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-user1.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/dsn-authenticated.txt b/test/files/emails/nc_raw/dsn/authenticated.txt similarity index 100% rename from test/test-files/email-templates/dsn-authenticated.txt rename to test/files/emails/nc_raw/dsn/authenticated.txt diff --git a/test/test-files/email-templates/dsn-unauthenticated.txt b/test/files/emails/nc_raw/dsn/unauthenticated.txt similarity index 100% rename from test/test-files/email-templates/dsn-unauthenticated.txt rename to test/files/emails/nc_raw/dsn/unauthenticated.txt diff --git a/test/test-files/email-templates/postscreen.txt b/test/files/emails/nc_raw/postscreen.txt similarity index 100% rename from test/test-files/email-templates/postscreen.txt rename to test/files/emails/nc_raw/postscreen.txt diff --git a/test/test-files/email-templates/smtp-only.txt b/test/files/emails/nc_raw/smtp-only.txt similarity index 100% rename from test/test-files/email-templates/smtp-only.txt rename to test/files/emails/nc_raw/smtp-only.txt diff --git a/test/test-files/email-templates/non-existing-user.txt b/test/files/emails/non-existing-user.txt similarity index 67% rename from test/test-files/email-templates/non-existing-user.txt rename to test/files/emails/non-existing-user.txt index 406f6755..3d92470e 100644 --- a/test/test-files/email-templates/non-existing-user.txt +++ b/test/files/emails/non-existing-user.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: nouser@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message non-existing-user.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/postgrey.txt b/test/files/emails/postgrey.txt similarity index 66% rename from test/test-files/email-templates/postgrey.txt rename to test/files/emails/postgrey.txt index 33a3b153..cdfe8f93 100644 --- a/test/test-files/email-templates/postgrey.txt +++ b/test/files/emails/postgrey.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Postgrey Test Message This is a test mail. - -. -QUIT diff --git a/test/files/emails/postscreen.txt b/test/files/emails/postscreen.txt new file mode 100644 index 00000000..732ac897 --- /dev/null +++ b/test/files/emails/postscreen.txt @@ -0,0 +1,5 @@ +From: Docker Mail Server +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message postscreen.txt +This is a test mail for postscreen. diff --git a/test/test-files/email-templates/send-privacy-email.txt b/test/files/emails/privacy.txt similarity index 61% rename from test/test-files/email-templates/send-privacy-email.txt rename to test/files/emails/privacy.txt index 0c51ec5b..1d3a1b96 100644 --- a/test/test-files/email-templates/send-privacy-email.txt +++ b/test/files/emails/privacy.txt @@ -1,15 +1,6 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -mail from: -rcpt to: -data From: Some User To: Some User User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:52.0) Gecko/20100101 Thunderbird/52.2.1 Subject: Test ESMTP Auth LOGIN and remove privacy This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/quota-exceeded.txt b/test/files/emails/quota-exceeded.txt similarity index 98% rename from test/test-files/email-templates/quota-exceeded.txt rename to test/files/emails/quota-exceeded.txt index 71d221a1..c5281637 100644 --- a/test/test-files/email-templates/quota-exceeded.txt +++ b/test/files/emails/quota-exceeded.txt @@ -1,7 +1,3 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: quotauser@otherdomain.tld -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 @@ -20,6 +16,3 @@ Et voluptatum nobis ut odio voluptatem et quibusdam fugit ut libero sapiente vel Sit sint obcaecati et reiciendis tenetur aut dolorum culpa. Ab veritatis maxime qui necessitatibus facilis eum voluptate asperiores non totam omnis. Nam modi officia in reiciendis odit sit rerum laudantium est rerum voluptatem ut fugit cupiditate! Sit atque sint aut delectus omnis ut asperiores enim quo reprehenderit quae! In quasi nemo ut error totam ut quia harum ut commodi tenetur? Non quod dolorum eum explicabo labore vel asperiores quas est perferendis nulla eum nemo tenetur. Ut libero blanditiis ex voluptatibus repudiandae ab reiciendis nemo id debitis impedit hic quia incidunt sed quam excepturi ut magnam odit. Qui dolor deleniti aut sunt voluptas aut blanditiis distinctio nam omnis deleniti hic omnis rerum eum magni voluptatem. Nam labore facere eum molestiae dolorum ea consectetur praesentium ut cupiditate iste ad magnam aut neque maiores! Et excepturi ducimus ut nemo voluptas eum voluptas nihil hic perferendis quos vel quasi nesciunt est praesentium dolore hic quia quis. Et maxime ducimus ea cupiditate voluptatem ad quia dolores! Sed quos quaerat vel aperiam minus non sapiente quia ut ratione dolore eum officiis rerum. Non dolor vitae qui facilis dignissimos aut voluptate odit et ullam consequuntur. Et laudantium perspiciatis sit nisi temporibus a temporibus itaque ut iure dolor a voluptatum mollitia eos officia nobis et quibusdam voluptas. Amet eligendi eos nulla corporis et blanditiis nihil vel eveniet veritatis et sunt perferendis id molestiae eius! Quo harum quod aut nemo autem ut adipisci sint sed quia sunt. Aut voluptas error ut quae perferendis eos adipisci internos. Nam rerum fugiat aut minima nostrum quo repellendus quas exercitationem tenetur. Et molestiae architecto id quibusdam reprehenderit et magnam aliquam! Quo tempora veritatis At dolorem sint ex nulla blanditiis At voluptas laudantium est molestiae exercitationem et sequi voluptates aut ipsa atque. Et animi ipsum aut atque recusandae ea nemo ullam non quisquam quos sit libero sint vel libero delectus. Eos labore quidem a velit obcaecati nam explicabo consequatur eos maxime blanditiis? Et ipsam molestiae non quia explicabo ex galisum repudiandae et tempora veniam. Sed optio repellendus ut consequatur temporibus et harum quas hic ipsa officia? Aut dolores ipsum sit nulla dignissimos id quia perferendis aut dolores dolor et quibusdam porro aut Quis consequatur. - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-pass.txt b/test/files/emails/rspamd/pass.txt similarity index 57% rename from test/test-files/email-templates/rspamd-pass.txt rename to test/files/emails/rspamd/pass.txt index 0f244740..ce9286b1 100644 --- a/test/test-files/email-templates/rspamd-pass.txt +++ b/test/files/emails/rspamd/pass.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: pass@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message rspamd-pass.txt +Subject: Test Message rspamd/pass.txt This mail should pass and Rspamd should not mark it. - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-spam-header.txt b/test/files/emails/rspamd/spam-header.txt similarity index 70% rename from test/test-files/email-templates/rspamd-spam-header.txt rename to test/files/emails/rspamd/spam-header.txt index 7be1a56d..8722e42f 100644 --- a/test/test-files/email-templates/rspamd-spam-header.txt +++ b/test/files/emails/rspamd/spam-header.txt @@ -1,12 +1,5 @@ -HELO mail.example.test -MAIL FROM: spam-header@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-spam-header.txt YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-spam.txt b/test/files/emails/rspamd/spam.txt similarity index 70% rename from test/test-files/email-templates/rspamd-spam.txt rename to test/files/emails/rspamd/spam.txt index 88bd719c..c561e779 100644 --- a/test/test-files/email-templates/rspamd-spam.txt +++ b/test/files/emails/rspamd/spam.txt @@ -1,12 +1,5 @@ -HELO mail.example.test -MAIL FROM: spam@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-spam.txt XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-virus.txt b/test/files/emails/rspamd/virus.txt similarity index 70% rename from test/test-files/email-templates/rspamd-virus.txt rename to test/files/emails/rspamd/virus.txt index c745f261..cb18927d 100644 --- a/test/test-files/email-templates/rspamd-virus.txt +++ b/test/files/emails/rspamd/virus.txt @@ -1,12 +1,5 @@ -HELO mail.example.test -MAIL FROM: virus@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-virus.txt X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H* - -. -QUIT diff --git a/test/test-files/email-templates/root-email.txt b/test/files/emails/sendmail/root-email.txt similarity index 100% rename from test/test-files/email-templates/root-email.txt rename to test/files/emails/sendmail/root-email.txt diff --git a/test/test-files/email-templates/sieve-pipe.txt b/test/files/emails/sieve/pipe.txt similarity index 67% rename from test/test-files/email-templates/sieve-pipe.txt rename to test/files/emails/sieve/pipe.txt index f13dba87..4e8cfb39 100644 --- a/test/test-files/email-templates/sieve-pipe.txt +++ b/test/files/emails/sieve/pipe.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user2@otherdomain.tld -DATA From: Sieve-pipe-test To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Sieve pipe test message This is a test mail to sieve pipe. - -. -QUIT diff --git a/test/test-files/email-templates/sieve-spam-folder.txt b/test/files/emails/sieve/spam-folder.txt similarity index 64% rename from test/test-files/email-templates/sieve-spam-folder.txt rename to test/files/emails/sieve/spam-folder.txt index 8e802817..7ffd09a7 100644 --- a/test/test-files/email-templates/sieve-spam-folder.txt +++ b/test/files/emails/sieve/spam-folder.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Spambot To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message sieve-spam-folder.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/test-email.txt b/test/files/emails/test-email.txt similarity index 100% rename from test/test-files/email-templates/test-email.txt rename to test/files/emails/test-email.txt diff --git a/test/test-files/nc_templates/imap_special_use_folders.txt b/test/files/nc/imap_special_use_folders.txt similarity index 100% rename from test/test-files/nc_templates/imap_special_use_folders.txt rename to test/files/nc/imap_special_use_folders.txt diff --git a/test/test-files/nc_templates/postgrey_whitelist.txt b/test/files/nc/postgrey_whitelist.txt similarity index 100% rename from test/test-files/nc_templates/postgrey_whitelist.txt rename to test/files/nc/postgrey_whitelist.txt diff --git a/test/test-files/nc_templates/postgrey_whitelist_recipients.txt b/test/files/nc/postgrey_whitelist_recipients.txt similarity index 100% rename from test/test-files/nc_templates/postgrey_whitelist_recipients.txt rename to test/files/nc/postgrey_whitelist_recipients.txt diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt b/test/files/nc/rspamd_imap_move_to_inbox.txt similarity index 100% rename from test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt rename to test/files/nc/rspamd_imap_move_to_inbox.txt diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt b/test/files/nc/rspamd_imap_move_to_junk.txt similarity index 100% rename from test/test-files/nc_templates/rspamd_imap_move_to_junk.txt rename to test/files/nc/rspamd_imap_move_to_junk.txt diff --git a/test/test-files/ssl/custom-dhe-params.pem b/test/files/ssl/custom-dhe-params.pem similarity index 100% rename from test/test-files/ssl/custom-dhe-params.pem rename to test/files/ssl/custom-dhe-params.pem diff --git a/test/test-files/ssl/example.test/README.md b/test/files/ssl/example.test/README.md similarity index 100% rename from test/test-files/ssl/example.test/README.md rename to test/files/ssl/example.test/README.md diff --git a/test/test-files/ssl/example.test/cert.ecdsa.pem b/test/files/ssl/example.test/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/cert.ecdsa.pem rename to test/files/ssl/example.test/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/cert.rsa.pem b/test/files/ssl/example.test/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/cert.rsa.pem rename to test/files/ssl/example.test/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/key.ecdsa.pem b/test/files/ssl/example.test/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/key.ecdsa.pem rename to test/files/ssl/example.test/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/key.rsa.pem b/test/files/ssl/example.test/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/key.rsa.pem rename to test/files/ssl/example.test/key.rsa.pem diff --git a/test/test-files/ssl/example.test/testssl.txt b/test/files/ssl/example.test/testssl.txt similarity index 100% rename from test/test-files/ssl/example.test/testssl.txt rename to test/files/ssl/example.test/testssl.txt diff --git a/test/test-files/ssl/example.test/traefik.md b/test/files/ssl/example.test/traefik.md similarity index 100% rename from test/test-files/ssl/example.test/traefik.md rename to test/files/ssl/example.test/traefik.md diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json b/test/files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json rename to test/files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/key.rsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/key.rsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/rsa.acme.json b/test/files/ssl/example.test/with_ca/ecdsa/rsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/rsa.acme.json rename to test/files/ssl/example.test/with_ca/ecdsa/rsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json b/test/files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json rename to test/files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/cert.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/cert.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/ecdsa.acme.json b/test/files/ssl/example.test/with_ca/rsa/ecdsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/ecdsa.acme.json rename to test/files/ssl/example.test/with_ca/rsa/ecdsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/rsa/key.ecdsa.pem b/test/files/ssl/example.test/with_ca/rsa/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/rsa/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/key.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/key.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/rsa.acme.json b/test/files/ssl/example.test/with_ca/rsa/rsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/rsa.acme.json rename to test/files/ssl/example.test/with_ca/rsa/rsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json b/test/files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json rename to test/files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json diff --git a/test/helper/common.bash b/test/helper/common.bash index 8fb7854e..ab21ef60 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -469,5 +469,19 @@ function _print_mail_log_for_id() { _run_in_container grep -F "${MAIL_ID}" /var/log/mail.log } +# A simple wrapper for netcat (`nc`). This is useful when sending +# "raw" e-mails or doing IMAP-related work. +# +# @param ${1} = the file that is given to `nc` +# @param ${1} = custom parameters for `nc` [OPTIONAL] (default: 0.0.0.0 25) +function _nc_wrapper() { + local FILE=${1:?Must provide name of template file} + local NC_PARAMETERS=${2:-0.0.0.0 25} + + [[ -v CONTAINER_NAME ]] || return 1 + + _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}.txt" +} + # ? << Miscellaneous helper functions # ! ------------------------------------------------------------------- diff --git a/test/helper/sending.bash b/test/helper/sending.bash index 631617a1..48012178 100644 --- a/test/helper/sending.bash +++ b/test/helper/sending.bash @@ -8,11 +8,12 @@ # ! ATTENTION: This file requires helper functions from `common.sh`! # Sends a mail from localhost (127.0.0.1) to a container. To send -# a custom email, create a file at `test/test-files/`, +# a custom email, create a file at `test/files/`, # and provide `` as an argument to this function. # -# @param ${1} = template file (path) name -# @param ${2} = parameters for `nc` [OPTIONAL] (default: `0.0.0.0 25`) +# Parameters include all options that one can supply to `swaks` +# itself. The `--data` parameter expects a relative path from `emails/` +# where the contents will be implicitly provided to `swaks` via STDIN. # # ## Attention # @@ -23,17 +24,42 @@ # send the email but it will not make sure the mail queue is empty after the mail # has been sent. function _send_email() { - local TEMPLATE_FILE=${1:?Must provide name of template file} - local NC_PARAMETERS=${2:-0.0.0.0 25} + [[ -v CONTAINER_NAME ]] || return 1 - assert_not_equal "${NC_PARAMETERS}" '' - assert_not_equal "${CONTAINER_NAME:-}" '' + # Parameter defaults common to our testing needs: + local EHLO='mail.external.tld' + local FROM='user@external.tld' + local TO='user1@localhost.localdomain' + local SERVER='0.0.0.0' + local PORT=25 + # Extra options for `swaks` that aren't covered by the default options above: + local ADDITIONAL_SWAKS_OPTIONS=() + # Specifically for handling `--data` option below: + local FINAL_SWAKS_OPTIONS=() - _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${TEMPLATE_FILE}.txt" - assert_success + while [[ ${#} -gt 0 ]]; do + case "${1}" in + ( '--ehlo' ) EHLO=${2:?--ehlo given but no argument} ; shift 2 ;; + ( '--from' ) FROM=${2:?--from given but no argument} ; shift 2 ;; + ( '--to' ) TO=${2:?--to given but no argument} ; shift 2 ;; + ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;; + ( '--port' ) PORT=${2:?--port given but no argument} ; shift 2 ;; + ( '--data' ) + local TEMPLATE_FILE="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}.txt" + FINAL_SWAKS_OPTIONS+=('--data') + FINAL_SWAKS_OPTIONS+=('-') + FINAL_SWAKS_OPTIONS+=('<') + FINAL_SWAKS_OPTIONS+=("${TEMPLATE_FILE}") + shift 2 + ;; + ( * ) ADDITIONAL_SWAKS_OPTIONS+=("${1}") ; shift 1 ;; + esac + done + + _run_in_container_bash "swaks --server ${SERVER} --port ${PORT} --ehlo ${EHLO} --from ${FROM} --to ${TO} ${ADDITIONAL_SWAKS_OPTIONS[*]} ${FINAL_SWAKS_OPTIONS[*]}" } -# Like `_send_mail` with two major differences: +# Like `_send_email` with two major differences: # # 1. this function waits for the mail to be processed; there is no asynchronicity # because filtering the logs in a synchronous way is easier and safer! @@ -42,8 +68,7 @@ function _send_email() { # No. 2 is especially useful in case you send more than one email in a single # test file and need to assert certain log entries for each mail individually. # -# @param ${1} = template file (path) name -# @param ${2} = parameters for `nc` [OPTIONAL] (default: `0.0.0.0 25`) +# This function takes the same arguments as `_send_mail`. # # ## Attention # @@ -57,17 +82,13 @@ function _send_email() { # chosen. Sending more than one mail at any given point in time with this function # is UNDEFINED BEHAVIOR! function _send_email_and_get_id() { - local TEMPLATE_FILE=${1:?Must provide name of template file} - local NC_PARAMETERS=${2:-0.0.0.0 25} + [[ -v CONTAINER_NAME ]] || return 1 + + _wait_for_empty_mail_queue_in_container + _send_email "${@}" + _wait_for_empty_mail_queue_in_container + local MAIL_ID - - assert_not_equal "${NC_PARAMETERS}" '' - assert_not_equal "${CONTAINER_NAME:-}" '' - - _wait_for_empty_mail_queue_in_container - _send_email "${TEMPLATE_FILE}" - _wait_for_empty_mail_queue_in_container - # The unique ID Postfix (and other services) use may be different in length # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a # range to safely capture it. diff --git a/test/helper/setup.bash b/test/helper/setup.bash index 65e2999f..0dd57bd6 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -98,7 +98,7 @@ function _init_with_defaults() { # Common complimentary test files, read-only safe to share across containers: export TEST_FILES_CONTAINER_PATH='/tmp/docker-mailserver-test' - export TEST_FILES_VOLUME="${REPOSITORY_ROOT}/test/test-files:${TEST_FILES_CONTAINER_PATH}:ro" + export TEST_FILES_VOLUME="${REPOSITORY_ROOT}/test/files:${TEST_FILES_CONTAINER_PATH}:ro" # The config volume cannot be read-only as some data needs to be written at container startup # diff --git a/test/test-files/auth/added-smtp-auth-login-wrong.txt b/test/test-files/auth/added-smtp-auth-login-wrong.txt deleted file mode 100644 index a75856f1..00000000 --- a/test/test-files/auth/added-smtp-auth-login-wrong.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWlu -Bn3JKisq4HQ2RO== -QUIT diff --git a/test/test-files/auth/added-smtp-auth-login.txt b/test/test-files/auth/added-smtp-auth-login.txt deleted file mode 100644 index 5276b7f4..00000000 --- a/test/test-files/auth/added-smtp-auth-login.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -QUIT diff --git a/test/test-files/auth/added-smtp-auth-plain-wrong.txt b/test/test-files/auth/added-smtp-auth-plain-wrong.txt deleted file mode 100644 index 6ce5a383..00000000 --- a/test/test-files/auth/added-smtp-auth-plain-wrong.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWluAGFkZGVkQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgBCQURQQVNTV09SRA== -QUIT diff --git a/test/test-files/auth/added-smtp-auth-plain.txt b/test/test-files/auth/added-smtp-auth-plain.txt deleted file mode 100644 index ed48d77d..00000000 --- a/test/test-files/auth/added-smtp-auth-plain.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWluAGFkZGVkQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgBteXBhc3N3b3Jk -QUIT diff --git a/test/test-files/auth/sasl-ldap-smtp-auth.txt b/test/test-files/auth/sasl-ldap-smtp-auth.txt deleted file mode 100644 index df4d7db4..00000000 --- a/test/test-files/auth/sasl-ldap-smtp-auth.txt +++ /dev/null @@ -1,5 +0,0 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyQGxvY2FsaG9zdC5sb2NhbGRvbWFpbg== -c2VjcmV0 -QUIT diff --git a/test/test-files/auth/smtp-auth-login-wrong.txt b/test/test-files/auth/smtp-auth-login-wrong.txt deleted file mode 100644 index 39b4f01c..00000000 --- a/test/test-files/auth/smtp-auth-login-wrong.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -Bn3JKisq4HQ2RO== -QUIT diff --git a/test/test-files/auth/smtp-auth-login.txt b/test/test-files/auth/smtp-auth-login.txt deleted file mode 100644 index 50ff99f3..00000000 --- a/test/test-files/auth/smtp-auth-login.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -QUIT diff --git a/test/test-files/auth/smtp-auth-plain-wrong.txt b/test/test-files/auth/smtp-auth-plain-wrong.txt deleted file mode 100644 index d8d8ad2a..00000000 --- a/test/test-files/auth/smtp-auth-plain-wrong.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN WRONGPASSWORD -QUIT diff --git a/test/test-files/auth/smtp-auth-plain.txt b/test/test-files/auth/smtp-auth-plain.txt deleted file mode 100644 index 2e60fdc3..00000000 --- a/test/test-files/auth/smtp-auth-plain.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWluAHVzZXIxQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgBteXBhc3N3b3Jk -QUIT diff --git a/test/test-files/email-templates/existing-user2.txt b/test/test-files/email-templates/existing-user2.txt deleted file mode 100644 index 63554f27..00000000 --- a/test/test-files/email-templates/existing-user2.txt +++ /dev/null @@ -1,12 +0,0 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user2@otherdomain.tld -DATA -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-user2.txt -This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-user3.txt b/test/test-files/email-templates/existing-user3.txt deleted file mode 100644 index facd5328..00000000 --- a/test/test-files/email-templates/existing-user3.txt +++ /dev/null @@ -1,12 +0,0 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user3@localhost.localdomain -DATA -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:33 -0400 -Subject: Test Message existing-user1.txt -This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/smtp-ehlo.txt b/test/test-files/email-templates/smtp-ehlo.txt deleted file mode 100644 index 05524efd..00000000 --- a/test/test-files/email-templates/smtp-ehlo.txt +++ /dev/null @@ -1,2 +0,0 @@ -EHLO mail.localhost -QUIT diff --git a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats index 2c176235..81cf9bc1 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats @@ -225,9 +225,12 @@ function teardown_file() { _default_teardown ; } sleep 10 # send some big emails - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' + assert_success + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' + assert_success + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' + assert_success # check for quota warn message existence run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/ assert_success diff --git a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats index c2e9e6c7..e3e076a5 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats @@ -26,9 +26,11 @@ function setup_file() { _wait_for_smtp_port_in_container # Single mail sent from 'spam@spam.com' that is handled by User (relocate) and Global (copy) sieves for user1: - _send_email 'email-templates/sieve-spam-folder' + _send_email --data 'sieve/spam-folder' + assert_success # Mail for user2 triggers the sieve-pipe: - _send_email 'email-templates/sieve-pipe' + _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe' + assert_success _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats index 8ce03d9a..033a5bde 100644 --- a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats +++ b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats @@ -26,7 +26,8 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _wait_for_empty_mail_queue_in_container # Mail received should be stored as `u.1` (one file per message) @@ -47,7 +48,8 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _wait_for_empty_mail_queue_in_container # Mail received should be stored in `m.1` (1 or more messages) diff --git a/test/tests/parallel/set1/dovecot/special_use_folders.bats b/test/tests/parallel/set1/dovecot/special_use_folders.bats index e70899a0..fe1f554e 100644 --- a/test/tests/parallel/set1/dovecot/special_use_folders.bats +++ b/test/tests/parallel/set1/dovecot/special_use_folders.bats @@ -14,7 +14,8 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'normal delivery works' { - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new 1 } @@ -26,7 +27,7 @@ function teardown_file() { _default_teardown ; } } @test "(IMAP) special-use folders should be created when necessary" { - _send_email 'nc_templates/imap_special_use_folders' '-w 8 0.0.0.0 143' + _nc_wrapper 'nc/imap_special_use_folders' '-w 8 0.0.0.0 143' assert_output --partial 'Drafts' assert_output --partial 'Junk' assert_output --partial 'Trash' diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index 31608ef8..9232f90f 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -25,34 +25,35 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - _send_email 'email-templates/amavis-virus' + _send_email --from 'virus@external.tld' --data 'amavis/virus' + assert_success _wait_for_empty_mail_queue_in_container } function teardown_file() { _default_teardown ; } -@test "log files exist at /var/log/mail directory" { +@test 'log files exist at /var/log/mail directory' { _run_in_container_bash "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log' | wc -l" assert_success assert_output 3 } -@test "should be identified by Amavis" { +@test 'should be identified by Amavis' { _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log assert_success } -@test "freshclam cron is enabled" { +@test 'freshclam cron is enabled' { _run_in_container_bash "grep '/usr/bin/freshclam' -r /etc/cron.d" assert_success } -@test "env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly" { +@test 'env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly' { _run_in_container grep -q '^MaxFileSize 30M$' /etc/clamav/clamd.conf assert_success } -@test "rejects virus" { +@test 'rejects virus' { _run_in_container_bash "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep ' -> '" assert_success } diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index 8402422c..f2474cc0 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -12,12 +12,14 @@ function setup_file() { --env ENABLE_CLAMAV=0 --env ENABLE_SPAMASSASSIN=0 --env AMAVIS_LOGLEVEL=2 + --env PERMIT_DOCKER=container ) _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/spam_virus/fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats index 9ae30758..8a03ba04 100644 --- a/test/tests/parallel/set1/spam_virus/fail2ban.bats +++ b/test/tests/parallel/set1/spam_virus/fail2ban.bats @@ -73,8 +73,17 @@ function teardown_file() { @test "ban ip on multiple failed login" { CONTAINER1_IP=$(_get_container_ip "${CONTAINER1_NAME}") # Trigger a ban by failing to login twice: - CONTAINER_NAME=${CONTAINER2_NAME} _send_email 'auth/smtp-auth-login-wrong' "${CONTAINER1_IP} 465" - CONTAINER_NAME=${CONTAINER2_NAME} _send_email 'auth/smtp-auth-login-wrong' "${CONTAINER1_IP} 465" + for _ in {1..2}; do + CONTAINER_NAME=${CONTAINER2_NAME} _send_email \ + --server "${CONTAINER1_IP}" \ + --port 465 \ + --auth PLAIN \ + --auth-user user1@localhost.localdomain \ + --auth-password wrongpassword + assert_failure + assert_output --partial 'authentication failed' + assert_output --partial 'No authentication type succeeded' + done # Checking that CONTAINER2_IP is banned in "${CONTAINER1_NAME}" CONTAINER2_IP=$(_get_container_ip "${CONTAINER2_NAME}") diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index e32210ca..316e3350 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -51,17 +51,15 @@ function teardown_file() { _default_teardown ; } _reload_postfix # Send test mail (it should fail to deliver): - _send_test_mail '/tmp/docker-mailserver-test/email-templates/postgrey.txt' '25' + _send_email --from 'user@external.tld' --port 25 --data 'postgrey' + assert_failure + assert_output --partial 'Recipient address rejected: Delayed by Postgrey' # Confirm mail was greylisted: _should_have_log_entry \ 'action=greylist' \ 'reason=new' \ 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' - - _repeat_until_success_or_timeout 10 _run_in_container grep \ - 'Recipient address rejected: Delayed by Postgrey' \ - /var/log/mail/mail.log } # NOTE: This test case depends on the previous one @@ -69,7 +67,8 @@ function teardown_file() { _default_teardown ; } # Wait until `$POSTGREY_DELAY` seconds pass before trying again: sleep 3 # Retry delivering test mail (it should be trusted this time): - _send_test_mail '/tmp/docker-mailserver-test/email-templates/postgrey.txt' '25' + _send_email --from 'user@external.tld' --port 25 --data 'postgrey' + assert_success # Confirm postgrey permitted delivery (triplet is now trusted): _should_have_log_entry \ @@ -78,8 +77,9 @@ function teardown_file() { _default_teardown ; } 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' } - -# NOTE: These two whitelist tests use `test-files/nc_templates/` instead of `test-files/email-templates`. +# NOTE: These two whitelist tests use `files/nc/` instead of `files/emails`. +# `nc` option `-w 0` terminates the connection after sending the template, it does not wait for a response. +# This is required for port 10023, otherwise the connection never drops. # - This allows to bypass the SMTP protocol on port 25, and send data directly to Postgrey instead. # - Appears to be a workaround due to `client_name=localhost` when sent from Postfix. # - Could send over port 25 if whitelisting `localhost`, @@ -87,7 +87,7 @@ function teardown_file() { _default_teardown ; } # - It'd also cause the earlier greylist test to fail. # - TODO: Actually confirm whitelist feature works correctly as these test cases are using a workaround: @test "should whitelist sender 'user@whitelist.tld'" { - _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt' '10023' + _nc_wrapper 'nc/postgrey_whitelist' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ @@ -96,7 +96,7 @@ function teardown_file() { _default_teardown ; } } @test "should whitelist recipient 'user2@otherdomain.tld'" { - _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt' '10023' + _nc_wrapper 'nc/postgrey_whitelist_recipients' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ @@ -104,21 +104,10 @@ function teardown_file() { _default_teardown ; } 'client_address=127.0.0.1/32, sender=test@nonwhitelist.tld, recipient=user2@otherdomain.tld' } -function _send_test_mail() { - local MAIL_TEMPLATE=$1 - local PORT=${2:-25} - - # `-w 0` terminates the connection after sending the template, it does not wait for a response. - # This is required for port 10023, otherwise the connection never drops. - # It could increase the number of seconds to wait for port 25 to allow for asserting a response, - # but that would enforce the delay in tests for port 10023. - _run_in_container_bash "nc -w 0 0.0.0.0 ${PORT} < ${MAIL_TEMPLATE}" -} - function _should_have_log_entry() { - local ACTION=$1 - local REASON=$2 - local TRIPLET=$3 + local ACTION=${1} + local REASON=${2} + local TRIPLET=${3} # Allow some extra time for logs to update to avoids a false-positive failure: _run_until_success_or_timeout 10 _exec_in_container grep \ diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index a1ddeb29..377b2479 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -37,46 +37,35 @@ function teardown_file() { docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" } +# `POSTSCREEN_ACTION=enforce` (DMS default) should reject delivery with a 550 SMTP reply +# A legitimate mail client should speak SMTP by waiting it's turn, which postscreen defaults enforce (only on port 25) +# https://www.postfix.org/postconf.5.html#postscreen_greet_wait +# +# Use `nc` to send all SMTP commands at once instead (emulate a misbehaving client that should be rejected) +# NOTE: Postscreen only runs on port 25, avoid implicit ports in test methods @test 'should fail send when talking out of turn' { - CONTAINER_NAME=${CONTAINER2_NAME} _send_email 'email-templates/postscreen' "${CONTAINER1_IP} 25" + CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen' "${CONTAINER1_IP} 25" + # Expected postscreen log entry: assert_output --partial 'Protocol error' - # Expected postscreen log entry: - _run_in_container cat /var/log/mail/mail.log + _run_in_container cat /var/log/mail.log assert_output --partial 'COMMAND PIPELINING' + assert_output --partial 'DATA without valid RCPT' } @test "should successfully pass postscreen and get postfix greeting message (respecting postscreen_greet_wait time)" { - # NOTE: Sometimes fails on first attempt (trying too soon?), - # Instead of a `run` + asserting partial, Using repeat + internal grep match: - _repeat_until_success_or_timeout 10 _should_wait_turn_speaking_smtp \ - "${CONTAINER2_NAME}" \ - "${CONTAINER1_IP}" \ - '/tmp/docker-mailserver-test/email-templates/postscreen.txt' \ - '220 mail.example.test ESMTP' + # Configure `send_email()` to send from the mail client container (CONTAINER2_NAME) via ENV override, + # mail is sent to the DMS server container (CONTAINER1_NAME) via `--server` parameter: + CONTAINER_NAME=${CONTAINER2_NAME} _send_email --server "${CONTAINER1_IP}" --port 25 --data 'postscreen' + # NOTE: Cannot assert_success due to sender address not being resolvable. + # TODO: Uncomment when proper resolution of domain names is possible: + # assert_success - # Expected postscreen log entry: - _run_in_container cat /var/log/mail/mail.log + # TODO: Prefer this approach when `_send_email_and_get_id()` can support separate client and server containers: + # local MAIL_ID=$(_send_email_and_get_id --port 25 --data 'postscreen') + # _print_mail_log_for_id "${MAIL_ID}" + # assert_output --partial "stored mail into mailbox 'INBOX'" + + _run_in_container cat /var/log/mail.log assert_output --partial 'PASS NEW' } - -# When postscreen is active, it prevents the usual method of piping a file through nc: -# (Won't work: CONTAINER_NAME=${CLIENT_CONTAINER_NAME} _send_email "${SMTP_TEMPLATE}" "${TARGET_CONTAINER_IP} 25") -# The below workaround respects `postscreen_greet_wait` time (default 6 sec), talking to the mail-server in turn: -# https://www.postfix.org/postconf.5.html#postscreen_greet_wait -function _should_wait_turn_speaking_smtp() { - local CLIENT_CONTAINER_NAME=$1 - local TARGET_CONTAINER_IP=$2 - local SMTP_TEMPLATE=$3 - local EXPECTED=$4 - - # shellcheck disable=SC2016 - local UGLY_WORKAROUND='exec 3<>/dev/tcp/'"${TARGET_CONTAINER_IP}"'/25 && \ - while IFS= read -r cmd; do \ - head -1 <&3; \ - [[ ${cmd} == "EHLO"* ]] && sleep 6; \ - echo ${cmd} >&3; \ - done < '"${SMTP_TEMPLATE}" - - docker exec "${CLIENT_CONTAINER_NAME}" bash -c "${UGLY_WORKAROUND}" | grep "${EXPECTED}" -} diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index ba8a23f5..2e610d72 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -45,10 +45,10 @@ function setup_file() { # We will send 3 emails: the first one should pass just fine; the second one should # be rejected due to spam; the third one should be rejected due to a virus. - export MAIL_ID1=$(_send_email_and_get_id 'email-templates/rspamd-pass') - export MAIL_ID2=$(_send_email_and_get_id 'email-templates/rspamd-spam') - export MAIL_ID3=$(_send_email_and_get_id 'email-templates/rspamd-virus') - export MAIL_ID4=$(_send_email_and_get_id 'email-templates/rspamd-spam-header') + export MAIL_ID1=$(_send_email_and_get_id --from 'rspamd-pass@example.test' --data 'rspamd/pass') + export MAIL_ID2=$(_send_email_and_get_id --from 'rspamd-spam@example.test' --data 'rspamd/spam') + export MAIL_ID3=$(_send_email_and_get_id --from 'rspamd-virus@example.test' --data 'rspamd/virus') + export MAIL_ID4=$(_send_email_and_get_id --from 'rspamd-spam-header@example.test' --data 'rspamd/spam-header') for ID in MAIL_ID{1,2,3,4}; do [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; } @@ -256,7 +256,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "Junk" folder from "INBOX"; the first email we # sent should pass fine, hence we can now move it. - _send_email 'nc_templates/rspamd_imap_move_to_junk' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_junk' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log @@ -270,7 +270,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "INBOX" folder from "Junk"; there should be two mails # in the "Junk" folder, since the second email we sent during setup should # have landed in the Junk folder already. - _send_email 'nc_templates/rspamd_imap_move_to_inbox' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_inbox' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 94a9b9c4..fea23b0b 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -95,7 +95,7 @@ function teardown() { _default_teardown ; } function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis - _send_email 'email-templates/amavis-spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam' } function _should_be_received_by_amavis() { diff --git a/test/tests/parallel/set1/tls/dhparams.bats b/test/tests/parallel/set1/tls/dhparams.bats index 3157034c..8b3047d1 100644 --- a/test/tests/parallel/set1/tls/dhparams.bats +++ b/test/tests/parallel/set1/tls/dhparams.bats @@ -38,7 +38,7 @@ function teardown() { _default_teardown ; } # - A warning is raised about usage of potentially insecure parameters. @test "Custom" { export CONTAINER_NAME=${CONTAINER2_NAME} - local DH_PARAMS_CUSTOM='test/test-files/ssl/custom-dhe-params.pem' + local DH_PARAMS_CUSTOM='test/files/ssl/custom-dhe-params.pem' local DH_CHECKSUM_CUSTOM=$(sha512sum "${DH_PARAMS_CUSTOM}" | awk '{print $1}') _init_with_defaults diff --git a/test/tests/parallel/set1/tls/letsencrypt.bats b/test/tests/parallel/set1/tls/letsencrypt.bats index 91a05997..bcdb1758 100644 --- a/test/tests/parallel/set1/tls/letsencrypt.bats +++ b/test/tests/parallel/set1/tls/letsencrypt.bats @@ -88,7 +88,7 @@ function _initial_setup() { # All of these certs support both FQDNs (`mail.example.test` and `example.test`), # Except for the wildcard cert (`*.example.test`), that was created with `example.test` intentionally excluded from SAN. # We want to maintain the same FQDN (`mail.example.test`) between the _acme_ecdsa and _acme_rsa tests. - local LOCAL_BASE_PATH="${PWD}/test/test-files/ssl/example.test/with_ca/rsa" + local LOCAL_BASE_PATH="${PWD}/test/files/ssl/example.test/with_ca/rsa" function _prepare() { # Default `acme.json` for _acme_ecdsa test: @@ -240,7 +240,7 @@ function _copy_to_letsencrypt_storage() { FQDN_DIR=$(echo "${DEST}" | cut -d '/' -f1) mkdir -p "${TEST_TMP_CONFIG}/letsencrypt/${FQDN_DIR}" - if ! cp "${PWD}/test/test-files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}"; then + if ! cp "${PWD}/test/files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}"; then echo "Could not copy cert file '${SRC}'' to '${DEST}'" >&2 exit 1 fi diff --git a/test/tests/parallel/set1/tls/manual.bats b/test/tests/parallel/set1/tls/manual.bats index 2a55f14f..c082d6ed 100644 --- a/test/tests/parallel/set1/tls/manual.bats +++ b/test/tests/parallel/set1/tls/manual.bats @@ -20,7 +20,7 @@ function setup_file() { export TEST_DOMAIN='example.test' local CUSTOM_SETUP_ARGUMENTS=( - --volume "${PWD}/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/:/config/ssl/:ro" + --volume "${PWD}/test/files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/:/config/ssl/:ro" --env LOG_LEVEL='trace' --env SSL_TYPE='manual' --env TLS_LEVEL='modern' @@ -108,10 +108,10 @@ function teardown_file() { _default_teardown ; } @test "manual cert changes are picked up by check-for-changes" { printf '%s' 'someThingsChangedHere' \ - >>"$(pwd)/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" + >>"$(pwd)/test/files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" run timeout 15 docker exec "${CONTAINER_NAME}" bash -c "tail -F /var/log/supervisor/changedetector.log | sed '/Manual certificates have changed/ q'" assert_success - sed -i '/someThingsChangedHere/d' "$(pwd)/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" + sed -i '/someThingsChangedHere/d' "$(pwd)/test/files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" } diff --git a/test/tests/parallel/set2/tls_cipherlists.bats b/test/tests/parallel/set2/tls_cipherlists.bats index 2b9511b9..3429f516 100644 --- a/test/tests/parallel/set2/tls_cipherlists.bats +++ b/test/tests/parallel/set2/tls_cipherlists.bats @@ -17,7 +17,7 @@ function setup_file() { # Contains various certs for testing TLS support (read-only): export TLS_CONFIG_VOLUME - TLS_CONFIG_VOLUME="${PWD}/test/test-files/ssl/${TEST_DOMAIN}/:/config/ssl/:ro" + TLS_CONFIG_VOLUME="${PWD}/test/files/ssl/${TEST_DOMAIN}/:/config/ssl/:ro" # Used for connecting testssl and DMS containers via network name `TEST_DOMAIN`: # NOTE: If the network already exists, the test will fail to start diff --git a/test/tests/parallel/set3/container_configuration/hostname.bats b/test/tests/parallel/set3/container_configuration/hostname.bats index fcb84b28..f5774eef 100644 --- a/test/tests/parallel/set3/container_configuration/hostname.bats +++ b/test/tests/parallel/set3/container_configuration/hostname.bats @@ -207,7 +207,7 @@ function _should_have_correct_mail_headers() { # (eg: OVERRIDE_HOSTNAME or `--hostname mail --domainname example.test`) local EXPECTED_HOSTNAME=${3:-${EXPECTED_FQDN}} - _send_email 'email-templates/existing-user1' + _send_email --from 'user@external.tld' --data 'existing/user1' _wait_for_empty_mail_queue_in_container _count_files_in_directory_in_container '/var/mail/localhost.localdomain/user1/new/' '1' diff --git a/test/tests/parallel/set3/mta/dsn.bats b/test/tests/parallel/set3/mta/dsn.bats index dcbb79b6..a5228cfc 100644 --- a/test/tests/parallel/set3/mta/dsn.bats +++ b/test/tests/parallel/set3/mta/dsn.bats @@ -47,9 +47,11 @@ function teardown_file() { @test "should always send a DSN when requested" { export CONTAINER_NAME=${CONTAINER1_NAME} - _send_email 'email-templates/dsn-unauthenticated' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + # TODO replace with _send_email as soon as it supports DSN + # TODO ref: https://github.com/jetmore/swaks/issues/41 + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -60,7 +62,7 @@ function teardown_file() { @test "should only send a DSN when requested from ports 465/587" { export CONTAINER_NAME=${CONTAINER2_NAME} - _send_email 'email-templates/dsn-unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' _wait_for_empty_mail_queue_in_container # DSN requests can now only be made on ports 465 and 587, @@ -72,8 +74,8 @@ function teardown_file() { assert_failure # These ports are excluded via master.cf. - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -83,9 +85,9 @@ function teardown_file() { @test "should never send a DSN" { export CONTAINER_NAME=${CONTAINER3_NAME} - _send_email 'email-templates/dsn-unauthenticated' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container # DSN requests are rejected regardless of origin. diff --git a/test/tests/parallel/set3/mta/lmtp_ip.bats b/test/tests/parallel/set3/mta/lmtp_ip.bats index 8d35c062..d8be42d9 100644 --- a/test/tests/parallel/set3/mta/lmtp_ip.bats +++ b/test/tests/parallel/set3/mta/lmtp_ip.bats @@ -38,7 +38,7 @@ function teardown_file() { _default_teardown ; } @test "delivers mail to existing account" { _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' # send a test email + _send_email --data 'existing/user1' # send a test email # Verify delivery was successful, log line should look similar to: # postfix/lmtp[1274]: 0EA424ABE7D9: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.13, delays=0.07/0.01/0.01/0.05, dsn=2.0.0, status=sent (250 2.0.0 ixPpB+Zvv2P7BAAAUi6ngw Saved) diff --git a/test/tests/parallel/set3/mta/privacy.bats b/test/tests/parallel/set3/mta/privacy.bats index f8160827..4d4d82ba 100644 --- a/test/tests/parallel/set3/mta/privacy.bats +++ b/test/tests/parallel/set3/mta/privacy.bats @@ -25,7 +25,11 @@ function teardown_file() { _default_teardown ; } # this test covers https://github.com/docker-mailserver/docker-mailserver/issues/681 @test "(Postfix) remove privacy details of the sender" { - _run_in_container_bash "openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/email-templates/send-privacy-email.txt" + _send_email \ + --port 587 -tls --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --data 'privacy' assert_success _run_until_success_or_timeout 120 _exec_in_container_bash '[[ -d /var/mail/localhost.localdomain/user1/new ]]' diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index af98b2f4..169f374a 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -63,34 +63,55 @@ function setup_file() { # TODO: Move to clamav tests (For use when ClamAV is enabled): # _repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /var/run/clamav/clamd.ctl - # _send_email 'email-templates/amavis-virus' + # _send_email --from 'virus@external.tld' --data 'amavis/virus' # Required for 'delivers mail to existing alias': - _send_email 'email-templates/existing-alias-external' + _send_email --to alias1@localhost.localdomain --data 'existing/alias-external' # Required for 'delivers mail to existing alias with recipient delimiter': - _send_email 'email-templates/existing-alias-recipient-delimiter' + _send_email --to alias1~test@localhost.localdomain --data 'existing/alias-recipient-delimiter' # Required for 'delivers mail to existing catchall': - _send_email 'email-templates/existing-catchall-local' + _send_email --to wildcard@localdomain2.com --data 'existing/catchall-local' # Required for 'delivers mail to regexp alias': - _send_email 'email-templates/existing-regexp-alias-local' + _send_email --to test123@localhost.localdomain --data 'existing/regexp-alias-local' # Required for 'rejects mail to unknown user': - _send_email 'email-templates/non-existing-user' + _send_email --to nouser@localhost.localdomain --data 'non-existing-user' # Required for 'redirects mail to external aliases': - _send_email 'email-templates/existing-regexp-alias-external' - _send_email 'email-templates/existing-alias-local' + _send_email --to bounce-always@localhost.localdomain --data 'existing/regexp-alias-external' + _send_email --to alias2@localhost.localdomain --data 'existing/alias-local' # Required for 'rejects spam': - _send_email 'email-templates/amavis-spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam' # Required for 'delivers mail to existing account': - _send_email 'email-templates/existing-user1' - _send_email 'email-templates/existing-user2' - _send_email 'email-templates/existing-user3' - _send_email 'email-templates/existing-added' - _send_email 'email-templates/existing-user-and-cc-local-alias' - _send_email 'email-templates/sieve-spam-folder' - _send_email 'email-templates/sieve-pipe' - _run_in_container_bash 'sendmail root < /tmp/docker-mailserver-test/email-templates/root-email.txt' + _send_email --data 'existing/user1' + assert_success + _send_email --to user2@otherdomain.tld + assert_success + _send_email --to user3@localhost.localdomain + assert_success + _send_email --to added@localhost.localdomain --data 'existing/added' + assert_success + _send_email --to user1@localhost.localdomain --data 'existing/user-and-cc-local-alias' + assert_success + _send_email --data 'sieve/spam-folder' + assert_success + _send_email --to user2@otherdomain.tld --data 'sieve/pipe' + assert_success + _run_in_container_bash 'sendmail root < /tmp/docker-mailserver-test/emails/sendmail/root-email.txt' + assert_success +} + +function _unsuccessful() { + _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword + assert_failure + assert_output --partial 'authentication failed' + assert_output --partial 'No authentication type succeeded' +} + +function _successful() { + _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password mypassword --quit-after AUTH + assert_success + assert_output --partial 'Authentication successful' } @test "should succeed at emptying mail queue" { @@ -103,44 +124,35 @@ function setup_file() { } @test "should successfully authenticate with good password (plain)" { - _send_email 'auth/smtp-auth-plain' '-w 5 0.0.0.0 465' - assert_output --partial 'Authentication successful' + _successful PLAIN user1@localhost.localdomain } @test "should fail to authenticate with wrong password (plain)" { - _send_email 'auth/smtp-auth-plain-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful PLAIN user1@localhost.localdomain } @test "should successfully authenticate with good password (login)" { - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 465' - assert_output --partial 'Authentication successful' + _successful LOGIN user1@localhost.localdomain } @test "should fail to authenticate with wrong password (login)" { - _send_email 'auth/smtp-auth-login-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful LOGIN user1@localhost.localdomain } @test "[user: 'added'] should successfully authenticate with good password (plain)" { - _send_email 'auth/added-smtp-auth-plain' '-w 5 0.0.0.0 465' - assert_output --partial 'Authentication successful' + _successful PLAIN added@localhost.localdomain } @test "[user: 'added'] should fail to authenticate with wrong password (plain)" { - _send_email 'auth/added-smtp-auth-plain-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful PLAIN added@localhost.localdomain } @test "[user: 'added'] should successfully authenticate with good password (login)" { - _send_email 'auth/added-smtp-auth-login' '-w 5 0.0.0.0 465' - assert_success - assert_output --partial 'Authentication successful' + _successful LOGIN added@localhost.localdomain } @test "[user: 'added'] should fail to authenticate with wrong password (login)" { - _send_email 'auth/added-smtp-auth-login-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful LOGIN added@localhost.localdomain } # TODO: Add a test covering case SPAMASSASSIN_SPAM_TO_INBOX=1 (default) @@ -258,7 +270,13 @@ function setup_file() { # Dovecot does not support SMTPUTF8, so while we can send we cannot receive # Better disable SMTPUTF8 support entirely if we can't handle it correctly @test "not advertising smtputf8" { - _send_email 'email-templates/smtp-ehlo' + # Query supported extensions; SMTPUTF8 should not be available. + # - This query requires a EHLO greeting to the destination server. + _send_email \ + --ehlo mail.external.tld \ + --protocol ESMTP \ + --server mail.example.test \ + --quit-after FIRST-EHLO refute_output --partial 'SMTPUTF8' } diff --git a/test/tests/parallel/set3/mta/smtponly.bats b/test/tests/parallel/set3/mta/smtponly.bats index 66123de6..7b1f8699 100644 --- a/test/tests/parallel/set3/mta/smtponly.bats +++ b/test/tests/parallel/set3/mta/smtponly.bats @@ -32,7 +32,16 @@ function teardown_file() { _default_teardown ; } assert_success # it looks as if someone tries to send mail to another domain outside of DMS - _send_email 'email-templates/smtp-only' + _send_email \ + --ehlo mail.origin.test \ + --protocol SSMTPA \ + --server mail.origin.test \ + --from user@origin.test \ + --to user@destination.test \ + --auth PLAIN \ + --auth-user user@origin.test \ + --auth-password secret + assert_success _wait_for_empty_mail_queue_in_container # this seemingly succeeds, but looking at the logs, it doesn't diff --git a/test/tests/serial/mail_pop3.bats b/test/tests/serial/mail_pop3.bats index cb07484a..008921e4 100644 --- a/test/tests/serial/mail_pop3.bats +++ b/test/tests/serial/mail_pop3.bats @@ -24,11 +24,13 @@ function teardown_file() { _default_teardown ; } } @test 'authentication works' { - _send_email 'auth/pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/pop3-auth' '-w 1 0.0.0.0 110' + assert_success } @test 'added user authentication works' { - _send_email 'auth/added-pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/added-pop3-auth' '-w 1 0.0.0.0 110' + assert_success } @test '/var/log/mail/mail.log is error-free' { diff --git a/test/tests/serial/mail_with_imap.bats b/test/tests/serial/mail_with_imap.bats index d729c142..eeccf888 100644 --- a/test/tests/serial/mail_with_imap.bats +++ b/test/tests/serial/mail_with_imap.bats @@ -21,7 +21,8 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test '(Dovecot) LDAP RIMAP connection and authentication works' { - _send_email 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + assert_success } @test '(SASLauthd) SASL RIMAP authentication works' { @@ -30,13 +31,30 @@ function teardown_file() { _default_teardown ; } } @test '(SASLauthd) RIMAP SMTP authentication works' { - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 25' - assert_output --partial 'Error: authentication not enabled' + _send_email \ + --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --quit-after AUTH + assert_failure + assert_output --partial 'Host did not advertise authentication' - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 465' + _send_email \ + --port 465 \ + --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 587' + _send_email \ + --port 587 \ + --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' } diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index b7b3884b..f2011d22 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -122,7 +122,6 @@ function setup_file() { # Extra ENV needed to support specific test-cases: local ENV_SUPPORT=( - --env PERMIT_DOCKER=container # Required for attempting SMTP auth on port 25 via nc # Required for openssl commands to be successul: # NOTE: snakeoil cert is created (for `docker-mailserver.invalid`) via Debian post-install script for Postfix package. # TODO: Use proper TLS cert @@ -249,7 +248,7 @@ function teardown() { # dovecot @test "dovecot: ldap imap connection and authentication works" { - _run_in_container_bash 'nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-ldap-auth.txt' + _nc_wrapper 'auth/imap-ldap-auth' '-w 1 0.0.0.0 143' assert_success } @@ -327,12 +326,25 @@ function teardown() { @test "spoofing (with LDAP): rejects sender forging" { _wait_for_smtp_port_in_container_to_respond dms-test_ldap - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed.txt' + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --ehlo mail \ + --from ldap@localhost.localdomain \ + --data 'auth/ldap-smtp-auth-spoofed' assert_output --partial 'Sender address rejected: not owned by user' } @test "spoofing (with LDAP): accepts sending as alias" { - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-alias.txt' + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --ehlo mail \ + --from postmaster@localhost.localdomain \ + --to some.user@localhost.localdomain \ + --data 'auth/ldap-smtp-auth-spoofed-alias' assert_output --partial 'End data with' } @@ -341,19 +353,42 @@ function teardown() { # Template used has invalid AUTH: https://github.com/docker-mailserver/docker-mailserver/pull/3006#discussion_r1073321432 skip 'TODO: This test seems to have been broken from the start (?)' - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt' + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user some.user.email@localhost.localdomain \ + --auth-password secret \ + --ehlo mail \ + --from randomspoofedaddress@localhost.localdomain \ + --to some.user@localhost.localdomain \ + --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception' assert_output --partial 'Sender address rejected: not owned by user' } @test "saslauthd: ldap smtp authentication" { - # Requires ENV `PERMIT_DOCKER=container` - _send_email 'auth/sasl-ldap-smtp-auth' '-w 5 0.0.0.0 25' - assert_output --partial 'Error: authentication not enabled' + _send_email \ + --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password wrongpassword \ + --quit-after AUTH + assert_failure + assert_output --partial 'Host did not advertise authentication' - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + _send_email \ + --port 465 -tlsc \ + --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' - _run_in_container_bash 'openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + _send_email \ + --port 587 -tls \ + --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' } @@ -391,7 +426,7 @@ function _should_successfully_deliver_mail_to() { local SENDER_ADDRESS='user@external.tld' local RECIPIENT_ADDRESS=${1:?Recipient address is required} local MAIL_STORAGE_RECIPIENT=${2:?Recipient storage location is required} - local MAIL_TEMPLATE='/tmp/docker-mailserver-test/email-templates/test-email.txt' + local MAIL_TEMPLATE='/tmp/docker-mailserver-test/emails/test-email.txt' _run_in_container_bash "sendmail -f ${SENDER_ADDRESS} ${RECIPIENT_ADDRESS} < ${MAIL_TEMPLATE}" _wait_for_empty_mail_queue_in_container diff --git a/test/tests/serial/permit_docker.bats b/test/tests/serial/permit_docker.bats index 85f00484..2ebf5e3e 100644 --- a/test/tests/serial/permit_docker.bats +++ b/test/tests/serial/permit_docker.bats @@ -13,7 +13,7 @@ setup_file() { PRIVATE_CONFIG=$(duplicate_config_for_container . mail_smtponly_second_network) docker create --name mail_smtponly_second_network \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e SMTP_ONLY=1 \ -e PERMIT_DOCKER=connected-networks \ -e OVERRIDE_HOSTNAME=mail.my-domain.com \ @@ -26,7 +26,7 @@ setup_file() { PRIVATE_CONFIG=$(duplicate_config_for_container . mail_smtponly_second_network_sender) docker run -d --name mail_smtponly_second_network_sender \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e SMTP_ONLY=1 \ -e PERMIT_DOCKER=connected-networks \ -e OVERRIDE_HOSTNAME=mail.my-domain.com \ @@ -39,7 +39,7 @@ setup_file() { # create another container that enforces authentication even on local connections docker run -d --name mail_smtponly_force_authentication \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e SMTP_ONLY=1 \ -e PERMIT_DOCKER=none \ -e OVERRIDE_HOSTNAME=mail.my-domain.com \ @@ -68,7 +68,7 @@ teardown_file() { _reload_postfix mail_smtponly_second_network # we should be able to send from the other container on the second network! - run docker exec mail_smtponly_second_network_sender /bin/sh -c "nc mail_smtponly_second_network 25 < /tmp/docker-mailserver-test/email-templates/smtp-only.txt" + run docker exec mail_smtponly_second_network_sender /bin/sh -c "nc mail_smtponly_second_network 25 < /tmp/docker-mailserver-test/emails/nc_raw/smtp-only.txt" assert_output --partial "250 2.0.0 Ok: queued as " repeat_in_container_until_success_or_timeout 60 mail_smtponly_second_network /bin/sh -c 'grep -cE "to=.*status\=sent" /var/log/mail/mail.log' } @@ -80,7 +80,7 @@ teardown_file() { _reload_postfix mail_smtponly_force_authentication # the mailserver should require authentication and a protocol error should occur when using TLS - run docker exec mail_smtponly_force_authentication /bin/sh -c "nc localhost 25 < /tmp/docker-mailserver-test/email-templates/smtp-only.txt" + run docker exec mail_smtponly_force_authentication /bin/sh -c "nc localhost 25 < /tmp/docker-mailserver-test/emails/nc_raw/smtp-only.txt" assert_output --partial "550 5.5.1 Protocol error" [[ ${status} -ge 0 ]] } diff --git a/test/tests/serial/test_helper.bats b/test/tests/serial/test_helper.bats index ecca3d85..a3ffa6cf 100644 --- a/test/tests/serial/test_helper.bats +++ b/test/tests/serial/test_helper.bats @@ -171,7 +171,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' # enable ClamAV to make message delivery slower, so we can detect it CONTAINER_NAME=$(docker run -d --rm \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e ENABLE_CLAMAV=1 \ -h mail.my-domain.com \ -t "${NAME}") @@ -186,7 +186,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' [[ ${SECONDS} -lt 5 ]] # fill the queue with a message - docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt" + docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/emails/amavis-virus.txt" # that should still be stuck in the queue ! TEST_TIMEOUT_IN_SECONDS=0 wait_for_empty_mail_queue_in_container "${CONTAINER_NAME}" @@ -203,7 +203,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' # enable ClamAV to make message delivery slower, so we can detect it CONTAINER_NAME=$(docker run -d --rm \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e ENABLE_CLAMAV=1 \ -h mail.my-domain.com \ -t "${NAME}") @@ -213,7 +213,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' wait_for_smtp_port_in_container "${CONTAINER_NAME}" || docker logs "${CONTAINER_NAME}" # fill the queue with a message - docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt" + docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/emails/amavis-virus.txt" # give it some time to clear the queue SECONDS=0 diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 094454f0..26deb541 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -80,11 +80,13 @@ function teardown_file() { _default_teardown ; } } @test "imap: authentication works" { - _send_email 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + assert_success } @test "imap: added user authentication works" { - _send_email 'auth/added-imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/added-imap-auth' '-w 1 0.0.0.0 143' + assert_success } # @@ -288,13 +290,34 @@ EOF @test "spoofing: rejects sender forging" { # rejection of spoofed sender _wait_for_smtp_port_in_container_to_respond - _run_in_container_bash "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/added-smtp-auth-spoofed.txt" + + # An authenticated user cannot use an envelope sender (MAIL FROM) + # address they do not own according to `main.cf:smtpd_sender_login_maps` lookup + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user added@localhost.localdomain \ + --auth-password mypassword \ + --ehlo mail \ + --from user2@localhost.localdomain \ + --data 'auth/added-smtp-auth-spoofed' assert_output --partial 'Sender address rejected: not owned by user' } @test "spoofing: accepts sending as alias" { - _run_in_container_bash "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/added-smtp-auth-spoofed-alias.txt | grep 'End data with'" + # An authenticated account should be able to send mail from an alias, + # Verifies `main.cf:smtpd_sender_login_maps` includes /etc/postfix/virtual + # The envelope sender address (MAIL FROM) is the lookup key + # to each table. Address is authorized when a result that maps to + # the DMS account is returned. + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --ehlo mail \ + --from alias1@localhost.localdomain \ + --data 'auth/added-smtp-auth-spoofed-alias' assert_success + assert_output --partial 'End data with' } # diff --git a/test/tests/serial/vmail-id.bats b/test/tests/serial/vmail-id.bats index b44670b2..0f54ea96 100644 --- a/test/tests/serial/vmail-id.bats +++ b/test/tests/serial/vmail-id.bats @@ -20,7 +20,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'should successfully deliver mail' { - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' _wait_for_empty_mail_queue_in_container # Should be successfully sent (received) by Postfix: From 25c7024cc4c7a6ee81be70144f6ecaf4fddf44ca Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 3 Jan 2024 02:02:59 +0100 Subject: [PATCH 100/125] security(Postfix): Protect against "SMTP Smuggling" attack (#3727) View `CHANGELOG.md` entry and PR for details. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 15 ++++++++++++++- target/postfix/main.cf | 6 ++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeeb843d..b6e6d906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.0.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.1.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Security + +DMS is now secured against the [recently published spoofing attack "SMTP Smuggling"](https://www.postfix.org/smtp-smuggling.html) that affected Postfix ([#3727](https://github.com/docker-mailserver/docker-mailserver/pull/3727)): +- Postfix upgraded from `3.5.18` to `3.5.23` which provides the [long-term fix with `smtpd_forbid_bare_newline = yes`](https://www.postfix.org/smtp-smuggling.html#long) +- If you are unable to upgrade to this release of DMS, you may follow [these instructions](https://github.com/docker-mailserver/docker-mailserver/issues/3719#issuecomment-1870865118) for applying the [short-term workaround](https://www.postfix.org/smtp-smuggling.html#short). +- This change should not cause compatibility concerns for legitimate mail clients, however if you use software like `netcat` to send mail to DMS (_like our test-suite previously did_) it may now be rejected (_especially with the the short-term workaround `smtpd_data_restrictions = reject_unauth_pipelining`_). +- **NOTE:** This Postfix update also includes the new parameter [`smtpd_forbid_bare_newline_exclusions`](https://www.postfix.org/postconf.5.html#smtpd_forbid_bare_newline_exclusions) which defaults to `$mynetworks` for excluding trusted mail clients excluded from the restriction. + - With our default `PERMIT_DOCKER=none` this is not a concern. + - Presently the Docker daemon config has `user-proxy: true` enabled by default. + - On a host that can be reached by IPv6, this will route to a DMS IPv4 only container implicitly through the Docker network bridge gateway which rewrites the source address. + - If your `PERMIT_DOCKER` setting allows that gateway IP, then it is part of `$mynetworks` and this attack would not be prevented from such connections. + - If this affects your deployment, refer to [our IPv6 docs](https://docker-mailserver.github.io/docker-mailserver/v13.2/config/advanced/ipv6/) for advice on handling IPv6 correctly in Docker. Alternatively [use our `postfix-main.cf`](https://docker-mailserver.github.io/docker-mailserver/v13.2/config/advanced/override-defaults/postfix/) to set `smtpd_forbid_bare_newline_exclusions=` as empty. + ### Updates - The test suite now uses `swaks` instead of `nc`, which has multiple benefits ([#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732)): diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 8c329c94..a9230347 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -57,6 +57,12 @@ smtpd_sender_restrictions = $dms_smtpd_sender_restrictions smtpd_discard_ehlo_keywords = silent-discard, dsn disable_vrfy_command = yes +# Security - Prevent SMTP Smuggling attack +# https://www.postfix.org/smtp-smuggling.html#long +smtpd_forbid_bare_newline = yes +# It is possible to exclude clients on trusted networks from this restriction (the upstream default is `$mynetwork`): +# smtpd_forbid_bare_newline_exclusions = $mynetworks + # Custom defined parameters for DMS: dms_smtpd_sender_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unknown_sender_domain # Submission ports 587 and 465 support for SPOOF_PROTECTION=1 From bf69ef248ead4bc4e9304383736f52e35183b48b Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 4 Jan 2024 22:13:13 +0100 Subject: [PATCH 101/125] Postfix: add `smtpd_data_restrictions = reject_unauth_pipelining` (#3744) * add `smtpd_data_restrictions = reject_unauth_pipelining` * fix: Skip restriction if trusted * add changelog entry * revert change to `postfix-amavis.cf` * Update CHANGELOG.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ target/postfix/main.cf | 5 +++-- test/tests/parallel/set1/spam_virus/postgrey_enabled.bats | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e6d906..0c490092 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ DMS is now secured against the [recently published spoofing attack "SMTP Smuggli - `swaks` handles pipelining correctly, hence we can now use `reject_unauth_pipelining` in Postfix's configuration. - `swaks` provides better CLI options that make many files superflous. - `swaks` can also replace `openssl s_client` and handles authentication on submission ports better. +- **Postfix:** + - We now defer rejection from unauthorized pipelining until the SMTP `DATA` command via `smtpd_data_restrictions` (_i.e. at the end of the mail transfer transaction_) ([#3744](https://github.com/docker-mailserver/docker-mailserver/pull/3744)) + - Prevously our configuration only handled this during the client and recipient restriction stages. Postfix will flag this activity when encountered, but the rejection now is handled at `DATA` where unauthorized pipelining would have been valid from this point. + - If you had the Amavis service enabled (default), this restriction was already in place. Otherwise the concerns expressed with `smtpd_data_restrictions = reject_unauth_pipelining` from the security section above apply. We have permitted trusted clients (_`$mynetworks` or authenticated_) to bypass this restriction. ## [v13.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.1.0) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index a9230347..1dc7bdbc 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -51,10 +51,11 @@ smtpd_helo_required = yes smtpd_delay_reject = yes smtpd_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, permit smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, reject_unauth_pipelining, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain -smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination, reject_unauth_pipelining +smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain +smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination smtpd_sender_restrictions = $dms_smtpd_sender_restrictions smtpd_discard_ehlo_keywords = silent-discard, dsn +smtpd_data_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_pipelining disable_vrfy_command = yes # Security - Prevent SMTP Smuggling attack diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 316e3350..389fc183 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -44,7 +44,7 @@ function teardown_file() { _default_teardown ; } # The other spam checks in `main.cf:smtpd_recipient_restrictions` would interfere with testing postgrey. _run_in_container sed -i \ -e 's/permit_sasl_authenticated.*policyd-spf,$//g' \ - -e 's/reject_unauth_pipelining.*reject_unknown_recipient_domain,$//g' \ + -e 's/reject_invalid_helo_hostname.*reject_unknown_recipient_domain,$//g' \ -e 's/reject_rbl_client.*inet:127\.0\.0\.1:10023$//g' \ -e 's/smtpd_recipient_restrictions =/smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:10023/g' \ /etc/postfix/main.cf From 8f391e4d5aa6449fcf8a4762df23cba98c52bc4c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 4 Jan 2024 22:14:51 +0100 Subject: [PATCH 102/125] release: v13.2.0 (#3746) * bump `VERSION` & adjust `CHANGELOG.md` * chore: Bump the unreleased base compare tag in `CHANGELOG.md` --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 5 ++++- VERSION | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c490092..842b60c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.1.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.2.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) + ### Security DMS is now secured against the [recently published spoofing attack "SMTP Smuggling"](https://www.postfix.org/smtp-smuggling.html) that affected Postfix ([#3727](https://github.com/docker-mailserver/docker-mailserver/pull/3727)): + - Postfix upgraded from `3.5.18` to `3.5.23` which provides the [long-term fix with `smtpd_forbid_bare_newline = yes`](https://www.postfix.org/smtp-smuggling.html#long) - If you are unable to upgrade to this release of DMS, you may follow [these instructions](https://github.com/docker-mailserver/docker-mailserver/issues/3719#issuecomment-1870865118) for applying the [short-term workaround](https://www.postfix.org/smtp-smuggling.html#short). - This change should not cause compatibility concerns for legitimate mail clients, however if you use software like `netcat` to send mail to DMS (_like our test-suite previously did_) it may now be rejected (_especially with the the short-term workaround `smtpd_data_restrictions = reject_unauth_pipelining`_). diff --git a/VERSION b/VERSION index e6ba3513..67aee239 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.1.0 +13.2.0 From 04f4ae4569a2b5dcaffb510cd09ef1fb6ba038a2 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 5 Jan 2024 09:07:31 +0100 Subject: [PATCH 103/125] Rspamd: add custom symbol scores for SPF, DKIM & DMARC (#3726) --- CHANGELOG.md | 5 + Dockerfile | 1 + target/rspamd/local.d/actions.conf | 11 ++- target/rspamd/scores.d/policies_group.conf | 108 +++++++++++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 target/rspamd/scores.d/policies_group.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index 842b60c1..bccbe0e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Updates + +- **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): + - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) + ## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) ### Security diff --git a/Dockerfile b/Dockerfile index 4d0e3568..f9802c2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,6 +106,7 @@ EOF # ----------------------------------------------- COPY target/rspamd/local.d/ /etc/rspamd/local.d/ +COPY target/rspamd/scores.d/* /etc/rspamd/scores.d/ # ----------------------------------------------- # --- LDAP & SpamAssassin's Cron ---------------- diff --git a/target/rspamd/local.d/actions.conf b/target/rspamd/local.d/actions.conf index b214c339..fb4c15b9 100644 --- a/target/rspamd/local.d/actions.conf +++ b/target/rspamd/local.d/actions.conf @@ -1,9 +1,12 @@ # documentation: https://rspamd.com/doc/configuration/metrics.html#actions # and https://rspamd.com/doc/configuration/metrics.html -#greylist = 4; -#add_header = 6; -#rewrite_subject = 7; -#reject = 15; +# These values work in conjunction with the symbol scores in +# `scores.d/*.conf`. When adjusting them, make sure to understand +# and to be able to explain the impact on the whole system. +greylist = 4; +add_header = 6; +rewrite_subject = 7; +reject = 11; subject = "***SPAM*** %s" diff --git a/target/rspamd/scores.d/policies_group.conf b/target/rspamd/scores.d/policies_group.conf new file mode 100644 index 00000000..5f9426e9 --- /dev/null +++ b/target/rspamd/scores.d/policies_group.conf @@ -0,0 +1,108 @@ +# Please refer to +# https://github.com/docker-mailserver/docker-mailserver/issues/3690 +# for understanding this file and its scores' values. + +symbols = { + # SPF + "R_SPF_ALLOW" { + weight = -1; + description = "SPF verification allows sending"; + groups = ["spf"]; + } + "R_SPF_NA" { + weight = 1.5; + description = "Missing SPF record"; + one_shot = true; + groups = ["spf"]; + } + "R_SPF_SOFTFAIL" { + weight = 2.5; + description = "SPF verification soft-failed"; + groups = ["spf"]; + } + "R_SPF_FAIL" { + weight = 4.5; + description = "SPF verification failed"; + groups = ["spf"]; + } + + "R_SPF_NEUTRAL" { # == R_SPF_NA + weight = 1.5; + description = "SPF policy is neutral"; + groups = ["spf"]; + } + "R_SPF_DNSFAIL" { # == R_SPF_SOFTFAIL + weight = 2.5; + description = "SPF DNS failure"; + groups = ["spf"]; + } + "R_SPF_PERMFAIL" { # == R_SPF_FAIL + weight = 4.5; + description = "SPF record is malformed or persistent DNS error"; + groups = ["spf"]; + } + + # DKIM + "R_DKIM_ALLOW" { + weight = -1; + description = "DKIM verification succeed"; + one_shot = true; + groups = ["dkim"]; + } + "R_DKIM_NA" { + weight = 0; + description = "Missing DKIM signature"; + one_shot = true; + groups = ["dkim"]; + } + "R_DKIM_TEMPFAIL" { + weight = 1.5; + description = "DKIM verification soft-failed"; + groups = ["dkim"]; + } + "R_DKIM_PERMFAIL" { + weight = 4.5; + description = "DKIM verification hard-failed (invalid)"; + groups = ["dkim"]; + } + + "R_DKIM_REJECT" { # == R_DKIM_PERMFAIL + weight = 4.5; + description = "DKIM verification failed"; + one_shot = true; + groups = ["dkim"]; + } + + # DMARC + "DMARC_NA" { + weight = 1; + description = "No DMARC record"; + groups = ["dmarc"]; + } + "DMARC_POLICY_QUARANTINE" { + weight = 1.5; + description = "DMARC quarantine policy"; + groups = ["dmarc"]; + } + "DMARC_POLICY_REJECT" { + weight = 2; + description = "DMARC reject policy"; + groups = ["dmarc"]; + } + + "DMARC_POLICY_ALLOW" { # no equivalent + weight = -1; + description = "DMARC permit policy"; + groups = ["dmarc"]; + } + "DMARC_POLICY_ALLOW_WITH_FAILURES" { # no equivalent + weight = -0.5; + description = "DMARC permit policy with DKIM/SPF failure"; + groups = ["dmarc"]; + } + "DMARC_POLICY_SOFTFAIL" { # == DMARC_POLICY_QUARANTINE + weight = 1.5; + description = "DMARC soft-failed"; + groups = ["dmarc"]; + } +} From 6082d5f8d04c353dc1f9de723f3acbdfb4a8fa32 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 6 Jan 2024 11:18:30 +1300 Subject: [PATCH 104/125] chore: Disable `smtputf8` support in config directly (#3750) * chore: Disable `smtputf8` support in config This was always configured disabled at runtime, better to just set explicitly in `main.cf` unless config diverges when Dovecot is enabled to opt-out of this feature. --- CHANGELOG.md | 2 ++ target/postfix/main.cf | 3 +++ target/scripts/startup/setup.d/postfix.sh | 3 --- test/tests/parallel/set3/mta/smtp_delivery.bats | 4 ++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bccbe0e9..5f229530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ All notable changes to this project will be documented in this file. The format ### Updates +- **Internal:** + - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 1dc7bdbc..495ad8a9 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -5,6 +5,9 @@ biff = no append_dot_mydomain = no readme_directory = no +# Disabled as not compatible with Dovecot +smtputf8_enable = no + # Basic configuration # myhostname = alias_maps = hash:/etc/aliases diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 58998376..5aec8636 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -19,9 +19,6 @@ function _setup_postfix_early() { postconf "inet_protocols = ${POSTFIX_INET_PROTOCOLS}" fi - __postfix__log 'trace' "Disabling SMTPUTF8 support" - postconf 'smtputf8_enable = no' - __postfix__log 'trace' "Configuring SASLauthd" if [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && [[ ! -f /etc/postfix/sasl/smtpd.conf ]]; then cat >/etc/postfix/sasl/smtpd.conf << EOF diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index 169f374a..f87f11ed 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -277,6 +277,10 @@ function _successful() { --protocol ESMTP \ --server mail.example.test \ --quit-after FIRST-EHLO + + # Ensure the output is actually related to what we want to refute against: + assert_output --partial 'EHLO mail.external.tld' + assert_output --partial '221 2.0.0 Bye' refute_output --partial 'SMTPUTF8' } From 0eb4ac771498e1eff91944e79c19c8605995bc07 Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 6 Jan 2024 17:07:00 +0100 Subject: [PATCH 105/125] tests: Replace `wc -l` with `grep -c` (#3752) --- CHANGELOG.md | 1 + .../parallel/set1/spam_virus/clamav.bats | 2 +- .../parallel/set3/scripts/setup_cli.bats | 20 +++++++++---------- test/tests/serial/tests.bats | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f229530..d8f8a0d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Internal:** + - tests: Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index 9232f90f..9c035f5b 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -33,7 +33,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'log files exist at /var/log/mail directory' { - _run_in_container_bash "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log' | wc -l" + _run_in_container_bash "ls -1 /var/log/mail/ | grep -c -E 'clamav|freshclam|mail.log'" assert_success assert_output 3 } diff --git a/test/tests/parallel/set3/scripts/setup_cli.bats b/test/tests/parallel/set3/scripts/setup_cli.bats index dca61358..76f16dce 100644 --- a/test/tests/parallel/set3/scripts/setup_cli.bats +++ b/test/tests/parallel/set3/scripts/setup_cli.bats @@ -204,12 +204,12 @@ function teardown_file() { _default_teardown ; } run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user2 51M assert_failure - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1" assert_success run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user@example.com 26M assert_success - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:26M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:26M\$' | grep 1" assert_success run grep "quota_user2@example.com" "${TEST_TMP_CONFIG}/dovecot-quotas.cf" @@ -220,12 +220,12 @@ function teardown_file() { _default_teardown ; } @test "delquota" { run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user@example.com 12M assert_success - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1" assert_success run ./setup.sh -c "${CONTAINER_NAME}" quota del unknown@domain.com assert_failure - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1" assert_success run ./setup.sh -c "${CONTAINER_NAME}" quota del quota_user@example.com @@ -260,13 +260,13 @@ function teardown_file() { _default_teardown ; } ./setup.sh -c "${CONTAINER_NAME}" relay add-domain example3.org smtp.relay.com 587 # check adding - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example1.org\s\+\[smtp.relay1.com\]:2525' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example1.org\s\+\[smtp.relay1.com\]:2525' | grep 1" assert_success # test default port - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example2.org\s\+\[smtp.relay2.com\]:25' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example2.org\s\+\[smtp.relay2.com\]:25' | grep 1" assert_success # test modifying - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example3.org\s\+\[smtp.relay.com\]:587' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example3.org\s\+\[smtp.relay.com\]:587' | grep 1" assert_success } @@ -276,16 +276,16 @@ function teardown_file() { _default_teardown ; } ./setup.sh -c "${CONTAINER_NAME}" relay add-auth example2.org smtp_user2 smtp_pass_new # test adding - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -e '^@example.org\s\+smtp_user:smtp_pass' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -c -e '^@example.org\s\+smtp_user:smtp_pass' | grep 1" assert_success # test updating - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -e '^@example2.org\s\+smtp_user2:smtp_pass_new' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -c -e '^@example2.org\s\+smtp_user2:smtp_pass_new' | grep 1" assert_success } @test "relay exclude-domain" { ./setup.sh -c "${CONTAINER_NAME}" relay exclude-domain example.org - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example.org\s*$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example.org\s*$' | grep 1" assert_success } diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 26deb541..752e325e 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -264,7 +264,7 @@ EOF # @test "amavis: config overrides" { - _run_in_container_bash "grep 'Test Verification' /etc/amavis/conf.d/50-user | wc -l" + _run_in_container_bash "grep -c 'Test Verification' /etc/amavis/conf.d/50-user" assert_success assert_output 1 } From 6d666512c1c2f8307b5d851d066a38eeb367d306 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 8 Jan 2024 09:34:24 +1300 Subject: [PATCH 106/125] ci: `.gitattributes` - Ensure `eol=lf` for shell scripts (#3755) * ci: `.gitattributes` - Ensure `eol=lf` for shell scripts - These files should always use LF for line endings during a checkout. - `Dockerfile` does not like building with HereDoc `RUN` scripts that expect LF. --- .gitattributes | 7 +++---- CHANGELOG.md | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitattributes b/.gitattributes index 869c153e..da5dadda 100644 --- a/.gitattributes +++ b/.gitattributes @@ -20,9 +20,8 @@ ## BUILD: .dockerignore text -Dockerfile text +Dockerfile text eol=lf Makefile -VERSION ## EXAMPLE (RUNTIME): *.env text @@ -75,8 +74,8 @@ target/postsrsd/** text ################################################# ## BATS -*.bash text -*.bats text +*.bash text eol=lf +*.bats text eol=lf ## CONFIG (test/config/) ### OpenLDAP image diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f8a0d0..39a0ab66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ All notable changes to this project will be documented in this file. The format - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) +### Fixes + +- **Internal:** + - `.gitattributes`: Always use LF line endings on checkout for files with shell script content ([#3755](https://github.com/docker-mailserver/docker-mailserver/pull/3755)) + ## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) ### Security From 8e2017320092a2db2cb56ae7255271774e0470c1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 09:40:01 +1300 Subject: [PATCH 107/125] docs: updated `CONTRIBUTORS.md` (#3757) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 2108 +++++++++++++++++++++++------------------------ 1 file changed, 1054 insertions(+), 1054 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 4103ad5f..05a4e6e4 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -55,13 +55,6 @@ Thanks goes to these wonderful people ✨ tomav - - - erik-wramner -
- erik-wramner - - polarathene @@ -69,6 +62,13 @@ Thanks goes to these wonderful people ✨ polarathene + + + erik-wramner +
+ erik-wramner + + chikamichi @@ -162,19 +162,19 @@ Thanks goes to these wonderful people ✨ crazystick - - - swiesend -
- swiesend - - svenyonson
svenyonson + + + + swiesend +
+ swiesend + @@ -213,10 +213,10 @@ Thanks goes to these wonderful people ✨ - - KyleOndy + + mindrunner
- KyleOndy + mindrunner @@ -228,17 +228,10 @@ Thanks goes to these wonderful people ✨ - - mindrunner + + KyleOndy
- mindrunner - - - - - m-a-v -
- m-a-v + KyleOndy @@ -248,6 +241,13 @@ Thanks goes to these wonderful people ✨ bilak + + + m-a-v +
+ m-a-v + + vortex852456 @@ -278,10 +278,46 @@ Thanks goes to these wonderful people ✨ - - dashohoxha + + moqmar
- dashohoxha + moqmar + + + + + pyy +
+ pyy + + + + + dennis95stumm +
+ dennis95stumm + + + + + arneke +
+ arneke + + + + + + akmet +
+ akmet + + + + + diiigle +
+ diiigle @@ -298,6 +334,13 @@ Thanks goes to these wonderful people ✨ mathuin + + + dashohoxha +
+ dashohoxha + + jamebus @@ -335,46 +378,68 @@ Thanks goes to these wonderful people ✨ - - diiigle + + m-schmoock
- diiigle + m-schmoock - - akmet + + mjung
- akmet + mjung - - arneke + + VanVan
- arneke + VanVan - - dennis95stumm + + andreasgerstmayr
- dennis95stumm + andreasgerstmayr - - moqmar + + davidszp
- moqmar + davidszp - - pyy + + kamuri
- pyy + kamuri + + + + + guardiande +
+ guardiande + + + + + Zehir +
+ Zehir + + + + + + weo +
+ weo @@ -390,8 +455,7 @@ Thanks goes to these wonderful people ✨
Birkenstab - - + BrandonSchmitt @@ -412,7 +476,8 @@ Thanks goes to these wonderful people ✨
citec - + + yajo @@ -420,6 +485,13 @@ Thanks goes to these wonderful people ✨ yajo + + + analogue +
+ analogue + + MakerMatrix @@ -427,6 +499,20 @@ Thanks goes to these wonderful people ✨ MakerMatrix + + + Rubytastic2 +
+ Rubytastic2 + + + + + reneploetz +
+ reneploetz + + jsonn @@ -443,110 +529,139 @@ Thanks goes to these wonderful people ✨ - - reneploetz + + keslerm
- reneploetz + keslerm - - Rubytastic2 + + castorinop
- Rubytastic2 + castorinop - - analogue + + p-fruck
- analogue + p-fruck - - weo + + Rillke
- weo + Rillke - - Zehir + + bobbravo2
- Zehir + bobbravo2 - - guardiande + + r-pufky
- guardiande + r-pufky - - kamuri + + vincentDcmps
- kamuri + vincentDcmps - - davidszp + + andymel123
- davidszp + andymel123 - - andreasgerstmayr + + bigpigeon
- andreasgerstmayr + bigpigeon - - mjung + + engelant
- mjung + engelant - - m-schmoock + + j-marz
- m-schmoock + j-marz - - VanVan + + lokipo
- VanVan + lokipo - - elbracht + + msheakoski
- elbracht + msheakoski - - aminvakil + + GoliathLabs
- aminvakil + GoliathLabs - - andrewlow + + tbutter
- andrewlow + tbutter + + + + + yogo1212 +
+ yogo1212 + + + + + mpanneck +
+ mpanneck + + + + + + willtho89 +
+ willtho89 + + + + + ubenmackin +
+ ubenmackin @@ -557,10 +672,24 @@ Thanks goes to these wonderful people ✨ - - ubenmackin + + andrewlow
- ubenmackin + andrewlow + + + + + aminvakil +
+ aminvakil + + + + + elbracht +
+ elbracht @@ -608,38 +737,38 @@ Thanks goes to these wonderful people ✨ - - fl42 + + nueaf
- fl42 + nueaf - - ipernet + + martinwepner
- ipernet + martinwepner - - H4R0 + + artonge
- H4R0 + artonge - - eltociear + + spacecowboy
- eltociear + spacecowboy - - jamesfryer + + jedateach
- jamesfryer + jedateach @@ -651,891 +780,38 @@ Thanks goes to these wonderful people ✨ - - jedateach + + fl42
- jedateach + fl42 - - spacecowboy + + H4R0
- spacecowboy + H4R0 - - artonge + + ipernet
- artonge + ipernet - - martinwepner + + jamesfryer
- martinwepner + jamesfryer - - nueaf + + eltociear
- nueaf - - - - - keslerm -
- keslerm - - - - - - castorinop -
- castorinop - - - - - p-fruck -
- p-fruck - - - - - Rillke -
- Rillke - - - - - bobbravo2 -
- bobbravo2 - - - - - r-pufky -
- r-pufky - - - - - vincentDcmps -
- vincentDcmps - - - - - - andymel123 -
- andymel123 - - - - - bigpigeon -
- bigpigeon - - - - - engelant -
- engelant - - - - - j-marz -
- j-marz - - - - - lokipo -
- lokipo - - - - - msheakoski -
- msheakoski - - - - - - GoliathLabs -
- GoliathLabs - - - - - yogo1212 -
- yogo1212 - - - - - mpanneck -
- mpanneck - - - - - willtho89 -
- willtho89 - - - - - tbutter -
- tbutter - - - - - 0xflotus -
- 0xflotus - - - - - - ifokeev -
- ifokeev - - - - - 20th -
- 20th - - - - - 2b -
- 2b - - - - - askz -
- askz - - - - - acch -
- acch - - - - - vifino -
- vifino - - - - - - kachkaev -
- kachkaev - - - - - alexanderneu -
- alexanderneu - - - - - ch3sh1r -
- ch3sh1r - - - - - eglia -
- eglia - - - - - groupmsl -
- groupmsl - - - - - green-anger -
- green-anger - - - - - - iRhonin -
- iRhonin - - - - - MrFreezeex -
- MrFreezeex - - - - - arunvc -
- arunvc - - - - - astrocket -
- astrocket - - - - - baxerus -
- baxerus - - - - - spock -
- spock - - - - - - erdos4d -
- erdos4d - - - - - crash7 -
- crash7 - - - - - auchri -
- auchri - - - - - arkanovicz -
- arkanovicz - - - - - CBeerta -
- CBeerta - - - - - damianmoore -
- damianmoore - - - - - - espitall -
- espitall - - - - - dkarski -
- dkarski - - - - - dbellavista -
- dbellavista - - - - - danielvandenberg95 -
- danielvandenberg95 - - - - - mlatorre31 -
- mlatorre31 - - - - - mazzz1y -
- mazzz1y - - - - - - aydodo -
- aydodo - - - - - vedtam -
- vedtam - - - - - edvorg -
- edvorg - - - - - eliroca -
- eliroca - - - - - ekkis -
- ekkis - - - - - ErikEngerd -
- ErikEngerd - - - - - - huncode -
- huncode - - - - - felixn -
- felixn - - - - - flole -
- flole - - - - - froks -
- froks - - - - - fkefer -
- fkefer - - - - - frugan-dev -
- frugan-dev - - - - - - Marsu31 -
- Marsu31 - - - - - glandais -
- glandais - - - - - GiovanH -
- GiovanH - - - - - harryyoud -
- harryyoud - - - - - HeySora -
- HeySora - - - - - sirgantrithon -
- sirgantrithon - - - - - - Influencer -
- Influencer - - - - - jcalfee -
- jcalfee - - - - - mivek -
- mivek - - - - - init-js -
- init-js - - - - - Jeidnx -
- Jeidnx - - - - - JiLleON -
- JiLleON - - - - - - jirislav -
- jirislav - - - - - jmccl -
- jmccl - - - - - jurekbarth -
- jurekbarth - - - - - JOduMonT -
- JOduMonT - - - - - Kaan88 -
- Kaan88 - - - - - akkumar -
- akkumar - - - - - - KCrawley -
- KCrawley - - - - - khuedoan -
- khuedoan - - - - - JustAnother1 -
- JustAnother1 - - - - - LeoWinterDE -
- LeoWinterDE - - - - - linhandev -
- linhandev - - - - - luke- -
- luke- - - - - - - LucidityCrash -
- LucidityCrash - - - - - MadsRC -
- MadsRC - - - - - madmath03 -
- madmath03 - - - - - maxemann96 -
- maxemann96 - - - - - dragetd -
- dragetd - - - - - michaeljensen -
- michaeljensen - - - - - - exhuma -
- exhuma - - - - - milas -
- milas - - - - - mcchots -
- mcchots - - - - - MohammedNoureldin -
- MohammedNoureldin - - - - - mpldr -
- mpldr - - - - - naveensrinivasan -
- naveensrinivasan - - - - - - neuralp -
- neuralp - - - - - radicand -
- radicand - - - - - nilshoell -
- nilshoell - - - - - nknapp -
- nknapp - - - - - pcqnt -
- pcqnt - - - - - OrvilleQ -
- OrvilleQ - - - - - - ovidiucp -
- ovidiucp - - - - - mrPjer -
- mrPjer - - - - - p3dda -
- p3dda - - - - - peter-hartmann -
- peter-hartmann - - - - - piwai -
- piwai - - - - - remoe -
- remoe - - - - - - romansey -
- romansey - - - - - MightySCollins -
- MightySCollins - - - - - 501st-alpha1 -
- 501st-alpha1 - - - - - klamann -
- klamann - - - - - svdb0 -
- svdb0 - - - - - 3ap -
- 3ap - - - - - - shyim -
- shyim - - - - - sjmudd -
- sjmudd - - - - - simonsystem -
- simonsystem - - - - - stephan-devop -
- stephan-devop + eltociear @@ -1544,15 +820,15 @@ Thanks goes to these wonderful people ✨
stigok - + + 5ven
5ven - - + syl20bnr @@ -1587,15 +863,15 @@ Thanks goes to these wonderful people ✨
Thiritin - + + tweibert
tweibert - - + torus @@ -1630,15 +906,15 @@ Thanks goes to these wonderful people ✨
Drakulix - + + vilisas
vilisas - - + 42wim @@ -1667,6 +943,300 @@ Thanks goes to these wonderful people ✨ allddd + + + 0xflotus +
+ 0xflotus + + + + + + nknapp +
+ nknapp + + + + + pcqnt +
+ pcqnt + + + + + OrvilleQ +
+ OrvilleQ + + + + + ovidiucp +
+ ovidiucp + + + + + mrPjer +
+ mrPjer + + + + + p3dda +
+ p3dda + + + + + + peter-hartmann +
+ peter-hartmann + + + + + piwai +
+ piwai + + + + + remoe +
+ remoe + + + + + romansey +
+ romansey + + + + + MightySCollins +
+ MightySCollins + + + + + 501st-alpha1 +
+ 501st-alpha1 + + + + + + klamann +
+ klamann + + + + + svdb0 +
+ svdb0 + + + + + 3ap +
+ 3ap + + + + + shyim +
+ shyim + + + + + sjmudd +
+ sjmudd + + + + + simonsystem +
+ simonsystem + + + + + + stephan-devop +
+ stephan-devop + + + + + millerjason +
+ millerjason + + + + + mplx +
+ mplx + + + + + odinis +
+ odinis + + + + + okamidash +
+ okamidash + + + + + olaf-mandel +
+ olaf-mandel + + + + + + ontheair81 +
+ ontheair81 + + + + + pravynandas +
+ pravynandas + + + + + presocratics +
+ presocratics + + + + + rhyst +
+ rhyst + + + + + rmlhuk +
+ rmlhuk + + + + + rriski +
+ rriski + + + + + + schnippl0r +
+ schnippl0r + + + + + smargold476 +
+ smargold476 + + + + + sportshead +
+ sportshead + + + + + squash +
+ squash + + + + + strarsis +
+ strarsis + + + + + tamueller +
+ tamueller + + + + + + vivacarvajalito +
+ vivacarvajalito + + + + + wligtenberg +
+ wligtenberg + + + + + wolkenschieber +
+ wolkenschieber + + + + + worldworm +
+ worldworm + + arcaine2 @@ -1819,153 +1389,583 @@ Thanks goes to these wonderful people ✨ - - millerjason + + auchri
- millerjason + auchri - - mplx + + arkanovicz
- mplx + arkanovicz - - odinis + + CBeerta
- odinis + CBeerta - - okamidash + + damianmoore
- okamidash + damianmoore - - olaf-mandel + + espitall
- olaf-mandel + espitall - - ontheair81 + + dkarski
- ontheair81 + dkarski - - pravynandas + + dbellavista
- pravynandas + dbellavista - - presocratics + + danielvandenberg95
- presocratics + danielvandenberg95 - - rhyst + + mlatorre31
- rhyst + mlatorre31 - - rmlhuk + + mazzz1y
- rmlhuk + mazzz1y - - rriski + + aydodo
- rriski + aydodo - - schnippl0r + + vedtam
- schnippl0r + vedtam - - smargold476 + + edvorg
- smargold476 + edvorg - - sportshead + + eliroca
- sportshead + eliroca - - squash + + ekkis
- squash + ekkis - - strarsis + + ErikEngerd
- strarsis + ErikEngerd - - tamueller + + huncode
- tamueller + huncode - - vivacarvajalito + + felixn
- vivacarvajalito + felixn - - wligtenberg + + flole
- wligtenberg + flole - - wolkenschieber + + froks
- wolkenschieber + froks - - worldworm + + ifokeev
- worldworm + ifokeev + + + + + 20th +
+ 20th + + + + + 2b +
+ 2b + + + + + + askz +
+ askz + + + + + acch +
+ acch + + + + + vifino +
+ vifino + + + + + kachkaev +
+ kachkaev + + + + + alexanderneu +
+ alexanderneu + + + + + ch3sh1r +
+ ch3sh1r + + + + + + eglia +
+ eglia + + + + + groupmsl +
+ groupmsl + + + + + green-anger +
+ green-anger + + + + + iRhonin +
+ iRhonin + + + + + MrFreezeex +
+ MrFreezeex + + + + + arunvc +
+ arunvc + + + + + + astrocket +
+ astrocket + + + + + baxerus +
+ baxerus + + + + + spock +
+ spock + + + + + erdos4d +
+ erdos4d + + + + + crash7 +
+ crash7 + + + + + fkefer +
+ fkefer + + + + + + khuedoan +
+ khuedoan + + + + + JustAnother1 +
+ JustAnother1 + + + + + LeoWinterDE +
+ LeoWinterDE + + + + + linhandev +
+ linhandev + + + + + luke- +
+ luke- + + + + + LucidityCrash +
+ LucidityCrash + + + + + + MadsRC +
+ MadsRC + + + + + madmath03 +
+ madmath03 + + + + + maxemann96 +
+ maxemann96 + + + + + dragetd +
+ dragetd + + + + + michaeljensen +
+ michaeljensen + + + + + exhuma +
+ exhuma + + + + + + milas +
+ milas + + + + + mcchots +
+ mcchots + + + + + MohammedNoureldin +
+ MohammedNoureldin + + + + + mpldr +
+ mpldr + + + + + naveensrinivasan +
+ naveensrinivasan + + + + + neuralp +
+ neuralp + + + + + + radicand +
+ radicand + + + + + nilshoell +
+ nilshoell + + + + + frugan-dev +
+ frugan-dev + + + + + Marsu31 +
+ Marsu31 + + + + + glandais +
+ glandais + + + + + GiovanH +
+ GiovanH + + + + + + harryyoud +
+ harryyoud + + + + + HeySora +
+ HeySora + + + + + sirgantrithon +
+ sirgantrithon + + + + + Influencer +
+ Influencer + + + + + jcalfee +
+ jcalfee + + + + + mivek +
+ mivek + + + + + + init-js +
+ init-js + + + + + Jeidnx +
+ Jeidnx + + + + + JiLleON +
+ JiLleON + + + + + jirislav +
+ jirislav + + + + + jmccl +
+ jmccl + + + + + jurekbarth +
+ jurekbarth + + + + + + JOduMonT +
+ JOduMonT + + + + + Kaan88 +
+ Kaan88 + + + + + akkumar +
+ akkumar + + + + + KCrawley +
+ KCrawley From 5e28c17cf4c72d6b1ebda35c5af9f411d2652159 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:07:38 +1300 Subject: [PATCH 108/125] docs: SpamAssassin ENV docs refactor (#3756) * chore: Log `SPAMASSASSIN_SPAM_TO_INBOX=1` ENV correctly ENV name logged was incomplete. * docs: Update SA related ENV docs * fix: Log level `warning` should be `warn` * docs: FAQ - Revise outdated SA entry * chore: Antispam / Antivirus => Anti-spam / Anti-virus * docs: ENV - Additional revisions to SA ENV * docs: ENV - Move `ENABLE_SPAMASSASSIN_KAM` --- CHANGELOG.md | 3 + Dockerfile | 2 +- README.md | 2 +- .../config/advanced/optional-config.md | 2 +- docs/content/config/environment.md | 179 ++++++++++++++---- docs/content/faq.md | 70 ++----- docs/content/index.md | 2 +- docs/mkdocs.yml | 2 +- mailserver.env | 12 +- .../scripts/startup/setup.d/security/misc.sh | 6 +- 10 files changed, 181 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39a0ab66..074c3ead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ All notable changes to this project will be documented in this file. The format - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) +- **Docs:** + - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) + ### Fixes diff --git a/Dockerfile b/Dockerfile index f9802c2a..65d818ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -319,7 +319,7 @@ LABEL org.opencontainers.image.title="docker-mailserver" LABEL org.opencontainers.image.vendor="The Docker Mailserver Organization" LABEL org.opencontainers.image.authors="The Docker Mailserver Organization on GitHub" LABEL org.opencontainers.image.licenses="MIT" -LABEL org.opencontainers.image.description="A fullstack but simple mail server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.). Only configuration files, no SQL database." +LABEL org.opencontainers.image.description="A fullstack but simple mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). Only configuration files, no SQL database." LABEL org.opencontainers.image.url="https://github.com/docker-mailserver" LABEL org.opencontainers.image.documentation="https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md" LABEL org.opencontainers.image.source="https://github.com/docker-mailserver/docker-mailserver" diff --git a/README.md b/README.md index b3072cd9..8918c2e9 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## :page_with_curl: About -A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.). Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. Originally created by @tomav, this project is now maintained by volunteers since January 2021. +A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. Originally created by @tomav, this project is now maintained by volunteers since January 2021. ## :bulb: Documentation diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 8a43e4db..21f82a3b 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -33,7 +33,7 @@ This is a list of all configuration files and directories which are optional or - **ldap-aliases.cf:** Configuration for the virtual alias mapping `virtual_alias_maps`. See the [`setup-stack.sh`][github-commit-setup-stack.sh-L411] script. - **ldap-domains.cf:** Configuration for the virtual domain mapping `virtual_mailbox_domains`. See the [`setup-stack.sh`][github-commit-setup-stack.sh-L411] script. - **whitelist_clients.local:** Whitelisted domains, not considered by postgrey. Enter one host or domain per line. -- **spamassassin-rules.cf:** Antispam rules for Spamassassin. (Docs: [FAQ - SpamAssassin Rules][docs-faq-spamrules]) +- **spamassassin-rules.cf:** Anti-spam rules for Spamassassin. (Docs: [FAQ - SpamAssassin Rules][docs-faq-spamrules]) - **fail2ban-fail2ban.cf:** Additional config options for `fail2ban.cf`. (Docs: [Fail2Ban][docs-fail2ban]) - **fail2ban-jail.cf:** Additional config options for fail2ban's jail behaviour. (Docs: [Fail2Ban][docs-fail2ban]) - **amavis.cf:** replaces the `/etc/amavis/conf.d/50-user` file diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 71807c87..c3c074fd 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -316,28 +316,32 @@ Note: More information at ##### MOVE_SPAM_TO_JUNK -When enabled, e-mails marked with the - -1. `X-Spam: Yes` header added by Rspamd -2. `X-Spam-Flag: YES` header added by SpamAssassin (requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)) - -will be automatically moved to the Junk folder (with the help of a Sieve script). - - 0 => Spam messages will be delivered in the mailbox. - **1** => Spam messages will be delivered in the `Junk` folder. +Routes mail identified as spam into the recipient(s) Junk folder (_via a Dovecot Sieve script_). + +!!! info + + Mail is received as spam when it has been marked with either header: + + - `X-Spam: Yes` (_added by Rspamd_) + - `X-Spam-Flag: YES` (_added by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) + ##### MARK_SPAM_AS_READ -Enable to treat received spam as "read" (_avoids notification to MUA client of new mail_). - -Mail is received as spam when it has been marked with either header: - -1. `X-Spam: Yes` (_by Rspamd_) -2. `X-Spam-Flag: YES` (_by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) - - **0** => disabled - 1 => Spam messages will be marked as read +Enable to treat received spam as "read" (_avoids notification to MUA client of new mail_). + +!!! info + + Mail is received as spam when it has been marked with either header: + + - `X-Spam: Yes` (_added by Rspamd_) + - `X-Spam-Flag: YES` (_added by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) + #### Rspamd ##### ENABLE_RSPAMD @@ -515,63 +519,170 @@ Changes the interval in which log files are rotated. - **0** => SpamAssassin is disabled - 1 => SpamAssassin is enabled -##### SPAMASSASSIN_SPAM_TO_INBOX +??? info "SpamAssassin analyzes incoming mail and assigns a spam score" -- 0 => Spam messages will be bounced (_rejected_) without any notification (_dangerous_). -- **1** => Spam messages will be delivered to the inbox and tagged as spam using `SA_SPAM_SUBJECT`. + Integration with Amavis involves processing mail based on the assigned spam score via [`SA_TAG`, `SA_TAG2` and `SA_KILL`][amavis-docs::spam-score]. + + These settings have equivalent ENV supported by DMS for easy adjustments, as documented below. + +[amavis-docs::spam-score]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#tagkill ##### ENABLE_SPAMASSASSIN_KAM -[KAM](https://mcgrail.com/template/projects#KAM1) is a 3rd party SpamAssassin ruleset, provided by the McGrail Foundation. If SpamAssassin is enabled, KAM can be used in addition to the default ruleset. - - **0** => KAM disabled - 1 => KAM enabled +[KAM](https://mcgrail.com/template/projects#KAM1) is a 3rd party SpamAssassin ruleset, provided by the McGrail Foundation. If SpamAssassin is enabled, KAM can be used in addition to the default ruleset. + +##### SPAMASSASSIN_SPAM_TO_INBOX + +- 0 => (_Amavis action: `D_BOUNCE`_): Spam messages will be bounced (_rejected_) without any notification (_dangerous_). +- **1** => (_Amavis action: `D_PASS`_): Spam messages will be delivered to the inbox. + +!!! note + + The Amavis action configured by this setting: + + - Influences the behaviour of the [`SA_KILL`](#sa_kill) setting. + - Applies to the Amavis config parameters `$final_spam_destiny` and `$final_bad_header_destiny`. + +!!! note "This ENV setting is related to" + + - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk) + - [`MARK_SPAM_AS_READ=1`](#mark_spam_as_read) + - [`SA_SPAM_SUBJECT`](#sa_spam_subject) + ##### SA_TAG -- **2.0** => add spam info headers if at, or above that level +- **2.0** => add 'spam info' headers at, or above this spam score -Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1` +Mail is not yet considered spam at this spam score, but for purposes like diagnostics it can be useful to identify mail with a spam score at a lower bound than `SA_TAG2`. + +??? example "`X-Spam` headers appended to mail" + + Send a simple mail to a local DMS account `hello@example.com`: + + ```bash + docker exec dms swaks --server 0.0.0.0 --to hello@example.com --body 'spam' + ``` + + Inspecting the raw mail you will notice several `X-Spam` headers were added to the mail like this: + + ``` + X-Spam-Flag: NO + X-Spam-Score: 4.162 + X-Spam-Level: **** + X-Spam-Status: No, score=4.162 tagged_above=2 required=4 + tests=[BODY_SINGLE_WORD=1, DKIM_ADSP_NXDOMAIN=0.8, + NO_DNS_FOR_FROM=0.379, NO_RECEIVED=-0.001, NO_RELAYS=-0.001, + PYZOR_CHECK=1.985] autolearn=no autolearn_force=no + ``` + + !!! info "The `X-Spam-Score` is `4.162`" + + High enough for `SA_TAG` to trigger adding these headers, but not high enough for `SA_TAG2` (_which would set `X-Spam-Flag: YES` instead_). ##### SA_TAG2 -- **6.31** => add 'spam detected' headers at that level +- **6.31** => add 'spam detected' headers at, or above this level -Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1` +When a spam score is high enough, mark mail as spam (_Appends the mail header: `X-Spam-Flag: YES`_). + +!!! info "Interaction with other ENV" + + - [`SA_SPAM_SUBJECT`](#sa_spam_subject) modifies the mail subject to better communicate spam mail to the user. + - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SA_SPAM_SUBJECT`. ##### SA_KILL -- **10.0** => triggers spam evasive actions +- **10.0** => quarantine + triggers action to handle spam -!!! note "This SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1`" +Controls the spam score threshold for triggering an action on mail that has a high spam score. - By default, DMS is configured to quarantine spam emails. +??? tip "Choosing an appropriate `SA_KILL` value" - If emails are quarantined, they are compressed and stored in a location dependent on the `ONE_DIR` setting above. To inhibit this behaviour and deliver spam emails, set this to a very high value e.g. `100.0`. + The value should be high enough to be represent confidence in mail as spam: - If `ONE_DIR=1` (default) the location is `/var/mail-state/lib-amavis/virusmails/`, or if `ONE_DIR=0`: `/var/lib/amavis/virusmails/`. These paths are inside the docker container. + - Too low: The action taken may prevent legitimate mail (ham) that was incorrectly detected as spam from being delivered successfully. + - Too high: Allows more spam to bypass the `SA_KILL` trigger (_how to treat mail with high confidence that it is actually spam_). + + Experiences from DMS users with these settings has been [collected here][gh-issue::sa-tunables-insights], along with [some direct configuration guides][gh-issue::sa-tunables-guides] (_under "Resources for references"_). + +[gh-issue::sa-tunables-insights]: https://github.com/docker-mailserver/docker-mailserver/pull/3058#issuecomment-1420268148 +[gh-issue::sa-tunables-guides]: https://github.com/docker-mailserver/docker-mailserver/pull/3058#issuecomment-1416547911 + +??? info "Trigger action" + + DMS will configure Amavis with either of these actions based on the DMS [`SPAMASSASSIN_SPAM_TO_INBOX`](#spamassassin_spam_to_inbox) ENV setting: + + - `D_PASS` (**default**): + - Accept mail and deliver it to the recipient(s), despite the high spam score. A copy is still stored in quarantine. + - This is a good default to start with until you are more confident in an `SA_KILL` threshold that won't accidentally discard / bounce legitimate mail users are expecting to arrive but is detected as spam. + - `D_BOUNCE`: + - Additionally sends a bounce notification (DSN). + - The [DSN is suppressed][amavis-docs::actions] (_no bounce sent_) when the spam score exceeds the Amavis `$sa_dsn_cutoff_level` config setting (default: `10`). With the DMS `SA_KILL` default also being `10`, no DSN will ever be sent. + - `D_REJECT` / `D_DISCARD`: + - These two aren't configured by DMS, but are valid alternative action values if configuring Amavis directly. + +??? note "Quarantined mail" + + When mail has a spam score that reaches the `SA_KILL` threshold: + + - [It will be quarantined][amavis-docs::quarantine] regardless of the `SA_KILL` action to perform. + - With `D_PASS` the delivered mail also appends an `X-Quarantine-ID` mail header. The ID value of this header is part of the quarantined file name. + + If emails are quarantined, they are compressed and stored at a location dependent on the [`ONE_DIR`](#one_dir) setting: + + - `ONE_DIR=1` (default): `/var/mail-state/lib-amavis/virusmails/` + - `ONE_DIR=0`: `/var/lib/amavis/virusmails/` + + !!! tip + + Easily list mail stored in quarantine with `find` and the quarantine path: + + ```bash + find /var/lib/amavis/virusmails -type f + ``` + +[amavis-docs::actions]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#actions +[amavis-docs::quarantine]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#quarantine ##### SA_SPAM_SUBJECT -- **\*\*\*SPAM\*\*\*** => add tag to subject if spam detected +Adds a prefix to the subject header when mail is marked as spam (_via [`SA_TAG2`](#sa_tag2)_). -Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1`. Add the SpamAssassin score to the subject line by inserting the keyword \_SCORE\_: **\*\*\*SPAM(\_SCORE\_)\*\*\***. +- **`'***SPAM*** '`** => A string value to use as a mail subject prefix. +- `undef` => Opt-out of modifying the subject for mail marked as spam. + +??? example "Including trailing white-space" + + Add trailing white-space by quote wrapping the value: `SA_SPAM_SUBJECT='[SPAM] '` + +??? example "Including the associated spam score" + + The [`_SCORE_` tag][sa-docs::score-tag] will be substituted with the SpamAssassin score: `SA_SPAM_SUBJECT=***SPAM(_SCORE_)***`. + +[sa-docs::score-tag]: https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING ##### SA_SHORTCIRCUIT_BAYES_SPAM - **1** => will activate SpamAssassin short circuiting for bayes spam detection. -This will uncomment the respective line in ```/etc/spamassasin/local.cf``` +This will uncomment the respective line in `/etc/spamassasin/local.cf` -Note: activate this only if you are confident in your bayes database for identifying spam. +!!! warning + + Activate this only if you are confident in your bayes database for identifying spam. ##### SA_SHORTCIRCUIT_BAYES_HAM - **1** => will activate SpamAssassin short circuiting for bayes ham detection -This will uncomment the respective line in ```/etc/spamassasin/local.cf``` +This will uncomment the respective line in `/etc/spamassasin/local.cf` -Note: activate this only if you are confident in your bayes database for identifying ham. +!!! warning + + Activate this only if you are confident in your bayes database for identifying ham. #### Fetchmail diff --git a/docs/content/faq.md b/docs/content/faq.md index 4da64b60..f666b102 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -378,18 +378,7 @@ When you run DMS with the ENV variable `ONE_DIR=1` (default), this directory wil #### How can I manage my custom SpamAssassin rules? -Antispam rules are managed in `docker-data/dms/config/spamassassin-rules.cf`. - -#### What are acceptable `SA_SPAM_SUBJECT` values? - -For no subject set `SA_SPAM_SUBJECT=undef`. - -For a trailing white-space subject one can define the whole variable with quotes in `compose.yaml`: - -```yaml -environment: - - "SA_SPAM_SUBJECT=[SPAM] " -``` +Anti-spam rules are managed in `docker-data/dms/config/spamassassin-rules.cf`. #### Why are SpamAssassin `x-headers` not inserted into my `subdomain.example.com` subdomain emails? @@ -479,59 +468,39 @@ The following configuration works nicely: file: ./docker-data/dms/cron/sa-learn ``` -With the default settings, SpamAssassin will require 200 mails trained for spam (for example with the method explained above) and 200 mails trained for ham (using the same command as above but using `--ham` and providing it with some ham mails). Until you provided these 200+200 mails, SpamAssassin will not take the learned mails into account. For further reference, see the [SpamAssassin Wiki](https://wiki.apache.org/spamassassin/BayesNotWorking). +With the default settings, SpamAssassin will require 200 mails trained for spam (for example with the method explained above) and 200 mails trained for ham (using the same command as above but using `--ham` and providing it with some ham mails). + +- Until you provided these 200+200 mails, SpamAssassin will not take the learned mails into account. +- For further reference, see the [SpamAssassin Wiki](https://wiki.apache.org/spamassassin/BayesNotWorking). #### How do I have more control about what SpamAssassin is filtering? -By default, SPAM and INFECTED emails are put to a quarantine which is not very straight forward to access. Several config settings are affecting this behavior: +This is related to Amavis processing the mail after SpamAssassin has analyzed it and assigned a spam score. -First, make sure you have the proper thresholds set: +- DMS provides some [common SA tunables via ENV][docs::env::sa_env]. +- Additional configuration can be managed with the DMS config volume by providing `docker-data/dms/config/amavis.cf`. -```conf -SA_TAG=-100000.0 -SA_TAG2=3.75 -SA_KILL=100000.0 -``` +#### How can I send quarantined mail to a mailbox? -- The very negative value in `SA_TAG` makes sure, that all emails have the SpamAssassin headers included. -- `SA_TAG2` is the actual threshold to set the YES/NO flag for spam detection. -- `SA_KILL` needs to be very high, to make sure nothing is bounced at all (`SA_KILL` superseeds `SPAMASSASSIN_SPAM_TO_INBOX`) +SPAM and INFECTED emails that [reach the `SA_KILL` threshold are archived into quarantine][docs::env::sa_kill]. -Make sure everything (including SPAM) is delivered to the inbox and not quarantined: - -```conf -SPAMASSASSIN_SPAM_TO_INBOX=1 -``` - -Use `MOVE_SPAM_TO_JUNK=1` or create a sieve script which puts spam to the Junk folder: - -```sieve -require ["comparator-i;ascii-numeric","relational","fileinto"]; - -if header :contains "X-Spam-Flag" "YES" { - fileinto "Junk"; -} elsif allof ( - not header :matches "x-spam-score" "-*", - header :value "ge" :comparator "i;ascii-numeric" "x-spam-score" "3.75" -) { - fileinto "Junk"; -} -``` - -Create a dedicated mailbox for emails which are infected/bad header and everything amavis is blocking by default and put its address into `docker-data/dms/config/amavis.cf` +Instead of a quarantine folder, you can use a dedicated mailbox instead. Create an account like `quarantine@example.com` and create `docker-data/dms/config/amavis.cf`: ```cf -$clean_quarantine_to = "amavis\@example.com"; -$virus_quarantine_to = "amavis\@example.com"; -$banned_quarantine_to = "amavis\@example.com"; -$bad_header_quarantine_to = "amavis\@example.com"; -$spam_quarantine_to = "amavis\@example.com"; +$clean_quarantine_to = "quarantine\@example.com"; +$virus_quarantine_to = "quarantine\@example.com"; +$banned_quarantine_to = "quarantine\@example.com"; +$bad_header_quarantine_to = "quarantine\@example.com"; +$spam_quarantine_to = "quarantine\@example.com"; ``` [fail2ban-customize]: ./config/security/fail2ban.md [docs-maintenance]: ./config/advanced/maintenance/update-and-cleanup.md [docs-override-postfix]: ./config/advanced/override-defaults/postfix.md [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md +[docs-optional-configuration]: ./config/advanced/optional-config.md +[docs::env::sa_env]: ./config/environment.md#spamassassin +[docs::env::sa_kill]: ./config/environment.md#sa_kill [github-comment-baredomain]: https://github.com/docker-mailserver/docker-mailserver/issues/3048#issuecomment-1432358353 [github-comment-override-hostname]: https://github.com/docker-mailserver/docker-mailserver/issues/1731#issuecomment-753968425 [github-issue-95]: https://github.com/docker-mailserver/docker-mailserver/issues/95 @@ -542,4 +511,3 @@ $spam_quarantine_to = "amavis\@example.com"; [github-issue-1792]: https://github.com/docker-mailserver/docker-mailserver/pull/1792 [hanscees-userpatches]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-user-patches.sh [mail-state-folders]: https://github.com/docker-mailserver/docker-mailserver/blob/c7e498194546416fb7231cb03254e77e085d18df/target/scripts/startup/misc-stack.sh#L24-L33 -[docs-optional-configuration]: ./config/advanced/optional-config.md diff --git a/docs/content/index.md b/docs/content/index.md index 51be4fb8..ff1214b1 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -14,7 +14,7 @@ This documentation provides you not only with the basic setup and configuration ## About -`docker-mailserver`, or DMS for short, is a production-ready fullstack but simple mail server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.). It employs only configuration files, no SQL database. The image is focused around the slogan "Keep it simple and versioned". +`docker-mailserver`, or DMS for short, is a production-ready fullstack but simple mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). It employs only configuration files, no SQL database. The image is focused around the slogan "Keep it simple and versioned". ## Contents diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index bd41a798..6441dffe 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,6 +1,6 @@ # Site specific: site_name: 'Docker Mailserver' -site_description: 'A fullstack but simple mail-server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.) using Docker.' +site_description: 'A fullstack but simple mail-server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.) using Docker.' site_author: 'docker-mailserver (Github Organization)' copyright: '

© Docker Mailserver Organization
This project is licensed under the MIT license.

' diff --git a/mailserver.env b/mailserver.env index e84157a2..0c2e1e40 100644 --- a/mailserver.env +++ b/mailserver.env @@ -368,9 +368,6 @@ DOVECOT_INET_PROTOCOLS=all ENABLE_SPAMASSASSIN=0 -# deliver spam messages in the inbox (eventually tagged using SA_SPAM_SUBJECT) -SPAMASSASSIN_SPAM_TO_INBOX=1 - # KAM is a 3rd party SpamAssassin ruleset, provided by the McGrail Foundation. # If SpamAssassin is enabled, KAM can be used in addition to the default ruleset. # - **0** => KAM disabled @@ -379,23 +376,26 @@ SPAMASSASSIN_SPAM_TO_INBOX=1 # Note: only has an effect if `ENABLE_SPAMASSASSIN=1` ENABLE_SPAMASSASSIN_KAM=0 +# deliver spam messages to the inbox (tagged using SA_SPAM_SUBJECT) +SPAMASSASSIN_SPAM_TO_INBOX=1 + # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required) MOVE_SPAM_TO_JUNK=1 # spam messages wil be marked as read MARK_SPAM_AS_READ=0 -# add spam info headers if at, or above that level: +# add 'spam info' headers at, or above this level SA_TAG=2.0 -# add 'spam detected' headers at that level +# add 'spam detected' headers at, or above this level SA_TAG2=6.31 # triggers spam evasive actions SA_KILL=10.0 # add tag to subject if spam detected -SA_SPAM_SUBJECT=***SPAM***** +SA_SPAM_SUBJECT='***SPAM*** ' # ----------------------------------------------- # --- Fetchmail Section ------------------------- diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 78c1e60a..170f46fb 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -111,7 +111,7 @@ function __setup__security__spamassassin() { if [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 1 ]]; then _log 'trace' 'Configuring Spamassassin/Amavis to send SPAM to inbox' - _log 'debug' 'SPAM_TO_INBOX=1 is set. SA_KILL will be ignored.' + _log 'debug' "'SPAMASSASSIN_SPAM_TO_INBOX=1' is set. The 'SA_KILL' ENV will be ignored." sed -i "s|\$final_spam_destiny.*=.*$|\$final_spam_destiny = D_PASS;|g" /etc/amavis/conf.d/49-docker-mailserver sed -i "s|\$final_bad_header_destiny.*=.*$|\$final_bad_header_destiny = D_PASS;|g" /etc/amavis/conf.d/49-docker-mailserver @@ -265,7 +265,7 @@ EOF chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_to_junk.{sieve,svbin} if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]]; then - _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MOVE_SPAM_TO_JUNK=1' to work" + _log 'warn' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MOVE_SPAM_TO_JUNK=1' to work" fi else _log 'debug' 'Spam emails will not be moved to the Junk folder' @@ -290,7 +290,7 @@ EOF chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_mark_as_read.{sieve,svbin} if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]]; then - _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MARK_SPAM_AS_READ=1' to work" + _log 'warn' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MARK_SPAM_AS_READ=1' to work" fi else _log 'debug' 'Spam emails will not be marked as read' From ac25fb495bce9b21c1b643126aca7b0b26556ea2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:39:28 +0100 Subject: [PATCH 109/125] chore(deps): Bump docker/metadata-action from 5.4.0 to 5.5.0 (#3762) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index b0a32b93..00771221 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.4.0 + uses: docker/metadata-action@v5.5.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 8d8936dfac81d762c03177d83876c69cb321e342 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:40:50 +0000 Subject: [PATCH 110/125] chore(deps): Bump anchore/scan-action from 3.3.8 to 3.4.0 (#3761) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 08b4fd29..d7cf6932 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.8 + uses: anchore/scan-action@v3.4.0 id: scan with: image: mailserver-testing:ci From aba218e6d7f30fbf4717178581c3e04537bea178 Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 10 Jan 2024 00:31:30 +0100 Subject: [PATCH 111/125] Fix jaq: Download platform specific binary (#3766) * choose architecture dynamically --- CHANGELOG.md | 1 + target/scripts/build/packages.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 074c3ead..86302566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ All notable changes to this project will be documented in this file. The format - **Internal:** - `.gitattributes`: Always use LF line endings on checkout for files with shell script content ([#3755](https://github.com/docker-mailserver/docker-mailserver/pull/3755)) + - Fix missing 'jaq' binary for ARM architecture ([#3766](https://github.com/docker-mailserver/docker-mailserver/pull/3766)) ## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 566d5441..7a4b60f6 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -192,7 +192,7 @@ function _install_getmail() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' - curl -sL https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-x86_64-unknown-linux-musl -o /usr/bin/jaq && chmod +x /usr/bin/jaq + curl -sL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq } function _remove_data_after_package_installations() { From 06fab3f12964b9ce5d5d6e889de28598635c6cf5 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 11 Jan 2024 10:34:08 +0100 Subject: [PATCH 112/125] tests: streamline tests and helpers further (#3747) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 7 + target/scripts/build/packages.sh | 10 +- test/files/emails/existing/added.txt | 5 - test/files/emails/existing/alias-external.txt | 5 - test/files/emails/existing/alias-local.txt | 5 - .../existing/alias-recipient-delimiter.txt | 5 - test/files/emails/existing/catchall-local.txt | 5 - .../emails/existing/regexp-alias-external.txt | 5 - .../emails/existing/regexp-alias-local.txt | 5 - .../existing/user-and-cc-local-alias.txt | 6 - test/files/emails/existing/user1.txt | 5 - test/files/emails/non-existing-user.txt | 5 - test/files/emails/rspamd/pass.txt | 5 - test/files/emails/rspamd/spam-header.txt | 5 - test/files/emails/rspamd/spam.txt | 5 - test/files/emails/rspamd/virus.txt | 5 - test/helper/common.bash | 4 +- test/helper/sending.bash | 171 ++++++++++++------ .../parallel/set1/dovecot/dovecot_quotas.bats | 9 +- .../parallel/set1/dovecot/dovecot_sieve.bats | 6 +- .../set1/dovecot/mailbox_format_dbox.bats | 6 +- .../set1/dovecot/special_use_folders.bats | 5 +- .../parallel/set1/spam_virus/clamav.bats | 3 +- .../disabled_clamav_spamassassin.bats | 3 +- .../parallel/set1/spam_virus/fail2ban.bats | 2 +- .../set1/spam_virus/postgrey_enabled.bats | 9 +- .../parallel/set1/spam_virus/postscreen.bats | 14 +- .../parallel/set1/spam_virus/rspamd_full.bats | 38 ++-- .../set1/spam_virus/spam_junk_folder.bats | 2 +- .../container_configuration/hostname.bats | 2 +- test/tests/parallel/set3/mta/dsn.bats | 18 +- test/tests/parallel/set3/mta/lmtp_ip.bats | 2 +- test/tests/parallel/set3/mta/privacy.bats | 5 +- .../parallel/set3/mta/smtp_delivery.bats | 42 ++--- test/tests/serial/mail_pop3.bats | 4 +- test/tests/serial/mail_with_imap.bats | 12 +- test/tests/serial/mail_with_ldap.bats | 27 +-- test/tests/serial/tests.bats | 14 +- test/tests/serial/vmail-id.bats | 4 +- 39 files changed, 243 insertions(+), 247 deletions(-) delete mode 100644 test/files/emails/existing/added.txt delete mode 100644 test/files/emails/existing/alias-external.txt delete mode 100644 test/files/emails/existing/alias-local.txt delete mode 100644 test/files/emails/existing/alias-recipient-delimiter.txt delete mode 100644 test/files/emails/existing/catchall-local.txt delete mode 100644 test/files/emails/existing/regexp-alias-external.txt delete mode 100644 test/files/emails/existing/regexp-alias-local.txt delete mode 100644 test/files/emails/existing/user-and-cc-local-alias.txt delete mode 100644 test/files/emails/existing/user1.txt delete mode 100644 test/files/emails/non-existing-user.txt delete mode 100644 test/files/emails/rspamd/pass.txt delete mode 100644 test/files/emails/rspamd/spam-header.txt delete mode 100644 test/files/emails/rspamd/spam.txt delete mode 100644 test/files/emails/rspamd/virus.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 86302566..382b9efa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ All notable changes to this project will be documented in this file. The format ### Updates +- **Tests**: + - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747)): + - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. + - `swaks` version is now the latest from Github releases instead of the Debian package. + - `_nc_wrapper`, `_send_mail` and related helpers expect the `.txt` filepath extension again. + - `sending.bash` helper methods were refactored to better integrate `swaks` and accommodate different usage contexts. + - `test/files/emails/existing/` files were removed similar to previous removal of SMTP auth files as they became redundant with `swaks`. - **Internal:** - tests: Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 7a4b60f6..e3607f48 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -80,7 +80,7 @@ function _install_packages() { # `bind9-dnsutils` provides the `dig` command # `iputils-ping` provides the `ping` command DEBUG_PACKAGES=( - bind9-dnsutils iputils-ping less nano swaks + bind9-dnsutils iputils-ping less nano ) apt-get "${QUIET}" --no-install-recommends install \ @@ -192,7 +192,15 @@ function _install_getmail() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' + _log 'trace' 'Installing jaq' curl -sL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq + + _log 'trace' 'Installing swaks' + local SWAKS_VERSION='20240103.0' + local SWAKS_RELEASE="swaks-${SWAKS_VERSION}" + curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" | tar -xz + mv "${SWAKS_RELEASE}/swaks" /usr/local/bin + rm -r "${SWAKS_RELEASE}" } function _remove_data_after_package_installations() { diff --git a/test/files/emails/existing/added.txt b/test/files/emails/existing/added.txt deleted file mode 100644 index 827b681f..00000000 --- a/test/files/emails/existing/added.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-added.txt -This is a test mail. diff --git a/test/files/emails/existing/alias-external.txt b/test/files/emails/existing/alias-external.txt deleted file mode 100644 index 03f1af6c..00000000 --- a/test/files/emails/existing/alias-external.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-alias-external.txt -This is a test mail. diff --git a/test/files/emails/existing/alias-local.txt b/test/files/emails/existing/alias-local.txt deleted file mode 100644 index 9b481a98..00000000 --- a/test/files/emails/existing/alias-local.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local Alias -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-alias-local.txt -This is a test mail. diff --git a/test/files/emails/existing/alias-recipient-delimiter.txt b/test/files/emails/existing/alias-recipient-delimiter.txt deleted file mode 100644 index 07cb8d40..00000000 --- a/test/files/emails/existing/alias-recipient-delimiter.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local Alias With Delimiter -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-alias-recipient-delimiter.txt -This is a test mail. diff --git a/test/files/emails/existing/catchall-local.txt b/test/files/emails/existing/catchall-local.txt deleted file mode 100644 index ab3e1988..00000000 --- a/test/files/emails/existing/catchall-local.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-catchall-local.txt -This is a test mail. diff --git a/test/files/emails/existing/regexp-alias-external.txt b/test/files/emails/existing/regexp-alias-external.txt deleted file mode 100644 index b50ac90f..00000000 --- a/test/files/emails/existing/regexp-alias-external.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-regexp-alias-external.txt -This is a test mail. diff --git a/test/files/emails/existing/regexp-alias-local.txt b/test/files/emails/existing/regexp-alias-local.txt deleted file mode 100644 index e45b7c6c..00000000 --- a/test/files/emails/existing/regexp-alias-local.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-regexp-alias-local.txt -This is a test mail. diff --git a/test/files/emails/existing/user-and-cc-local-alias.txt b/test/files/emails/existing/user-and-cc-local-alias.txt deleted file mode 100644 index 37814f91..00000000 --- a/test/files/emails/existing/user-and-cc-local-alias.txt +++ /dev/null @@ -1,6 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Cc: Existing Local Alias -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-user-and-cc-local-alias.txt -This is a test mail. diff --git a/test/files/emails/existing/user1.txt b/test/files/emails/existing/user1.txt deleted file mode 100644 index 23d49dc9..00000000 --- a/test/files/emails/existing/user1.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-user1.txt -This is a test mail. diff --git a/test/files/emails/non-existing-user.txt b/test/files/emails/non-existing-user.txt deleted file mode 100644 index 3d92470e..00000000 --- a/test/files/emails/non-existing-user.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message non-existing-user.txt -This is a test mail. diff --git a/test/files/emails/rspamd/pass.txt b/test/files/emails/rspamd/pass.txt deleted file mode 100644 index ce9286b1..00000000 --- a/test/files/emails/rspamd/pass.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message rspamd/pass.txt -This mail should pass and Rspamd should not mark it. diff --git a/test/files/emails/rspamd/spam-header.txt b/test/files/emails/rspamd/spam-header.txt deleted file mode 100644 index 8722e42f..00000000 --- a/test/files/emails/rspamd/spam-header.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 21 Jan 2023 11:11:11 +0000 -Subject: Test Message rspamd-spam-header.txt -YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X diff --git a/test/files/emails/rspamd/spam.txt b/test/files/emails/rspamd/spam.txt deleted file mode 100644 index c561e779..00000000 --- a/test/files/emails/rspamd/spam.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 21 Jan 2023 11:11:11 +0000 -Subject: Test Message rspamd-spam.txt -XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X diff --git a/test/files/emails/rspamd/virus.txt b/test/files/emails/rspamd/virus.txt deleted file mode 100644 index cb18927d..00000000 --- a/test/files/emails/rspamd/virus.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 21 Jan 2023 11:11:11 +0000 -Subject: Test Message rspamd-virus.txt -X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H* diff --git a/test/helper/common.bash b/test/helper/common.bash index ab21ef60..0891bf8c 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -466,7 +466,7 @@ function _print_mail_log_for_id() { local MAIL_ID=${1:?Mail ID must be provided} local CONTAINER_NAME=$(__handle_container_name "${2:-}") - _run_in_container grep -F "${MAIL_ID}" /var/log/mail.log + _run_in_container grep -E "${MAIL_ID}" /var/log/mail.log } # A simple wrapper for netcat (`nc`). This is useful when sending @@ -480,7 +480,7 @@ function _nc_wrapper() { [[ -v CONTAINER_NAME ]] || return 1 - _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}.txt" + _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}" } # ? << Miscellaneous helper functions diff --git a/test/helper/sending.bash b/test/helper/sending.bash index 48012178..529e36aa 100644 --- a/test/helper/sending.bash +++ b/test/helper/sending.bash @@ -7,68 +7,118 @@ # ! ATTENTION: This file is loaded by `common.sh` - do not load it yourself! # ! ATTENTION: This file requires helper functions from `common.sh`! -# Sends a mail from localhost (127.0.0.1) to a container. To send -# a custom email, create a file at `test/files/`, -# and provide `` as an argument to this function. +# Sends an e-mail from the container named by the environment variable `CONTAINER_NAME` +# to the same or another container. # -# Parameters include all options that one can supply to `swaks` -# itself. The `--data` parameter expects a relative path from `emails/` -# where the contents will be implicitly provided to `swaks` via STDIN. +# To send a custom email, you can +# +# 1. create a file at `test/files/` and provide `` via `--data` as an argument to this function; +# 2. use this function without the `--data` argument, in which case we provide a default; +# 3. provide data inline (`--data `). +# +# The very first parameter **may** be `--expect-rejection` - use it of you expect the mail transaction to not finish +# successfully. All other (following) parameters include all options that one can supply to `swaks` itself. +# As mentioned before, the `--data` parameter expects a value of either: +# +# - A relative path from `test/files/emails/` +# - An "inline" data string (e.g., `Date: 1 Jan 2024\nSubject: This is a test`) +# +# ## Output +# +# This functions prints the output of the transaction that `swaks` prints. # # ## Attention # # This function assumes `CONTAINER_NAME` to be properly set (to the container # name the command should be executed in)! # -# This function will just send the email in an "asynchronous" fashion, i.e. it will -# send the email but it will not make sure the mail queue is empty after the mail -# has been sent. +# This function will send the email in an "asynchronous" fashion, +# it will return without waiting for the Postfix mail queue to be emptied. function _send_email() { - [[ -v CONTAINER_NAME ]] || return 1 + local RETURN_VALUE=0 + local COMMAND_STRING - # Parameter defaults common to our testing needs: - local EHLO='mail.external.tld' - local FROM='user@external.tld' - local TO='user1@localhost.localdomain' - local SERVER='0.0.0.0' - local PORT=25 - # Extra options for `swaks` that aren't covered by the default options above: - local ADDITIONAL_SWAKS_OPTIONS=() - # Specifically for handling `--data` option below: - local FINAL_SWAKS_OPTIONS=() + function __parse_arguments() { + [[ -v CONTAINER_NAME ]] || return 1 - while [[ ${#} -gt 0 ]]; do - case "${1}" in - ( '--ehlo' ) EHLO=${2:?--ehlo given but no argument} ; shift 2 ;; - ( '--from' ) FROM=${2:?--from given but no argument} ; shift 2 ;; - ( '--to' ) TO=${2:?--to given but no argument} ; shift 2 ;; - ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;; - ( '--port' ) PORT=${2:?--port given but no argument} ; shift 2 ;; - ( '--data' ) - local TEMPLATE_FILE="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}.txt" - FINAL_SWAKS_OPTIONS+=('--data') - FINAL_SWAKS_OPTIONS+=('-') - FINAL_SWAKS_OPTIONS+=('<') - FINAL_SWAKS_OPTIONS+=("${TEMPLATE_FILE}") - shift 2 - ;; - ( * ) ADDITIONAL_SWAKS_OPTIONS+=("${1}") ; shift 1 ;; - esac - done + # Parameter defaults common to our testing needs: + local EHLO='mail.external.tld' + local FROM='user@external.tld' + local TO='user1@localhost.localdomain' + local SERVER='0.0.0.0' + local PORT=25 + # Extra options for `swaks` that aren't covered by the default options above: + local ADDITIONAL_SWAKS_OPTIONS=() + local DATA_WAS_SUPPLIED=0 - _run_in_container_bash "swaks --server ${SERVER} --port ${PORT} --ehlo ${EHLO} --from ${FROM} --to ${TO} ${ADDITIONAL_SWAKS_OPTIONS[*]} ${FINAL_SWAKS_OPTIONS[*]}" + while [[ ${#} -gt 0 ]]; do + case "${1}" in + ( '--ehlo' ) EHLO=${2:?--ehlo given but no argument} ; shift 2 ;; + ( '--from' ) FROM=${2:?--from given but no argument} ; shift 2 ;; + ( '--to' ) TO=${2:?--to given but no argument} ; shift 2 ;; + ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;; + ( '--port' ) PORT=${2:?--port given but no argument} ; shift 2 ;; + ( '--data' ) + ADDITIONAL_SWAKS_OPTIONS+=('--data') + local FILE_PATH="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}" + if _exec_in_container_bash "[[ -e ${FILE_PATH} ]]"; then + ADDITIONAL_SWAKS_OPTIONS+=("@${FILE_PATH}") + else + ADDITIONAL_SWAKS_OPTIONS+=("'${2}'") + fi + shift 2 + DATA_WAS_SUPPLIED=1 + ;; + ( * ) ADDITIONAL_SWAKS_OPTIONS+=("'${1}'") ; shift 1 ;; + esac + done + + if [[ ${DATA_WAS_SUPPLIED} -eq 0 ]]; then + # Fallback template (without the implicit `Message-Id` + `X-Mailer` headers from swaks): + # NOTE: It is better to let Postfix generate and append the `Message-Id` header itself, + # as it will contain the Queue ID for tracking in logs (which is also returned in swaks output). + ADDITIONAL_SWAKS_OPTIONS+=('--data') + ADDITIONAL_SWAKS_OPTIONS+=("'Date: %DATE%\nTo: %TO_ADDRESS%\nFrom: %FROM_ADDRESS%\nSubject: test %DATE%\n%NEW_HEADERS%\n%BODY%\n'") + fi + + echo "swaks --server '${SERVER}' --port '${PORT}' --ehlo '${EHLO}' --from '${FROM}' --to '${TO}' ${ADDITIONAL_SWAKS_OPTIONS[*]}" + } + + if [[ ${1:-} == --expect-rejection ]]; then + shift 1 + COMMAND_STRING=$(__parse_arguments "${@}") + _run_in_container_bash "${COMMAND_STRING}" + RETURN_VALUE=${?} + else + COMMAND_STRING=$(__parse_arguments "${@}") + _run_in_container_bash "${COMMAND_STRING}" + assert_success + fi + + # shellcheck disable=SC2154 + echo "${output}" + return "${RETURN_VALUE}" } # Like `_send_email` with two major differences: # # 1. this function waits for the mail to be processed; there is no asynchronicity -# because filtering the logs in a synchronous way is easier and safer! -# 2. this function prints an ID one can later filter by to check logs +# because filtering the logs in a synchronous way is easier and safer; +# 2. this function takes the name of a variable and inserts IDs one can later +# filter by to check logs. # # No. 2 is especially useful in case you send more than one email in a single # test file and need to assert certain log entries for each mail individually. # -# This function takes the same arguments as `_send_mail`. +# The first argument has to be the name of the variable that the e-mail ID is stored in. +# The second argument **can** be the flag `--expect-rejection`. +# +# - If this flag is supplied, the function does not check whether the whole mail delivery +# transaction was successful. Additionally the queue ID will be retrieved differently. +# - CAUTION: It must still be possible to `grep` for the Message-ID that Postfix +# generated in the mail log; otherwise this function fails. +# +# The rest of the arguments are the same as `_send_email`. # # ## Attention # @@ -82,20 +132,35 @@ function _send_email() { # chosen. Sending more than one mail at any given point in time with this function # is UNDEFINED BEHAVIOR! function _send_email_and_get_id() { - [[ -v CONTAINER_NAME ]] || return 1 + # Export the variable denoted by ${1} so everyone has access + export "${1:?Mail ID must be set for _send_email_and_get_id}" + # Get a "reference" to the content of the variable denoted by ${1} so we can manipulate the content + local -n ID_ENV_VAR_REF=${1:?} + # Prepare the message ID header here because we will shift away ${1} later + local MID="<${1}@dms-tests>" + # Get rid of ${1} so only the arguments for swaks remain + shift 1 - _wait_for_empty_mail_queue_in_container - _send_email "${@}" - _wait_for_empty_mail_queue_in_container - - local MAIL_ID + local QUEUE_ID # The unique ID Postfix (and other services) use may be different in length # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a # range to safely capture it. - MAIL_ID=$(_exec_in_container tac /var/log/mail.log \ - | grep -E -m 1 'postfix/smtpd.*: [A-Z0-9]+: client=localhost' \ - | grep -E -o '[A-Z0-9]{9,12}' || true) + local QUEUE_ID_REGEX='[A-Z0-9]{9,12}' - assert_not_equal "${MAIL_ID}" '' - echo "${MAIL_ID}" + _wait_for_empty_mail_queue_in_container + local OUTPUT=$(_send_email "${@}" --header "Message-Id: ${MID}") + _wait_for_empty_mail_queue_in_container + + # We store Postfix's queue ID first + ID_ENV_VAR_REF=$(_exec_in_container tac /var/log/mail.log \ + | grep -E "postfix/cleanup.*: ${QUEUE_ID_REGEX}:.*message-id=${MID}" \ + | grep -E --only-matching --max-count 1 "${QUEUE_ID_REGEX}" || :) + # But we also requre potential Dovecot sieve output, which requires the mesage ID, + # so we need to provide the message ID too. + ID_ENV_VAR_REF+="|${MID}" + + # Last but not least, we perform plausibility checks on the IDs. + assert_not_equal "${ID_ENV_VAR_REF}" '' + run echo "${ID_ENV_VAR_REF}" + assert_line --regexp "^${QUEUE_ID_REGEX}\|${MID}$" } diff --git a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats index 81cf9bc1..8e71bf73 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats @@ -225,12 +225,9 @@ function teardown_file() { _default_teardown ; } sleep 10 # send some big emails - _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' - assert_success - _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' - assert_success - _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' - assert_success + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt' + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt' + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt' # check for quota warn message existence run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/ assert_success diff --git a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats index e3e076a5..6acbc4a5 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats @@ -26,11 +26,9 @@ function setup_file() { _wait_for_smtp_port_in_container # Single mail sent from 'spam@spam.com' that is handled by User (relocate) and Global (copy) sieves for user1: - _send_email --data 'sieve/spam-folder' - assert_success + _send_email --data 'sieve/spam-folder.txt' # Mail for user2 triggers the sieve-pipe: - _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe' - assert_success + _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe.txt' _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats index 033a5bde..b9f6d8f6 100644 --- a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats +++ b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats @@ -26,8 +26,7 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' - assert_success + _send_email _wait_for_empty_mail_queue_in_container # Mail received should be stored as `u.1` (one file per message) @@ -48,8 +47,7 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' - assert_success + _send_email _wait_for_empty_mail_queue_in_container # Mail received should be stored in `m.1` (1 or more messages) diff --git a/test/tests/parallel/set1/dovecot/special_use_folders.bats b/test/tests/parallel/set1/dovecot/special_use_folders.bats index fe1f554e..4965b844 100644 --- a/test/tests/parallel/set1/dovecot/special_use_folders.bats +++ b/test/tests/parallel/set1/dovecot/special_use_folders.bats @@ -14,8 +14,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'normal delivery works' { - _send_email --data 'existing/user1' - assert_success + _send_email _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new 1 } @@ -27,7 +26,7 @@ function teardown_file() { _default_teardown ; } } @test "(IMAP) special-use folders should be created when necessary" { - _nc_wrapper 'nc/imap_special_use_folders' '-w 8 0.0.0.0 143' + _nc_wrapper 'nc/imap_special_use_folders.txt' '-w 8 0.0.0.0 143' assert_output --partial 'Drafts' assert_output --partial 'Junk' assert_output --partial 'Trash' diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index 9c035f5b..a916896a 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -25,8 +25,7 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - _send_email --from 'virus@external.tld' --data 'amavis/virus' - assert_success + _send_email --from 'virus@external.tld' --data 'amavis/virus.txt' _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index f2474cc0..5ec28396 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -18,8 +18,7 @@ function setup_file() { _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' - assert_success + _send_email _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/spam_virus/fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats index 8a03ba04..f451befd 100644 --- a/test/tests/parallel/set1/spam_virus/fail2ban.bats +++ b/test/tests/parallel/set1/spam_virus/fail2ban.bats @@ -74,7 +74,7 @@ function teardown_file() { CONTAINER1_IP=$(_get_container_ip "${CONTAINER1_NAME}") # Trigger a ban by failing to login twice: for _ in {1..2}; do - CONTAINER_NAME=${CONTAINER2_NAME} _send_email \ + CONTAINER_NAME=${CONTAINER2_NAME} _send_email --expect-rejection \ --server "${CONTAINER1_IP}" \ --port 465 \ --auth PLAIN \ diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 389fc183..5538d3bf 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -51,7 +51,7 @@ function teardown_file() { _default_teardown ; } _reload_postfix # Send test mail (it should fail to deliver): - _send_email --from 'user@external.tld' --port 25 --data 'postgrey' + _send_email --expect-rejection --from 'user@external.tld' --port 25 --data 'postgrey.txt' assert_failure assert_output --partial 'Recipient address rejected: Delayed by Postgrey' @@ -67,8 +67,7 @@ function teardown_file() { _default_teardown ; } # Wait until `$POSTGREY_DELAY` seconds pass before trying again: sleep 3 # Retry delivering test mail (it should be trusted this time): - _send_email --from 'user@external.tld' --port 25 --data 'postgrey' - assert_success + _send_email --from 'user@external.tld' --port 25 --data 'postgrey.txt' # Confirm postgrey permitted delivery (triplet is now trusted): _should_have_log_entry \ @@ -87,7 +86,7 @@ function teardown_file() { _default_teardown ; } # - It'd also cause the earlier greylist test to fail. # - TODO: Actually confirm whitelist feature works correctly as these test cases are using a workaround: @test "should whitelist sender 'user@whitelist.tld'" { - _nc_wrapper 'nc/postgrey_whitelist' '-w 0 0.0.0.0 10023' + _nc_wrapper 'nc/postgrey_whitelist.txt' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ @@ -96,7 +95,7 @@ function teardown_file() { _default_teardown ; } } @test "should whitelist recipient 'user2@otherdomain.tld'" { - _nc_wrapper 'nc/postgrey_whitelist_recipients' '-w 0 0.0.0.0 10023' + _nc_wrapper 'nc/postgrey_whitelist_recipients.txt' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index 377b2479..92333f98 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -44,7 +44,7 @@ function teardown_file() { # Use `nc` to send all SMTP commands at once instead (emulate a misbehaving client that should be rejected) # NOTE: Postscreen only runs on port 25, avoid implicit ports in test methods @test 'should fail send when talking out of turn' { - CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen' "${CONTAINER1_IP} 25" + CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen.txt' "${CONTAINER1_IP} 25" # Expected postscreen log entry: assert_output --partial 'Protocol error' @@ -56,14 +56,10 @@ function teardown_file() { @test "should successfully pass postscreen and get postfix greeting message (respecting postscreen_greet_wait time)" { # Configure `send_email()` to send from the mail client container (CONTAINER2_NAME) via ENV override, # mail is sent to the DMS server container (CONTAINER1_NAME) via `--server` parameter: - CONTAINER_NAME=${CONTAINER2_NAME} _send_email --server "${CONTAINER1_IP}" --port 25 --data 'postscreen' - # NOTE: Cannot assert_success due to sender address not being resolvable. - # TODO: Uncomment when proper resolution of domain names is possible: - # assert_success - - # TODO: Prefer this approach when `_send_email_and_get_id()` can support separate client and server containers: - # local MAIL_ID=$(_send_email_and_get_id --port 25 --data 'postscreen') - # _print_mail_log_for_id "${MAIL_ID}" + # TODO: Use _send_email_and_get_id when proper resolution of domain names is possible: + CONTAINER_NAME=${CONTAINER2_NAME} _send_email --expect-rejection --server "${CONTAINER1_IP}" --port 25 --data 'postscreen.txt' + # CONTAINER_NAME=${CONTAINER2_NAME} _send_email_and_get_id MAIL_ID_POSTSCREEN --server "${CONTAINER1_IP}" --data 'postscreen.txt' + # _print_mail_log_for_id "${MAIL_ID_POSTSCREEN}" # assert_output --partial "stored mail into mailbox 'INBOX'" _run_in_container cat /var/log/mail.log diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 2e610d72..b83ac353 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -43,16 +43,24 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - # We will send 3 emails: the first one should pass just fine; the second one should - # be rejected due to spam; the third one should be rejected due to a virus. - export MAIL_ID1=$(_send_email_and_get_id --from 'rspamd-pass@example.test' --data 'rspamd/pass') - export MAIL_ID2=$(_send_email_and_get_id --from 'rspamd-spam@example.test' --data 'rspamd/spam') - export MAIL_ID3=$(_send_email_and_get_id --from 'rspamd-virus@example.test' --data 'rspamd/virus') - export MAIL_ID4=$(_send_email_and_get_id --from 'rspamd-spam-header@example.test' --data 'rspamd/spam-header') + # ref: https://rspamd.com/doc/gtube_patterns.html + local GTUBE_SUFFIX='*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' - for ID in MAIL_ID{1,2,3,4}; do - [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; } - done + # We will send 4 emails: + # 1. The first one should pass just fine + _send_email_and_get_id MAIL_ID_PASS + # 2. The second one should be rejected (GTUBE pattern) + _send_email_and_get_id MAIL_ID_REJECT --expect-rejection --body "XJS${GTUBE_SUFFIX}" + # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) + # shellcheck disable=SC2016 + _send_email_and_get_id MAIL_ID_VIRUS --expect-rejection \ + --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' + # 4. The fourth one will receive an added header (GTUBE pattern) + _send_email_and_get_id MAIL_ID_HEADER --body "YJS${GTUBE_SUFFIX}" + + _run_in_container cat /var/log/mail.log + assert_success + refute_output --partial 'inet:localhost:11332: Connection refused' } function teardown_file() { _default_teardown ; } @@ -104,7 +112,7 @@ function teardown_file() { _default_teardown ; } @test 'normal mail passes fine' { _service_log_should_contain_string 'rspamd' 'F \(no action\)' - _print_mail_log_for_id "${MAIL_ID1}" + _print_mail_log_for_id "${MAIL_ID_PASS}" assert_output --partial "stored mail into mailbox 'INBOX'" _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 @@ -114,7 +122,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'S \(reject\)' _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID2}" + _print_mail_log_for_id "${MAIL_ID_REJECT}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 Gtube pattern' @@ -125,7 +133,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'T \(reject\)' _service_log_should_contain_string 'rspamd' 'reject "ClamAV FOUND VIRUS "Eicar-Signature"' - _print_mail_log_for_id "${MAIL_ID3}" + _print_mail_log_for_id "${MAIL_ID_VIRUS}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"' refute_output --partial "stored mail into mailbox 'INBOX'" @@ -214,7 +222,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'S \(add header\)' _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID4}" + _print_mail_log_for_id "${MAIL_ID_HEADER}" assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 @@ -256,7 +264,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "Junk" folder from "INBOX"; the first email we # sent should pass fine, hence we can now move it. - _nc_wrapper 'nc/rspamd_imap_move_to_junk' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_junk.txt' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log @@ -270,7 +278,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "INBOX" folder from "Junk"; there should be two mails # in the "Junk" folder, since the second email we sent during setup should # have landed in the Junk folder already. - _nc_wrapper 'nc/rspamd_imap_move_to_inbox' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_inbox.txt' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index fea23b0b..2c3a522d 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -95,7 +95,7 @@ function teardown() { _default_teardown ; } function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis - _send_email --from 'spam@external.tld' --data 'amavis/spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' } function _should_be_received_by_amavis() { diff --git a/test/tests/parallel/set3/container_configuration/hostname.bats b/test/tests/parallel/set3/container_configuration/hostname.bats index f5774eef..a525ecb2 100644 --- a/test/tests/parallel/set3/container_configuration/hostname.bats +++ b/test/tests/parallel/set3/container_configuration/hostname.bats @@ -207,7 +207,7 @@ function _should_have_correct_mail_headers() { # (eg: OVERRIDE_HOSTNAME or `--hostname mail --domainname example.test`) local EXPECTED_HOSTNAME=${3:-${EXPECTED_FQDN}} - _send_email --from 'user@external.tld' --data 'existing/user1' + _send_email --from 'user@external.tld' _wait_for_empty_mail_queue_in_container _count_files_in_directory_in_container '/var/mail/localhost.localdomain/user1/new/' '1' diff --git a/test/tests/parallel/set3/mta/dsn.bats b/test/tests/parallel/set3/mta/dsn.bats index a5228cfc..9a65b147 100644 --- a/test/tests/parallel/set3/mta/dsn.bats +++ b/test/tests/parallel/set3/mta/dsn.bats @@ -49,9 +49,9 @@ function teardown_file() { # TODO replace with _send_email as soon as it supports DSN # TODO ref: https://github.com/jetmore/swaks/issues/41 - _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -62,7 +62,7 @@ function teardown_file() { @test "should only send a DSN when requested from ports 465/587" { export CONTAINER_NAME=${CONTAINER2_NAME} - _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt' _wait_for_empty_mail_queue_in_container # DSN requests can now only be made on ports 465 and 587, @@ -74,8 +74,8 @@ function teardown_file() { assert_failure # These ports are excluded via master.cf. - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -85,9 +85,9 @@ function teardown_file() { @test "should never send a DSN" { export CONTAINER_NAME=${CONTAINER3_NAME} - _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container # DSN requests are rejected regardless of origin. diff --git a/test/tests/parallel/set3/mta/lmtp_ip.bats b/test/tests/parallel/set3/mta/lmtp_ip.bats index d8be42d9..201cb237 100644 --- a/test/tests/parallel/set3/mta/lmtp_ip.bats +++ b/test/tests/parallel/set3/mta/lmtp_ip.bats @@ -38,7 +38,7 @@ function teardown_file() { _default_teardown ; } @test "delivers mail to existing account" { _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' # send a test email + _send_email # Verify delivery was successful, log line should look similar to: # postfix/lmtp[1274]: 0EA424ABE7D9: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.13, delays=0.07/0.01/0.01/0.05, dsn=2.0.0, status=sent (250 2.0.0 ixPpB+Zvv2P7BAAAUi6ngw Saved) diff --git a/test/tests/parallel/set3/mta/privacy.bats b/test/tests/parallel/set3/mta/privacy.bats index 4d4d82ba..614e2e87 100644 --- a/test/tests/parallel/set3/mta/privacy.bats +++ b/test/tests/parallel/set3/mta/privacy.bats @@ -26,11 +26,10 @@ function teardown_file() { _default_teardown ; } # this test covers https://github.com/docker-mailserver/docker-mailserver/issues/681 @test "(Postfix) remove privacy details of the sender" { _send_email \ - --port 587 -tls --auth LOGIN \ + --port 587 -tls --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ - --data 'privacy' - assert_success + --data 'privacy.txt' _run_until_success_or_timeout 120 _exec_in_container_bash '[[ -d /var/mail/localhost.localdomain/user1/new ]]' assert_success diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index f87f11ed..329f36b2 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -63,46 +63,43 @@ function setup_file() { # TODO: Move to clamav tests (For use when ClamAV is enabled): # _repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /var/run/clamav/clamd.ctl - # _send_email --from 'virus@external.tld' --data 'amavis/virus' + # _send_email --from 'virus@external.tld' --data 'amavis/virus.txt' # Required for 'delivers mail to existing alias': - _send_email --to alias1@localhost.localdomain --data 'existing/alias-external' + _send_email --to alias1@localhost.localdomain --header "Subject: Test Message existing-alias-external" # Required for 'delivers mail to existing alias with recipient delimiter': - _send_email --to alias1~test@localhost.localdomain --data 'existing/alias-recipient-delimiter' + _send_email --to alias1~test@localhost.localdomain --header 'Subject: Test Message existing-alias-recipient-delimiter' # Required for 'delivers mail to existing catchall': - _send_email --to wildcard@localdomain2.com --data 'existing/catchall-local' + _send_email --to wildcard@localdomain2.com --header 'Subject: Test Message existing-catchall-local' # Required for 'delivers mail to regexp alias': - _send_email --to test123@localhost.localdomain --data 'existing/regexp-alias-local' + _send_email --to test123@localhost.localdomain --header 'Subject: Test Message existing-regexp-alias-local' # Required for 'rejects mail to unknown user': - _send_email --to nouser@localhost.localdomain --data 'non-existing-user' + _send_email --expect-rejection --to nouser@localhost.localdomain + assert_failure # Required for 'redirects mail to external aliases': - _send_email --to bounce-always@localhost.localdomain --data 'existing/regexp-alias-external' - _send_email --to alias2@localhost.localdomain --data 'existing/alias-local' + _send_email --to bounce-always@localhost.localdomain + _send_email --to alias2@localhost.localdomain # Required for 'rejects spam': - _send_email --from 'spam@external.tld' --data 'amavis/spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' # Required for 'delivers mail to existing account': - _send_email --data 'existing/user1' - assert_success + _send_email --header 'Subject: Test Message existing-user1' _send_email --to user2@otherdomain.tld - assert_success _send_email --to user3@localhost.localdomain - assert_success - _send_email --to added@localhost.localdomain --data 'existing/added' - assert_success - _send_email --to user1@localhost.localdomain --data 'existing/user-and-cc-local-alias' - assert_success - _send_email --data 'sieve/spam-folder' - assert_success - _send_email --to user2@otherdomain.tld --data 'sieve/pipe' - assert_success + _send_email --to added@localhost.localdomain --header 'Subject: Test Message existing-added' + _send_email \ + --to user1@localhost.localdomain \ + --header 'Subject: Test Message existing-user-and-cc-local-alias' \ + --cc 'alias2@localhost.localdomain' + _send_email --data 'sieve/spam-folder.txt' + _send_email --to user2@otherdomain.tld --data 'sieve/pipe.txt' _run_in_container_bash 'sendmail root < /tmp/docker-mailserver-test/emails/sendmail/root-email.txt' assert_success } function _unsuccessful() { - _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword + _send_email --expect-rejection --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword --quit-after AUTH assert_failure assert_output --partial 'authentication failed' assert_output --partial 'No authentication type succeeded' @@ -110,7 +107,6 @@ function _unsuccessful() { function _successful() { _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password mypassword --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' } diff --git a/test/tests/serial/mail_pop3.bats b/test/tests/serial/mail_pop3.bats index 008921e4..d815e1ee 100644 --- a/test/tests/serial/mail_pop3.bats +++ b/test/tests/serial/mail_pop3.bats @@ -24,12 +24,12 @@ function teardown_file() { _default_teardown ; } } @test 'authentication works' { - _nc_wrapper 'auth/pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/pop3-auth.txt' '-w 1 0.0.0.0 110' assert_success } @test 'added user authentication works' { - _nc_wrapper 'auth/added-pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/added-pop3-auth.txt' '-w 1 0.0.0.0 110' assert_success } diff --git a/test/tests/serial/mail_with_imap.bats b/test/tests/serial/mail_with_imap.bats index eeccf888..94f1d519 100644 --- a/test/tests/serial/mail_with_imap.bats +++ b/test/tests/serial/mail_with_imap.bats @@ -21,7 +21,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test '(Dovecot) LDAP RIMAP connection and authentication works' { - _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @@ -31,8 +31,8 @@ function teardown_file() { _default_teardown ; } } @test '(SASLauthd) RIMAP SMTP authentication works' { - _send_email \ - --auth LOGIN \ + _send_email --expect-rejection \ + --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --quit-after AUTH @@ -41,20 +41,18 @@ function teardown_file() { _default_teardown ; } _send_email \ --port 465 \ - --auth LOGIN \ + --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' _send_email \ --port 587 \ - --auth LOGIN \ + --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' } diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index f2011d22..f3ee1cc1 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -248,7 +248,7 @@ function teardown() { # dovecot @test "dovecot: ldap imap connection and authentication works" { - _nc_wrapper 'auth/imap-ldap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-ldap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @@ -326,25 +326,26 @@ function teardown() { @test "spoofing (with LDAP): rejects sender forging" { _wait_for_smtp_port_in_container_to_respond dms-test_ldap - _send_email \ - --port 465 -tlsc --auth LOGIN \ + _send_email --expect-rejection \ + --port 465 -tlsc --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password secret \ --ehlo mail \ --from ldap@localhost.localdomain \ - --data 'auth/ldap-smtp-auth-spoofed' + --data 'auth/ldap-smtp-auth-spoofed.txt' + assert_failure assert_output --partial 'Sender address rejected: not owned by user' } @test "spoofing (with LDAP): accepts sending as alias" { _send_email \ - --port 465 -tlsc --auth LOGIN \ + --port 465 -tlsc --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password secret \ --ehlo mail \ --from postmaster@localhost.localdomain \ --to some.user@localhost.localdomain \ - --data 'auth/ldap-smtp-auth-spoofed-alias' + --data 'auth/ldap-smtp-auth-spoofed-alias.txt' assert_output --partial 'End data with' } @@ -353,20 +354,21 @@ function teardown() { # Template used has invalid AUTH: https://github.com/docker-mailserver/docker-mailserver/pull/3006#discussion_r1073321432 skip 'TODO: This test seems to have been broken from the start (?)' - _send_email \ - --port 465 -tlsc --auth LOGIN \ + _send_email --expect-rejection \ + --port 465 -tlsc --auth PLAIN \ --auth-user some.user.email@localhost.localdomain \ --auth-password secret \ --ehlo mail \ --from randomspoofedaddress@localhost.localdomain \ --to some.user@localhost.localdomain \ - --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception' + --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt' + assert_failure assert_output --partial 'Sender address rejected: not owned by user' } @test "saslauthd: ldap smtp authentication" { - _send_email \ - --auth LOGIN \ + _send_email --expect-rejection \ + --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password wrongpassword \ --quit-after AUTH @@ -379,12 +381,11 @@ function teardown() { --auth-user some.user@localhost.localdomain \ --auth-password secret \ --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' _send_email \ --port 587 -tls \ - --auth LOGIN \ + --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password secret \ --quit-after AUTH diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 752e325e..a344bd8d 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -80,12 +80,12 @@ function teardown_file() { _default_teardown ; } } @test "imap: authentication works" { - _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @test "imap: added user authentication works" { - _nc_wrapper 'auth/added-imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/added-imap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @@ -293,13 +293,13 @@ EOF # An authenticated user cannot use an envelope sender (MAIL FROM) # address they do not own according to `main.cf:smtpd_sender_login_maps` lookup - _send_email \ - --port 465 -tlsc --auth LOGIN \ + _send_email --expect-rejection \ + --port 465 -tlsc --auth PLAIN \ --auth-user added@localhost.localdomain \ --auth-password mypassword \ --ehlo mail \ --from user2@localhost.localdomain \ - --data 'auth/added-smtp-auth-spoofed' + --data 'auth/added-smtp-auth-spoofed.txt' assert_output --partial 'Sender address rejected: not owned by user' } @@ -310,12 +310,12 @@ EOF # to each table. Address is authorized when a result that maps to # the DMS account is returned. _send_email \ - --port 465 -tlsc --auth LOGIN \ + --port 465 -tlsc --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --ehlo mail \ --from alias1@localhost.localdomain \ - --data 'auth/added-smtp-auth-spoofed-alias' + --data 'auth/added-smtp-auth-spoofed-alias.txt' assert_success assert_output --partial 'End data with' } diff --git a/test/tests/serial/vmail-id.bats b/test/tests/serial/vmail-id.bats index 0f54ea96..2541f4f8 100644 --- a/test/tests/serial/vmail-id.bats +++ b/test/tests/serial/vmail-id.bats @@ -20,7 +20,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'should successfully deliver mail' { - _send_email --data 'existing/user1' + _send_email --header 'Subject: Test Message existing-user1' _wait_for_empty_mail_queue_in_container # Should be successfully sent (received) by Postfix: @@ -31,7 +31,7 @@ function teardown_file() { _default_teardown ; } # Verify successful delivery via Dovecot to `/var/mail` account by searching for the subject: _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep -R \ - 'Subject: Test Message existing-user1.txt' \ + 'Subject: Test Message existing-user1' \ '/var/mail/localhost.localdomain/user1/new/' assert_success _should_output_number_of_lines 1 From 52c4582f7b431e69fd8eae67b97bbb330614d0b5 Mon Sep 17 00:00:00 2001 From: Keval Kapdee Date: Fri, 12 Jan 2024 20:45:14 +0000 Subject: [PATCH 113/125] feat: Auth - OAuth2 (Dovecot PassDB) (#3480) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .gitignore | 1 + CHANGELOG.md | 7 +++ Dockerfile | 7 +++ README.md | 1 + docs/content/config/advanced/auth-oauth2.md | 69 +++++++++++++++++++++ docs/content/config/environment.md | 24 ++++++- docs/mkdocs.yml | 1 + mailserver.env | 12 ++++ target/dovecot/10-auth.conf | 1 + target/dovecot/auth-oauth2.conf.ext | 7 +++ target/dovecot/dovecot-oauth2.conf.ext | 4 ++ target/scripts/start-mailserver.sh | 5 ++ target/scripts/startup/setup.d/oauth2.sh | 11 ++++ target/scripts/startup/variables-stack.sh | 6 ++ test/config/oauth2/provider.py | 56 +++++++++++++++++ test/files/auth/imap-oauth2-auth.txt | 4 ++ test/tests/serial/mail_with_oauth2.bats | 66 ++++++++++++++++++++ 17 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 docs/content/config/advanced/auth-oauth2.md create mode 100644 target/dovecot/auth-oauth2.conf.ext create mode 100644 target/dovecot/dovecot-oauth2.conf.ext create mode 100644 target/scripts/startup/setup.d/oauth2.sh create mode 100644 test/config/oauth2/provider.py create mode 100644 test/files/auth/imap-oauth2-auth.txt create mode 100644 test/tests/serial/mail_with_oauth2.bats diff --git a/.gitignore b/.gitignore index 50d22a22..79a4dc3c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ################################################# .env +compose.override.yaml docs/site/ docker-data/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 382b9efa..e3881941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Features + + - **Authentication with OIDC / OAuth 2.0** 🎉 + - DMS now supports authentication via OAuth2 (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) from capable services (_like Roundcube_). + - This does not replace the need for an `ACCOUNT_PROVISIONER` (`FILE` / `LDAP`), which is required for an account to receive or send mail. + - Successful authentication (_via Dovecot PassDB_) still requires an existing account (_lookup via Dovecot UserDB_). + ### Updates - **Tests**: diff --git a/Dockerfile b/Dockerfile index 65d818ab..2118622d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,6 +108,13 @@ EOF COPY target/rspamd/local.d/ /etc/rspamd/local.d/ COPY target/rspamd/scores.d/* /etc/rspamd/scores.d/ +# ----------------------------------------------- +# --- OAUTH2 ------------------------------------ +# ----------------------------------------------- + +COPY target/dovecot/auth-oauth2.conf.ext /etc/dovecot/conf.d +COPY target/dovecot/dovecot-oauth2.conf.ext /etc/dovecot + # ----------------------------------------------- # --- LDAP & SpamAssassin's Cron ---------------- # ----------------------------------------------- diff --git a/README.md b/README.md index 8918c2e9..e81d21ad 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,4 @@ If you have issues, please search through [the documentation][documentation::web - Support for [LetsEncrypt](https://letsencrypt.org/), manual and self-signed certificates - A [setup script](https://docker-mailserver.github.io/docker-mailserver/latest/config/setup.sh) for easy configuration and maintenance - SASLauthd with LDAP authentication +- OAuth2 authentication (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) diff --git a/docs/content/config/advanced/auth-oauth2.md b/docs/content/config/advanced/auth-oauth2.md new file mode 100644 index 00000000..963a6c2c --- /dev/null +++ b/docs/content/config/advanced/auth-oauth2.md @@ -0,0 +1,69 @@ +--- +title: 'Advanced | Basic OAuth2 Authentication' +--- + +## Introduction + +!!! warning "This is only a supplement to the existing account provisioners" + + Accounts must still be managed via the configured [`ACCOUNT_PROVISIONER`][env::account-provisioner] (FILE or LDAP). + + Reasoning for this can be found in [#3480][gh-pr::oauth2]. Future iterations on this feature may allow it to become a full account provisioner. + +[gh-pr::oauth2]: https://github.com/docker-mailserver/docker-mailserver/pull/3480 +[env::account-provisioner]: ../environment.md#account_provisioner + +The present OAuth2 support provides the capability for 3rd-party applications such as Roundcube to authenticate with DMS (dovecot) by using a token obtained from an OAuth2 provider, instead of passing passwords around. + +## Example (Authentik & Roundcube) + +This example assumes you have: + +- A working DMS server set up +- An Authentik server set up ([documentation](https://goauthentik.io/docs/installation/)) +- A Roundcube server set up (either [docker](https://hub.docker.com/r/roundcube/roundcubemail/) or [bare metal](https://github.com/roundcube/roundcubemail/wiki/Installation)) + +!!! example "Setup Instructions" + + === "1. Docker Mailserver" + Edit the following values in `mailserver.env`: + ```env + # ----------------------------------------------- + # --- OAUTH2 Section ---------------------------- + # ----------------------------------------------- + + # empty => OAUTH2 authentication is disabled + # 1 => OAUTH2 authentication is enabled + ENABLE_OAUTH2=1 + + # Specify the user info endpoint URL of the oauth2 provider + OAUTH2_INTROSPECTION_URL=https://authentik.example.com/application/o/userinfo/ + ``` + + === "2. Authentik" + 1. Create a new OAuth2 provider + 2. Note the client id and client secret + 3. Set the allowed redirect url to the equivalent of `https://roundcube.example.com/index.php/login/oauth` for your RoundCube instance. + + === "3. Roundcube" + Add the following to `oauth2.inc.php` ([documentation](https://github.com/roundcube/roundcubemail/wiki/Configuration)): + + ```php + $config['oauth_provider'] = 'generic'; + $config['oauth_provider_name'] = 'Authentik'; + $config['oauth_client_id'] = ''; + $config['oauth_client_secret'] = ''; + $config['oauth_auth_uri'] = 'https://authentik.example.com/application/o/authorize/'; + $config['oauth_token_uri'] = 'https://authentik.example.com/application/o/token/'; + $config['oauth_identity_uri'] = 'https://authentik.example.com/application/o/userinfo/'; + + // Optional: disable SSL certificate check on HTTP requests to OAuth server. For possible values, see: + // http://docs.guzzlephp.org/en/stable/request-options.html#verify + $config['oauth_verify_peer'] = false; + + $config['oauth_scope'] = 'email openid profile'; + $config['oauth_identity_fields'] = ['email']; + + // Boolean: automatically redirect to OAuth login when opening Roundcube without a valid session + $config['oauth_login_redirect'] = false; + ``` diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index c3c074fd..61e604aa 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -54,7 +54,15 @@ The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage m Configures the provisioning source of user accounts (including aliases) for user queries and authentication by services managed by DMS (_Postfix and Dovecot_). -User provisioning via OIDC is planned for the future, see [this tracking issue](https://github.com/docker-mailserver/docker-mailserver/issues/2713). +!!! tip "OAuth2 Support" + + Presently DMS supports OAuth2 only as an supplementary authentication method. + + - A third-party service must provide a valid token for the user which Dovecot validates with the authentication service provider. To enable this feature reference the [OAuth2 configuration example guide][docs::auth::oauth2-config-guide]. + - User accounts must be provisioned to receive mail via one of the supported `ACCOUNT_PROVISIONER` providers. + - User provisioning via OIDC is planned for the future, see [this tracking issue](https://github.com/docker-mailserver/docker-mailserver/issues/2713). + +[docs::auth::oauth2-config-guide]: ./advanced/auth-oauth2.md - **empty** => use FILE - LDAP => use LDAP authentication @@ -716,10 +724,20 @@ Enable or disable `getmail`. - **5** => `getmail` The number of minutes for the interval. Min: 1; Max: 30; Default: 5. + +#### OAUTH2 + +##### ENABLE_OAUTH2 + +- **empty** => OAUTH2 authentication is disabled +- 1 => OAUTH2 authentication is enabled + +##### OAUTH2_INTROSPECTION_URL + +- => Specify the user info endpoint URL of the oauth2 provider (_eg: `https://oauth2.example.com/userinfo/`_) + #### LDAP - - ##### LDAP_START_TLS - **empty** => no diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 6441dffe..7df1acbe 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -142,6 +142,7 @@ nav: - 'Postfix': config/advanced/override-defaults/postfix.md - 'Modifications via Script': config/advanced/override-defaults/user-patches.md - 'LDAP Authentication': config/advanced/auth-ldap.md + - 'OAuth2 Authentication': config/advanced/auth-oauth2.md - 'Email Filtering with Sieve': config/advanced/mail-sieve.md - 'Email Gathering with Fetchmail': config/advanced/mail-fetchmail.md - 'Email Gathering with Getmail': config/advanced/mail-getmail.md diff --git a/mailserver.env b/mailserver.env index 0c2e1e40..c9793a99 100644 --- a/mailserver.env +++ b/mailserver.env @@ -419,6 +419,18 @@ ENABLE_GETMAIL=0 # The number of minutes for the interval. Min: 1; Max: 30. GETMAIL_POLL=5 +# ----------------------------------------------- +# --- OAUTH2 Section ---------------------------- +# ----------------------------------------------- + +# empty => OAUTH2 authentication is disabled +# 1 => OAUTH2 authentication is enabled +ENABLE_OAUTH2= + +# Specify the user info endpoint URL of the oauth2 provider +# Example: https://oauth2.example.com/userinfo/ +OAUTH2_INTROSPECTION_URL= + # ----------------------------------------------- # --- LDAP Section ------------------------------ # ----------------------------------------------- diff --git a/target/dovecot/10-auth.conf b/target/dovecot/10-auth.conf index f71289e9..260832fb 100644 --- a/target/dovecot/10-auth.conf +++ b/target/dovecot/10-auth.conf @@ -123,6 +123,7 @@ auth_mechanisms = plain login #!include auth-sql.conf.ext #!include auth-ldap.conf.ext !include auth-passwdfile.inc +#!include auth-oauth2.conf.ext #!include auth-checkpassword.conf.ext #!include auth-vpopmail.conf.ext #!include auth-static.conf.ext diff --git a/target/dovecot/auth-oauth2.conf.ext b/target/dovecot/auth-oauth2.conf.ext new file mode 100644 index 00000000..6096d1e4 --- /dev/null +++ b/target/dovecot/auth-oauth2.conf.ext @@ -0,0 +1,7 @@ +auth_mechanisms = $auth_mechanisms oauthbearer xoauth2 + +passdb { + driver = oauth2 + mechanisms = xoauth2 oauthbearer + args = /etc/dovecot/dovecot-oauth2.conf.ext +} diff --git a/target/dovecot/dovecot-oauth2.conf.ext b/target/dovecot/dovecot-oauth2.conf.ext new file mode 100644 index 00000000..6998ed08 --- /dev/null +++ b/target/dovecot/dovecot-oauth2.conf.ext @@ -0,0 +1,4 @@ +introspection_url = +# Dovecot defaults: +introspection_mode = auth +username_attribute = email diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 2129b74a..1f352229 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -71,6 +71,11 @@ function _register_functions() { ;; esac + if [[ ${ENABLE_OAUTH2} -eq 1 ]]; then + _environment_variables_oauth2 + _register_setup_function '_setup_oauth2' + fi + if [[ ${ENABLE_SASLAUTHD} -eq 1 ]]; then _environment_variables_saslauthd _register_setup_function '_setup_saslauthd' diff --git a/target/scripts/startup/setup.d/oauth2.sh b/target/scripts/startup/setup.d/oauth2.sh new file mode 100644 index 00000000..20e9ffd1 --- /dev/null +++ b/target/scripts/startup/setup.d/oauth2.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +function _setup_oauth2() { + _log 'debug' 'Setting up OAUTH2' + + # Enable OAuth2 PassDB (Authentication): + sedfile -i -e '/\!include auth-oauth2\.conf\.ext/s/^#//' /etc/dovecot/conf.d/10-auth.conf + _replace_by_env_in_file 'OAUTH2_' '/etc/dovecot/dovecot-oauth2.conf.ext' + + return 0 +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 2660ce89..fc38a39d 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -151,6 +151,12 @@ function __environment_variables_general_setup() { VARS[UPDATE_CHECK_INTERVAL]="${UPDATE_CHECK_INTERVAL:=1d}" } +function _environment_variables_oauth2() { + _log 'debug' 'Setting OAUTH2-related environment variables now' + + VARS[OAUTH2_INTROSPECTION_URL]="${OAUTH2_INTROSPECTION_URL:=}" +} + # This function handles environment variables related to LDAP. # NOTE: SASLAuthd and Dovecot LDAP support inherit these common ENV. function _environment_variables_ldap() { diff --git a/test/config/oauth2/provider.py b/test/config/oauth2/provider.py new file mode 100644 index 00000000..22fc8129 --- /dev/null +++ b/test/config/oauth2/provider.py @@ -0,0 +1,56 @@ +# OAuth2 mock service +# +# Dovecot will query this service with the token it was provided. +# If the session for the token is valid, a response provides an attribute to perform a UserDB lookup on (default: email). + +import json +import base64 +from http.server import BaseHTTPRequestHandler, HTTPServer + +# OAuth2.0 Bearer token (paste into https://jwt.io/ to check it's contents). +# You should never need to edit this unless you REALLY need to change the issuer. +token = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vcHJvdmlkZXIuZXhhbXBsZS50ZXN0OjgwMDAvIiwic3ViIjoiODJjMWMzMzRkY2M2ZTMxMWFlNGFhZWJmZTk0NmM1ZTg1OGYwNTVhZmYxY2U1YTM3YWE3Y2M5MWFhYjE3ZTM1YyIsImF1ZCI6Im1haWxzZXJ2ZXIiLCJ1aWQiOiI4OU4zR0NuN1M1Y090WkZNRTVBeVhNbmxURFdVcnEzRmd4YWlyWWhFIn0.zuCytArbphhJn9XT_y9cBdGqDCNo68tBrtOwPIsuKNyF340SaOuZa0xarZofygytdDpLtYr56QlPTKImi-n1ZWrHkRZkwrQi5jQ-j_n2hEAL0vUToLbDnXYfc5q2w7z7X0aoCmiK8-fV7Kx4CVTM7riBgpElf6F3wNAIcX6R1ijUh6ISCL0XYsdogf8WUNZipXY-O4R7YHXdOENuOp3G48hWhxuUh9PsUqE5yxDwLsOVzCTqg9S5gxPQzF2eCN9J0I2XiIlLKvLQPIZ2Y_K7iYvVwjpNdgb4xhm9wuKoIVinYkF_6CwIzAawBWIDJAbix1IslkUPQMGbupTDtOgTiQ" + +# This is the string the user-facing client (e.g. Roundcube) should send via IMAP to Dovecot. +# We include the user and the above token separated by '\1' chars as per the XOAUTH2 spec. +xoauth2 = base64.b64encode(f"user=user1@localhost.localdomain\1auth=Bearer {token}\1\1".encode("utf-8")) +# If changing the user above, use the new output from the below line with the contents of the AUTHENTICATE command in test/test-files/auth/imap-oauth2-auth.txt +print("XOAUTH2 string: " + str(xoauth2)) + + +class HTTPRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + auth = self.headers.get("Authorization") + if auth is None: + self.send_response(401) + self.end_headers() + return + if len(auth.split()) != 2: + self.send_response(401) + self.end_headers() + return + auth = auth.split()[1] + # Valid session, respond with JSON containing the expected `email` claim to match as Dovecot username: + if auth == token: + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({ + "email": "user1@localhost.localdomain", + "email_verified": True, + "sub": "82c1c334dcc6e311ae4aaebfe946c5e858f055aff1ce5a37aa7cc91aab17e35c" + }).encode("utf-8")) + else: + self.send_response(401) + self.end_headers() + +server = HTTPServer(('', 80), HTTPRequestHandler) +print("Starting server", flush=True) + +try: + server.serve_forever() +except KeyboardInterrupt: + print() + print("Received keyboard interrupt") +finally: + print("Exiting") diff --git a/test/files/auth/imap-oauth2-auth.txt b/test/files/auth/imap-oauth2-auth.txt new file mode 100644 index 00000000..825fabda --- /dev/null +++ b/test/files/auth/imap-oauth2-auth.txt @@ -0,0 +1,4 @@ +a0 NOOP See test/config/oauth2/provider.py to generate the below XOAUTH2 string +a1 AUTHENTICATE XOAUTH2 dXNlcj11c2VyMUBsb2NhbGhvc3QubG9jYWxkb21haW4BYXV0aD1CZWFyZXIgZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBjM01pT2lKb2RIUndPaTh2Y0hKdmRtbGtaWEl1WlhoaGJYQnNaUzUwWlhOME9qZ3dNREF2SWl3aWMzVmlJam9pT0RKak1XTXpNelJrWTJNMlpUTXhNV0ZsTkdGaFpXSm1aVGswTm1NMVpUZzFPR1l3TlRWaFptWXhZMlUxWVRNM1lXRTNZMk01TVdGaFlqRTNaVE0xWXlJc0ltRjFaQ0k2SW0xaGFXeHpaWEoyWlhJaUxDSjFhV1FpT2lJNE9VNHpSME51TjFNMVkwOTBXa1pOUlRWQmVWaE5ibXhVUkZkVmNuRXpSbWQ0WVdseVdXaEZJbjAuenVDeXRBcmJwaGhKbjlYVF95OWNCZEdxRENObzY4dEJydE93UElzdUtOeUYzNDBTYU91WmEweGFyWm9meWd5dGREcEx0WXI1NlFsUFRLSW1pLW4xWldySGtSWmt3clFpNWpRLWpfbjJoRUFMMHZVVG9MYkRuWFlmYzVxMnc3ejdYMGFvQ21pSzgtZlY3S3g0Q1ZUTTdyaUJncEVsZjZGM3dOQUljWDZSMWlqVWg2SVNDTDBYWXNkb2dmOFdVTlppcFhZLU80UjdZSFhkT0VOdU9wM0c0OGhXaHh1VWg5UHNVcUU1eXhEd0xzT1Z6Q1RxZzlTNWd4UFF6RjJlQ045SjBJMlhpSWxMS3ZMUVBJWjJZX0s3aVl2VndqcE5kZ2I0eGhtOXd1S29JVmluWWtGXzZDd0l6QWF3QldJREpBYml4MUlzbGtVUFFNR2J1cFREdE9nVGlRAQE= +a2 EXAMINE INBOX +a3 LOGOUT diff --git a/test/tests/serial/mail_with_oauth2.bats b/test/tests/serial/mail_with_oauth2.bats new file mode 100644 index 00000000..0d73bc54 --- /dev/null +++ b/test/tests/serial/mail_with_oauth2.bats @@ -0,0 +1,66 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +BATS_TEST_NAME_PREFIX='[OAuth2] ' +CONTAINER1_NAME='dms-test_oauth2' +CONTAINER2_NAME='dms-test_oauth2_provider' + +function setup_file() { + export DMS_TEST_NETWORK='test-network-oauth2' + export DMS_DOMAIN='example.test' + export FQDN_MAIL="mail.${DMS_DOMAIN}" + export FQDN_OAUTH2="oauth2.${DMS_DOMAIN}" + + # Link the test containers to separate network: + # NOTE: If the network already exists, test will fail to start. + docker network create "${DMS_TEST_NETWORK}" + + # Setup local oauth2 provider service: + docker run --rm -d --name "${CONTAINER2_NAME}" \ + --hostname "${FQDN_OAUTH2}" \ + --network "${DMS_TEST_NETWORK}" \ + --volume "${REPOSITORY_ROOT}/test/config/oauth2/:/app/" \ + docker.io/library/python:latest \ + python /app/provider.py + + _run_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER2_NAME} 2>&1 | grep 'Starting server'" + + # + # Setup DMS container + # + + # Add OAUTH2 configuration so that Dovecot can reach out to our mock provider (CONTAINER2) + local ENV_OAUTH2_CONFIG=( + --env ENABLE_OAUTH2=1 + --env OAUTH2_INTROSPECTION_URL=http://oauth2.example.test/userinfo/ + ) + + export CONTAINER_NAME=${CONTAINER1_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + "${ENV_OAUTH2_CONFIG[@]}" + + --hostname "${FQDN_MAIL}" + --network "${DMS_TEST_NETWORK}" + ) + + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_tcp_port_in_container 143 + + # Set default implicit container fallback for helpers: + export CONTAINER_NAME=${CONTAINER1_NAME} +} + +function teardown_file() { + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" + docker network rm "${DMS_TEST_NETWORK}" +} + + +@test "oauth2: imap connect and authentication works" { + # An initial connection needs to be made first, otherwise the auth attempt fails + _run_in_container_bash 'nc -vz 0.0.0.0 143' + + _nc_wrapper 'auth/imap-oauth2-auth.txt' '-w 1 0.0.0.0 143' + assert_output --partial 'Examine completed' +} From 71e1102749680545f1ac6cb1a2863ea1714e989c Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 12 Jan 2024 23:48:14 +0100 Subject: [PATCH 114/125] Tiny #3480 follow up: Add missing ENABLE_OAUTH2 var (#3775) --- target/scripts/startup/variables-stack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index fc38a39d..0b351a9e 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -83,6 +83,7 @@ function __environment_variables_general_setup() { VARS[ENABLE_FETCHMAIL]="${ENABLE_FETCHMAIL:=0}" VARS[ENABLE_GETMAIL]="${ENABLE_GETMAIL:=0}" VARS[ENABLE_MANAGESIEVE]="${ENABLE_MANAGESIEVE:=0}" + VARS[ENABLE_OAUTH2]="${ENABLE_OAUTH2:=0}" VARS[ENABLE_OPENDKIM]="${ENABLE_OPENDKIM:=1}" VARS[ENABLE_OPENDMARC]="${ENABLE_OPENDMARC:=1}" VARS[ENABLE_POLICYD_SPF]="${ENABLE_POLICYD_SPF:=1}" From e3331b0f44b1d9c1c513389ad5b6f3b09e51dc16 Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Sat, 13 Jan 2024 09:37:20 +0100 Subject: [PATCH 115/125] feat: Add MTA-STS support for outbound mail (#3592) * feat: add support for MTA-STS for outgoing mails * Hook-up mta-sts-daemon into basic process handling test * fix: Call python script directly The python3 shebang will run it, which will now meet the expectations of the process testing via pgrep. fail2ban has the same approach. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 8 +++-- Dockerfile | 9 ++++++ docs/content/config/best-practices/mta-sts.md | 30 +++++++++++++++++++ docs/content/config/environment.md | 9 ++++++ docs/mkdocs.yml | 3 +- mailserver.env | 6 ++++ target/mta-sts-daemon/mta-sts-daemon.yml | 7 +++++ target/scripts/build/packages.sh | 2 +- target/scripts/start-mailserver.sh | 5 ++++ target/scripts/startup/daemons-stack.sh | 1 + target/scripts/startup/setup.d/mail_state.sh | 2 ++ target/scripts/startup/setup.d/mta-sts.sh | 7 +++++ target/supervisor/conf.d/supervisor-app.conf | 12 ++++++++ .../process_check_restart.bats | 4 +++ 14 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 docs/content/config/best-practices/mta-sts.md create mode 100644 target/mta-sts-daemon/mta-sts-daemon.yml create mode 100644 target/scripts/startup/setup.d/mta-sts.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index e3881941..df9c6987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,14 @@ All notable changes to this project will be documented in this file. The format ### Features - - **Authentication with OIDC / OAuth 2.0** 🎉 - - DMS now supports authentication via OAuth2 (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) from capable services (_like Roundcube_). +- **Authentication with OIDC / OAuth 2.0** 🎉 + - DMS now supports authentication via OAuth2 (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) from capable services (_like Roundcube_). - This does not replace the need for an `ACCOUNT_PROVISIONER` (`FILE` / `LDAP`), which is required for an account to receive or send mail. - Successful authentication (_via Dovecot PassDB_) still requires an existing account (_lookup via Dovecot UserDB_). +- **MTA-STS** (_Optional support for mandatory outgoing TLS encryption_) + - If enabled and the outbound recipient has an MTA-STS policy set, TLS is mandatory for delivering to that recipient. + - Enable via the ENV `ENABLE_MTA_STS=1` + - Supported by major email service providers like Gmail, Yahoo and Outlook. ### Updates diff --git a/Dockerfile b/Dockerfile index 2118622d..e822632a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -199,6 +199,15 @@ COPY target/opendmarc/opendmarc.conf /etc/opendmarc.conf COPY target/opendmarc/default-opendmarc /etc/default/opendmarc COPY target/opendmarc/ignore.hosts /etc/opendmarc/ignore.hosts +# -------------------------------------------------- +# --- postfix-mta-sts-daemon ----------------------- +# -------------------------------------------------- +COPY target/mta-sts-daemon/mta-sts-daemon.yml /etc/mta-sts-daemon.yml +RUN < DNS block lists are disabled - 1 => DNS block lists are enabled +##### ENABLE_MTA_STS + +Enables MTA-STS support for outbound mail. + +- **0** => Disabled +- 1 => Enabled + +See [MTA-STS](best-practices/mta-sts.md) for further explanation. + ##### ENABLE_OPENDKIM Enables the OpenDKIM service. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7df1acbe..aaaaf51b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -122,8 +122,9 @@ nav: - 'Environment Variables': config/environment.md - 'User Management': config/user-management.md - 'Best Practices': - - 'DKIM, DMARC & SPF': config/best-practices/dkim_dmarc_spf.md - 'Auto-discovery': config/best-practices/autodiscover.md + - 'DKIM, DMARC & SPF': config/best-practices/dkim_dmarc_spf.md + - 'MTA-STS': config/best-practices/mta-sts.md - 'Security': - 'Understanding the Ports': config/security/understanding-the-ports.md - 'SSL/TLS': config/security/ssl.md diff --git a/mailserver.env b/mailserver.env index c9793a99..49bc2cca 100644 --- a/mailserver.env +++ b/mailserver.env @@ -354,6 +354,12 @@ POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME=0 # Note: More details at http://www.postfix.org/postconf.5.html#inet_protocols POSTFIX_INET_PROTOCOLS=all +# Enables MTA-STS support for outbound mail. +# More details: https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-mta-sts/ +# - **0** ==> MTA-STS disabled +# - 1 => MTA-STS enabled +ENABLE_MTA_STS=0 + # Choose TCP/IP protocols for dovecot to use # **all** => Listen on all interfaces # ipv4 => Listen only on IPv4 interfaces. Most likely you want this behind Docker. diff --git a/target/mta-sts-daemon/mta-sts-daemon.yml b/target/mta-sts-daemon/mta-sts-daemon.yml new file mode 100644 index 00000000..4d5d5e55 --- /dev/null +++ b/target/mta-sts-daemon/mta-sts-daemon.yml @@ -0,0 +1,7 @@ +# Docs: https://github.com/Snawoot/postfix-mta-sts-resolver/blob/master/man/mta-sts-daemon.yml.5.adoc +path: /var/run/mta-sts/daemon.sock +mode: 0666 +cache: + type: sqlite + options: + filename: "/var/lib/mta-sts/cache.db" diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index e3607f48..e57cfe07 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -68,7 +68,7 @@ function _install_packages() { ) POSTFIX_PACKAGES=( - pflogsumm postgrey postfix-ldap + pflogsumm postgrey postfix-ldap postfix-mta-sts-resolver postfix-pcre postfix-policyd-spf-python postsrsd ) diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 1f352229..56dfa1fb 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -120,6 +120,11 @@ function _register_functions() { _register_setup_function '_setup_apply_fixes_after_configuration' _register_setup_function '_environment_variables_export' + if [[ ${ENABLE_MTA_STS} -eq 1 ]]; then + _register_setup_function '_setup_mta_sts' + _register_start_daemon '_start_daemon_mta_sts_daemon' + fi + # ? >> Daemons _register_start_daemon '_start_daemon_cron' diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index 5476fc9f..a4cecf67 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -38,6 +38,7 @@ function _start_daemon_opendkim { _default_start_daemon 'opendkim' ; function _start_daemon_opendmarc { _default_start_daemon 'opendmarc' ; } function _start_daemon_postgrey { _default_start_daemon 'postgrey' ; } function _start_daemon_postsrsd { _default_start_daemon 'postsrsd' ; } +function _start_daemon_mta_sts_daemon { _default_start_daemon 'mta-sts-daemon' ; } function _start_daemon_rspamd { _default_start_daemon 'rspamd' ; } function _start_daemon_rspamd_redis { _default_start_daemon 'rspamd-redis' ; } function _start_daemon_rsyslog { _default_start_daemon 'rsyslog' ; } diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 73c2515b..9963bbcc 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -24,6 +24,7 @@ function _setup_save_states() { [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') + [[ ${ENABLE_MTA_STS} -eq 1 ]] && SERVICEDIRS+=('lib/mta-sts') [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && SERVICEDIRS+=('lib/redis') @@ -84,6 +85,7 @@ function _setup_save_states() { [[ ${ENABLE_AMAVIS} -eq 1 ]] && chown -R amavis:amavis "${STATEDIR}/lib-amavis" [[ ${ENABLE_CLAMAV} -eq 1 ]] && chown -R clamav:clamav "${STATEDIR}/lib-clamav" [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && chown -R fetchmail:nogroup "${STATEDIR}/lib-fetchmail" + [[ ${ENABLE_MTA_STS} -eq 1 ]] && chown -R _mta-sts:_mta-sts "${STATEDIR}/lib-mta-sts" [[ ${ENABLE_POSTGREY} -eq 1 ]] && chown -R postgrey:postgrey "${STATEDIR}/lib-postgrey" [[ ${ENABLE_RSPAMD} -eq 1 ]] && chown -R _rspamd:_rspamd "${STATEDIR}/lib-rspamd" [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && chown -R redis:redis "${STATEDIR}/lib-redis" diff --git a/target/scripts/startup/setup.d/mta-sts.sh b/target/scripts/startup/setup.d/mta-sts.sh new file mode 100644 index 00000000..7d1f88ea --- /dev/null +++ b/target/scripts/startup/setup.d/mta-sts.sh @@ -0,0 +1,7 @@ +#!/bin/bash + + +function _setup_mta_sts() { + _log 'trace' 'Adding MTA-STS lookup to the Postfix TLS policy map' + _add_to_or_update_postfix_main smtp_tls_policy_maps 'socketmap:unix:/var/run/mta-sts/daemon.sock:postfix' +} diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 431357d8..d64d3d72 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -157,3 +157,15 @@ autostart=false stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/bin/bash -l -c /usr/local/bin/update-check.sh + +# Docs: https://github.com/Snawoot/postfix-mta-sts-resolver/blob/master/man/mta-sts-daemon.1.adoc +[program:mta-sts-daemon] +startsecs=0 +stopwaitsecs=55 +autostart=false +autorestart=true +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log +command=/usr/bin/mta-sts-daemon --config /etc/mta-sts-daemon.yml +user=_mta-sts +environment=HOME=/var/lib/mta-sts diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index b559d21d..4b01454e 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -21,6 +21,7 @@ function teardown() { _default_teardown ; } # dovecot (/usr/sbin/dovecot) # fetchmail (/usr/bin/fetchmail) # fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - Started by fail2ban-wrapper.sh +# mta-sts-daemon (/usr/bin/bin/python3 /usr/bin/mta-sts-daemon) # postgrey (postgrey) - NOTE: This process lacks path information to match with `--full` in pgrep / pkill # postsrsd (/usr/sbin/postsrsd) - NOTE: Also matches the wrapper: `/bin/bash /usr/local/bin/postsrsd-wrapper.sh` # saslauthd (/usr/sbin/saslauthd) - x5 of the same process are found running (1 is a parent of 4) @@ -44,6 +45,7 @@ ENV_PROCESS_LIST=( dovecot fail2ban-server fetchmail + mta-sts-daemon opendkim opendmarc postgrey @@ -58,6 +60,7 @@ ENV_PROCESS_LIST=( --env ENABLE_CLAMAV=0 --env ENABLE_FAIL2BAN=0 --env ENABLE_FETCHMAIL=0 + --env ENABLE_MTA_STS=0 --env ENABLE_OPENDKIM=0 --env ENABLE_OPENDMARC=0 --env ENABLE_POSTGREY=0 @@ -93,6 +96,7 @@ ENV_PROCESS_LIST=( --env ENABLE_AMAVIS=1 --env ENABLE_FAIL2BAN=1 --env ENABLE_FETCHMAIL=1 + --env ENABLE_MTA_STS=1 --env ENABLE_OPENDKIM=1 --env ENABLE_OPENDMARC=1 --env FETCHMAIL_PARALLEL=1 From f794f65caa5479763fe9ce149b175d08f9f37b18 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 14 Jan 2024 14:08:27 +0100 Subject: [PATCH 116/125] docs: updated `CONTRIBUTORS.md` (#3777) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 345 ++++++++++++++++++++++++------------------------ 1 file changed, 176 insertions(+), 169 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 05a4e6e4..53254c78 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -321,46 +321,10 @@ Thanks goes to these wonderful people ✨ - - egavard + + kiliant
- egavard - - - - - mathuin -
- mathuin - - - - - dashohoxha -
- dashohoxha - - - - - jamebus -
- jamebus - - - - - - lukecyca -
- lukecyca - - - - - okainov -
- okainov + kiliant @@ -371,53 +335,68 @@ Thanks goes to these wonderful people ✨ - - kiliant + + okainov
- kiliant + okainov - - m-schmoock + + lukecyca
- m-schmoock - - - - - mjung -
- mjung + lukecyca - - VanVan + + jsonn
- VanVan + jsonn - - andreasgerstmayr + + jamebus
- andreasgerstmayr + jamebus - - davidszp + + dashohoxha
- davidszp + dashohoxha - - kamuri + + mathuin
- kamuri + mathuin + + + + + egavard +
+ egavard + + + + + weo +
+ weo + + + + + + Zehir +
+ Zehir @@ -428,18 +407,46 @@ Thanks goes to these wonderful people ✨ - - Zehir + + kamuri
- Zehir + kamuri + + + + + davidszp +
+ davidszp + + + + + andreasgerstmayr +
+ andreasgerstmayr + + + + + VanVan +
+ VanVan - - weo + + mjung
- weo + mjung + + + + + m-schmoock +
+ m-schmoock @@ -469,15 +476,15 @@ Thanks goes to these wonderful people ✨
Starbix - + + citec
citec - - + yajo @@ -492,13 +499,6 @@ Thanks goes to these wonderful people ✨ analogue - - - MakerMatrix -
- MakerMatrix - - Rubytastic2 @@ -514,10 +514,10 @@ Thanks goes to these wonderful people ✨ - - jsonn + + MakerMatrix
- jsonn + MakerMatrix @@ -642,26 +642,33 @@ Thanks goes to these wonderful people ✨ yogo1212 - - - mpanneck -
- mpanneck - - - willtho89
willtho89 + + + + + mpanneck +
+ mpanneck + - - ubenmackin + + aminvakil
- ubenmackin + aminvakil + + + + + elbracht +
+ elbracht @@ -679,17 +686,10 @@ Thanks goes to these wonderful people ✨ - - aminvakil + + ubenmackin
- aminvakil - - - - - elbracht -
- elbracht + ubenmackin @@ -780,10 +780,17 @@ Thanks goes to these wonderful people ✨ - - fl42 + + jamesfryer
- fl42 + jamesfryer + + + + + eltociear +
+ eltociear @@ -801,17 +808,10 @@ Thanks goes to these wonderful people ✨ - - jamesfryer + + fl42
- jamesfryer - - - - - eltociear -
- eltociear + fl42 @@ -944,10 +944,10 @@ Thanks goes to these wonderful people ✨ - - 0xflotus + + nilshoell
- 0xflotus + nilshoell @@ -1388,6 +1388,13 @@ Thanks goes to these wonderful people ✨ mchamplain + + + 0xflotus +
+ 0xflotus + + auchri @@ -1415,15 +1422,15 @@ Thanks goes to these wonderful people ✨
damianmoore - + + espitall
espitall - - + dkarski @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
mazzz1y - + + aydodo
aydodo - - + vedtam @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
ErikEngerd - + + huncode
huncode - - + felixn @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
20th - + + 2b
2b - - + askz @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
alexanderneu - + + ch3sh1r
ch3sh1r - - + eglia @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
MrFreezeex - + + arunvc
arunvc - - + astrocket @@ -1673,15 +1680,22 @@ Thanks goes to these wonderful people ✨
crash7 - + + fkefer
fkefer - - + + + + KCrawley +
+ KCrawley + + khuedoan @@ -1709,7 +1723,8 @@ Thanks goes to these wonderful people ✨
linhandev - + + luke- @@ -1723,8 +1738,7 @@ Thanks goes to these wonderful people ✨
LucidityCrash - - + MadsRC @@ -1752,7 +1766,8 @@ Thanks goes to these wonderful people ✨
dragetd - + + michaeljensen @@ -1766,8 +1781,7 @@ Thanks goes to these wonderful people ✨
exhuma - - + milas @@ -1795,7 +1809,8 @@ Thanks goes to these wonderful people ✨
mpldr - + + naveensrinivasan @@ -1809,8 +1824,7 @@ Thanks goes to these wonderful people ✨
neuralp - - + radicand @@ -1818,13 +1832,6 @@ Thanks goes to these wonderful people ✨ radicand - - - nilshoell -
- nilshoell - - frugan-dev @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
glandais - + + GiovanH
GiovanH - - + harryyoud @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
jcalfee - + + mivek
mivek - - + init-js @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
jmccl - + + jurekbarth
jurekbarth - - + JOduMonT @@ -1962,10 +1969,10 @@ Thanks goes to these wonderful people ✨ - - KCrawley + + thechubbypanda
- KCrawley + thechubbypanda From 3a142f97264eff31e3d72a3ce7420ccdb8e3aa4d Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:14:03 +0100 Subject: [PATCH 117/125] tests: small adjustments (#3772) --- CHANGELOG.md | 2 +- test/files/emails/amavis/spam.txt | 6 ------ test/helper/sending.bash | 15 +++++++++++---- .../parallel/set1/spam_virus/rspamd_full.bats | 18 ++++++++---------- .../set1/spam_virus/spam_junk_folder.bats | 2 +- .../tests/parallel/set3/mta/smtp_delivery.bats | 2 +- 6 files changed, 22 insertions(+), 23 deletions(-) delete mode 100644 test/files/emails/amavis/spam.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index df9c6987..0ade3c9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Tests**: - - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747)): + - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747) & [#3772](https://github.com/docker-mailserver/docker-mailserver/pull/3772)): - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. - `swaks` version is now the latest from Github releases instead of the Debian package. - `_nc_wrapper`, `_send_mail` and related helpers expect the `.txt` filepath extension again. diff --git a/test/files/emails/amavis/spam.txt b/test/files/emails/amavis/spam.txt deleted file mode 100644 index e8d26138..00000000 --- a/test/files/emails/amavis/spam.txt +++ /dev/null @@ -1,6 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message amavis/spam.txt -This is a test mail. -XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X diff --git a/test/helper/sending.bash b/test/helper/sending.bash index 529e36aa..e18dc1ac 100644 --- a/test/helper/sending.bash +++ b/test/helper/sending.bash @@ -141,14 +141,12 @@ function _send_email_and_get_id() { # Get rid of ${1} so only the arguments for swaks remain shift 1 - local QUEUE_ID # The unique ID Postfix (and other services) use may be different in length - # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a - # range to safely capture it. + # on different systems. Hence, we use a range to safely capture it. local QUEUE_ID_REGEX='[A-Z0-9]{9,12}' _wait_for_empty_mail_queue_in_container - local OUTPUT=$(_send_email "${@}" --header "Message-Id: ${MID}") + _send_email "${@}" --header "Message-Id: ${MID}" _wait_for_empty_mail_queue_in_container # We store Postfix's queue ID first @@ -164,3 +162,12 @@ function _send_email_and_get_id() { run echo "${ID_ENV_VAR_REF}" assert_line --regexp "^${QUEUE_ID_REGEX}\|${MID}$" } + +# Send a spam e-mail by utilizing GTUBE. +# +# Extra arguments given to this function will be supplied by `_send_email_and_get_id` directly. +function _send_spam() { + _send_email_and_get_id MAIL_ID_SPAM "${@}" \ + --from 'spam@external.tld' \ + --body 'XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' +} diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index b83ac353..f66e9231 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -43,20 +43,18 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - # ref: https://rspamd.com/doc/gtube_patterns.html - local GTUBE_SUFFIX='*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' - # We will send 4 emails: - # 1. The first one should pass just fine + # 1. The first one should pass just fine _send_email_and_get_id MAIL_ID_PASS - # 2. The second one should be rejected (GTUBE pattern) - _send_email_and_get_id MAIL_ID_REJECT --expect-rejection --body "XJS${GTUBE_SUFFIX}" - # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) + # 2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection) + _send_spam --expect-rejection + # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) # shellcheck disable=SC2016 _send_email_and_get_id MAIL_ID_VIRUS --expect-rejection \ --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' - # 4. The fourth one will receive an added header (GTUBE pattern) - _send_email_and_get_id MAIL_ID_HEADER --body "YJS${GTUBE_SUFFIX}" + # 4. The fourth one will receive an added header (Rspamd-specific GTUBE pattern for adding a spam header) + # ref: https://rspamd.com/doc/gtube_patterns.html + _send_email_and_get_id MAIL_ID_HEADER --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" _run_in_container cat /var/log/mail.log assert_success @@ -122,7 +120,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'S \(reject\)' _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID_REJECT}" + _print_mail_log_for_id "${MAIL_ID_SPAM}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 Gtube pattern' diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 2c3a522d..15ec4fe1 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -95,7 +95,7 @@ function teardown() { _default_teardown ; } function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis - _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' + _send_spam } function _should_be_received_by_amavis() { diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index 329f36b2..e851d94e 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -81,7 +81,7 @@ function setup_file() { _send_email --to bounce-always@localhost.localdomain _send_email --to alias2@localhost.localdomain # Required for 'rejects spam': - _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' + _send_spam # Required for 'delivers mail to existing account': _send_email --header 'Subject: Test Message existing-user1' From 1449629479ebf9dbd1e63fc78b0d7b9c1b6cd2ff Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 15 Jan 2024 11:23:23 +1300 Subject: [PATCH 118/125] fix: Revert quoting `SA_SPAM_SUBJECT` in `mailserver.env` (#3767) In Docker Compose `.env` files are parsed properly when values are wrapped with quotes. Trailing white-space is also discarded, like it would be with shell variables. This is not the case with `docker run` or other CRI like `podman` (_including it's compose equivalent support_). Those will parse the quotes to be included in a literal string value. Trailing white-space is also retained. Hence a default with a trailing space is not compatible across CRI. This change documents the default with additional context on how to include a trailing white-space with a custom value for the users CRI choice. It additionally clearly communicates the opt-out value for this feature. --- mailserver.env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mailserver.env b/mailserver.env index 49bc2cca..9b085c9f 100644 --- a/mailserver.env +++ b/mailserver.env @@ -401,7 +401,10 @@ SA_TAG2=6.31 SA_KILL=10.0 # add tag to subject if spam detected -SA_SPAM_SUBJECT='***SPAM*** ' +# The value `undef` opts-out of this feature. The value shown below is the default. +# NOTE: By default spam is delivered to a junk folder, reducing the value of adding a subject prefix. +# NOTE: If not using Docker Compose, other CRI may require the single quotes removed. +#SA_SPAM_SUBJECT='***SPAM*** ' # ----------------------------------------------- # --- Fetchmail Section ------------------------- From ce6ebcc0219eca96ead8d758973f14d94dbb7efe Mon Sep 17 00:00:00 2001 From: Den Date: Mon, 15 Jan 2024 02:10:03 +0200 Subject: [PATCH 119/125] docs: Rspamd DKIM config simplify via `path` setting (#3702) docs: Rspamd DKIM config (`dkim_signing.conf`) example has been simplified via `path` + `selector` settings. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/best-practices/dkim_dmarc_spf.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index da496aa5..ed56504c 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -156,6 +156,13 @@ DKIM is currently supported by either OpenDKIM or Rspamd: use_esld = true; check_pubkey = true; # you want to use this in the beginning + selector = "mail"; + # The path location is searched for a DKIM key with these variables: + # - `$domain` is sourced from the MIME mail message `From` header + # - `$selector` is configured for `mail` (as a default fallback) + path = "/tmp/docker-mailserver/dkim/keys/$domain/$selector.private"; + + # domain specific configurations can be provided below: domain { example.com { path = "/tmp/docker-mailserver/rspamd/dkim/mail.private"; From 265440b2bbc77971b579bd00ad92a89e8c02d9f3 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 15 Jan 2024 22:34:15 +1300 Subject: [PATCH 120/125] fix: Ensure `.svbin` files are newer than `.sieve` source files (#3779) --- CHANGELOG.md | 3 ++- target/scripts/startup/setup-stack.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ade3c9b..31c4819f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,10 @@ All notable changes to this project will be documented in this file. The format - **Docs:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) - ### Fixes +- **Dovecot:** + - During container startup for Dovecot Sieve, `.sievec` source files compiled to `.svbin` now have their `mtime` adjusted post setup to ensure it is always older than the associated `.svbin` file. This avoids superfluous error logs for sieve scripts that don't actually need to be compiled again ([#3779](https://github.com/docker-mailserver/docker-mailserver/pull/3779)) - **Internal:** - `.gitattributes`: Always use LF line endings on checkout for files with shell script content ([#3755](https://github.com/docker-mailserver/docker-mailserver/pull/3755)) - Fix missing 'jaq' binary for ARM architecture ([#3766](https://github.com/docker-mailserver/docker-mailserver/pull/3766)) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 060dadb2..f55cb548 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -20,6 +20,16 @@ function _setup() { ${FUNC} done + _setup_post +} + +function _setup_post() { + # Dovecot `.svbin` files must have a newer mtime than their `.sieve` source files, + # Modifications during setup to these files sometimes results in a common mtime value. + # Handled during post-setup as setup of Dovecot Sieve scripts is not centralized. + find /usr/lib/dovecot/ -iname '*.sieve' -exec touch -d '2 seconds ago' {} + + find /usr/lib/dovecot/ -iname '*.svbin' -exec touch -d '1 seconds ago' {} + + # All startup modifications to configs should have taken place before calling this: _prepare_for_change_detection } From 2bf52342500726e86362ee1ea5954b7d96533d32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 10:18:13 +1300 Subject: [PATCH 121/125] chore(deps): Bump anchore/scan-action from 3.4.0 to 3.5.0 (#3782) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index d7cf6932..896ee80e 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.4.0 + uses: anchore/scan-action@v3.5.0 id: scan with: image: mailserver-testing:ci From 068ceb1d1aaba8b682bf0be34dbc2a134d823643 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:38:08 +0100 Subject: [PATCH 122/125] docs: misc improvements (#3773) * correct misc typos We also seem to be favoring `behavior` over `behaviour`. * bump MkDocs version * resolve errors shown when buildg docs * improve the Rspamd page * behaviour -> behavior Streamline the usage of this word. The majority used behavior, so I opted to go with this way of spelling it. * Apply suggestions from code review --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/scripts/docs/build-docs.sh | 2 +- CHANGELOG.md | 1 + docs/content/config/advanced/auth-ldap.md | 2 +- docs/content/config/advanced/ipv6.md | 6 +- .../config/advanced/optional-config.md | 2 +- docs/content/config/advanced/podman.md | 2 +- docs/content/config/debugging.md | 4 +- docs/content/config/environment.md | 4 +- docs/content/config/security/rspamd.md | 61 ++++++++++++++++--- .../examples/use-cases/imap-folders.md | 2 +- docs/content/faq.md | 6 +- docs/content/index.md | 10 +-- docs/mkdocs.yml | 4 +- 13 files changed, 77 insertions(+), 29 deletions(-) diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index aff66ab5..5d1cab52 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -10,7 +10,7 @@ docker run \ --user "$(id -u):$(id -g)" \ --volume "${PWD}:/docs" \ --name "build-docs" \ - squidfunk/mkdocs-material:9.2 build --strict + squidfunk/mkdocs-material:9.5 build --strict # Remove unnecessary build artifacts: https://github.com/squidfunk/mkdocs-material/issues/2519 # site/ is the build output folder. diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c4819f..b871f0ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file. The format - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) - **Docs:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) + - Detailed how mail received is assigned a spam score by Rspamd and processed accordingly ([#3773](https://github.com/docker-mailserver/docker-mailserver/pull/3773)) ### Fixes diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index c275fd6c..ea24526d 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -56,7 +56,7 @@ These variables specify the LDAP filters that dovecot uses to determine if a use This is split into the following two lookups, both using `%u` as the placeholder for the full login name ([see dovecot documentation for a full list of placeholders](https://doc.dovecot.org/configuration_manual/config_file/config_variables/)). Usually you only need to set `DOVECOT_USER_FILTER`, in which case it will be used for both filters. - `DOVECOT_USER_FILTER` is used to get the account details (uid, gid, home directory, quota, ...) of a user. -- `DOVECOT_PASS_FILTER` is used to get the password information of the user, and is in pretty much all cases identical to `DOVECOT_USER_FILTER` (which is the default behaviour if left away). +- `DOVECOT_PASS_FILTER` is used to get the password information of the user, and is in pretty much all cases identical to `DOVECOT_USER_FILTER` (which is the default behavior if left away). If your directory doesn't have the [postfix-book schema](https://github.com/variablenix/ldap-mail-schema/blob/master/postfix-book.schema) installed, then you must change the internal attribute handling for dovecot. For this you have to change the `pass_attr` and the `user_attr` mapping, as shown in the example below: diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index f035523a..77dfb0c9 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -60,7 +60,7 @@ Enable `ip6tables` support so that Docker will manage IPv6 networking rules as w ``` - `experimental: true` is currently required for `ip6tables: true` to work. - - `userland-proxy` setting [can potentially affect connection behaviour][gh-pull-3244-proxy] for local connections. + - `userland-proxy` setting [can potentially affect connection behavior][gh-pull-3244-proxy] for local connections. Now restart the daemon if it's running: `systemctl restart docker`. @@ -99,7 +99,7 @@ Next, configure a network with an IPv6 subnet for your container with any of the ??? tip "Override the implicit `default` network" - You can optionally avoid the service assignment by [overriding the `default` user-defined network that Docker Compose generates](docker-docs-network-compose-default). Just replace `dms-ipv6` with `default`. + You can optionally avoid the service assignment by [overriding the `default` user-defined network that Docker Compose generates][docker-docs-network-compose-default]. Just replace `dms-ipv6` with `default`. The Docker Compose `default` bridge is not affected by settings for the default `bridge` (aka `docker0`) in `/etc/docker/daemon.json`. @@ -180,7 +180,7 @@ curl --max-time 5 http://[2001:db8::1]:80 !!! info "IPv6 ULA address priority" DNS lookups that have records for both IPv4 and IPv6 addresses (_eg: `localhost`_) may prefer IPv4 over IPv6 (ULA) for private addresses, whereas for public addresses IPv6 has priority. This shouldn't be anything to worry about, but can come across as a surprise when testing your IPv6 setup on the same host instead of from a remote client. - + The preference can be controlled with [`/etc/gai.conf`][networking-gai], and appears was configured this way based on [the assumption that IPv6 ULA would never be used with NAT][networking-gai-blog]. It should only affect the destination resolved for outgoing connections, which for IPv6 ULA should only really affect connections between your containers / host. In future [IPv6 ULA may also be prioritized][networking-gai-rfc]. [docker-subnets]: https://straz.to/2021-09-08-docker-address-pools/#what-are-the-default-address-pools-when-no-configuration-is-given-vanilla-pools diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 21f82a3b..31090050 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -35,7 +35,7 @@ This is a list of all configuration files and directories which are optional or - **whitelist_clients.local:** Whitelisted domains, not considered by postgrey. Enter one host or domain per line. - **spamassassin-rules.cf:** Anti-spam rules for Spamassassin. (Docs: [FAQ - SpamAssassin Rules][docs-faq-spamrules]) - **fail2ban-fail2ban.cf:** Additional config options for `fail2ban.cf`. (Docs: [Fail2Ban][docs-fail2ban]) -- **fail2ban-jail.cf:** Additional config options for fail2ban's jail behaviour. (Docs: [Fail2Ban][docs-fail2ban]) +- **fail2ban-jail.cf:** Additional config options for fail2ban's jail behavior. (Docs: [Fail2Ban][docs-fail2ban]) - **amavis.cf:** replaces the `/etc/amavis/conf.d/50-user` file - **dovecot.cf:** replaces `/etc/dovecot/local.conf`. (Docs: [Override Dovecot Defaults][docs-override-dovecot]) - **dovecot-quotas.cf:** list of custom quotas per mailbox. (Docs: [Accounts][docs-accounts-quota]) diff --git a/docs/content/config/advanced/podman.md b/docs/content/config/advanced/podman.md index b32288c1..4cb60e77 100644 --- a/docs/content/config/advanced/podman.md +++ b/docs/content/config/advanced/podman.md @@ -107,7 +107,7 @@ The `PERMIT_DOCKER` variable in the `mailserver.env` file allows to specify trus #### Use the slip4netns network driver The second workaround is slightly more complicated because the `compose.yaml` has to be modified. -As shown in the [fail2ban section](../../security/fail2ban/#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. +As shown in the [fail2ban section](../security/fail2ban.md#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. This network driver enables podman to correctly resolve IP addresses but it is not compatible with user defined networks which might be a problem depending on your setup. diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index d58430e1..2c89948d 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -107,9 +107,9 @@ This could be from outdated software, or running a system that isn't able to pro - **macOS:** DMS has limited support for macOS. Often an issue encountered is due to permissions related to the `volumes` config in `compose.yaml`. You may have luck [trying `gRPC FUSE`][gh-macos-support] as the file sharing implementation; [`VirtioFS` is the successor][docker-macos-virtiofs] but presently appears incompatible with DMS. - **Kernel:** Some systems provide [kernels with modifications (_replacing defaults and backporting patches_)][network::kernels-modified] to support running legacy software or kernels, complicating compatibility. This can be commonly experienced with products like NAS. -- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. +- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behavior of your DMS container, even with the latest Docker Engine installed. - **Container runtime:** Docker and Podman for example have subtle differences. DMS docs are primarily focused on Docker, but we try to document known issues where relevant. -- **Rootless containers:** Introduces additional differences in behaviour or requirements: +- **Rootless containers:** Introduces additional differences in behavior or requirements: - cgroup v2 is required for supporting rootless containers. - Differences such as for container networking which may further affect support for IPv6 and preserving the client IP (Remote address). Example with Docker rootless are [binding a port to a specific interface][docker-rootless-interface] and the choice of [port forwarding driver][docs-rootless-portdriver]. diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 7ae96fcd..1aa6799d 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -560,7 +560,7 @@ Changes the interval in which log files are rotated. The Amavis action configured by this setting: - - Influences the behaviour of the [`SA_KILL`](#sa_kill) setting. + - Influences the behavior of the [`SA_KILL`](#sa_kill) setting. - Applies to the Amavis config parameters `$final_spam_destiny` and `$final_bad_header_destiny`. !!! note "This ENV setting is related to" @@ -596,7 +596,7 @@ Mail is not yet considered spam at this spam score, but for purposes like diagno ``` !!! info "The `X-Spam-Score` is `4.162`" - + High enough for `SA_TAG` to trigger adding these headers, but not high enough for `SA_TAG2` (_which would set `X-Spam-Flag: YES` instead_). ##### SA_TAG2 diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index fe9bd5ea..283c2a95 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -22,15 +22,59 @@ The following environment variables are related to Rspamd: 5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -8. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +8. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] 9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) -With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. +With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd. + +[docs-spam-to-junk]: ../environment.md#move_spam_to_junk ## The Default Configuration ### Mode of Operation +!!! tip "Attention" + + Read this section carefully if you want to understand how Rspamd is integrated into DMS and how it works (on a surface level). + +Rspamd is integrated as a milter into DMS. When enabled, Postfix's `main.cf` configuration file includes the parameter `rspamd_milter = inet:localhost:11332`, which is added to `smtpd_milters`. As a milter, Rspamd can inspect incoming and outgoing e-mails. + +Each mail is assigned what Rspamd calls symbols: when an e-mail matches a specific criterion, the mail receives a symbol. Afterwards, Rspamd applies a _spam score_ (as usual with anti-spam software) to the e-mail. + +- The score itself is calculated by adding the values of the individual symbols applied earlier. The higher the spam score is, the more likely the e-mail is spam. +- Symbol values can be negative (i.e., these symbols indicate the mail is legitimate, maybe because [SPF and DKIM][docs-dkim-dmarc-spf] are verified successfully) or the symbol can be positive (i.e., these symbols indicate the e-mail is spam, maybe because the e-mail contains a lot of links). + +Rspamd then adds (a few) headers to the e-mail based on the spam score. Most important are `X-Spamd-Result`, which contains an overview of which symbols were applied. It could look like this: + +```txt +X-Spamd-Result default: False [-2.80 / 11.00]; R_SPF_NA(1.50)[no SPF record]; R_DKIM_ALLOW(-1.00)[example.com:s=dtag1]; DWL_DNSWL_LOW(-1.00)[example.com:dkim]; RWL_AMI_LASTHOP(-1.00)[192.0.2.42:from]; DMARC_POLICY_ALLOW(-1.00)[example.com,none]; RWL_MAILSPIKE_EXCELLENT(-0.40)[192.0.2.42:from]; FORGED_SENDER(0.30)[noreply@example.com,some-reply-address@bounce.example.com]; RCVD_IN_DNSWL_LOW(-0.10)[192.0.2.42:from]; MIME_GOOD(-0.10)[multipart/mixed,multipart/related,multipart/alternative,text/plain]; MIME_TRACE(0.00)[0:+,1:+,2:+,3:+,4:~,5:~,6:~]; RCVD_COUNT_THREE(0.00)[3]; RCPT_COUNT_ONE(0.00)[1]; REPLYTO_DN_EQ_FROM_DN(0.00)[]; ARC_NA(0.00)[]; TO_MATCH_ENVRCPT_ALL(0.00)[]; RCVD_TLS_LAST(0.00)[]; DKIM_TRACE(0.00)[example.com:+]; HAS_ATTACHMENT(0.00)[]; TO_DN_NONE(0.00)[]; FROM_NEQ_ENVFROM(0.00)[noreply@example.com,some-reply-address@bounce.example.com]; FROM_HAS_DN(0.00)[]; REPLYTO_DOM_NEQ_FROM_DOM(0.00)[]; PREVIOUSLY_DELIVERED(0.00)[receiver@anotherexample.com]; ASN(0.00)[asn:3320, ipnet:192.0.2.0/24, country:DE]; MID_RHS_MATCH_FROM(0.00)[]; MISSING_XM_UA(0.00)[]; HAS_REPLYTO(0.00)[some-reply-address@dms-reply.example.com] +``` + +And then there is a corresponding `X-Rspamd-Action` header, which shows the overall result and the action that is taken. In our example, it would be: + +```txt +X-Rspamd-Action no action +``` + +Since the score is `-2.80`, nothing will happen and the e-mail is not classified as spam. Our custom [`actions.conf`][rspamd-actions-config] defines what to do at certain scores: + +1. At a score of 4, the e-mail is to be _greylisted_; +2. At a score of 6, the e-mail is _marked with a header_ (`X-Spam: Yes`); +3. At a score of 7, the e-mail will additionally have their _subject re-written_ (appending a prefix like `[SPAM]`); +4. At a score of 11, the e-mail is outright _rejected_. + +--- + +There is more to spam analysis than meets the eye: we have not covered the [Bayes training and filters][rspamc-docs-bayes] here, nor have we talked about [Sieve rules for e-mails that are marked as spam][docs-spam-to-junk]. + +Even the calculation of the score with the individual symbols has been presented to you in a simplified manner. But with the knowledge from above, you're equipped to read on and use Rspamd confidently. Keep on reading to understand the integration even better - you will want to know about your anti-spam software, not only to keep the bad e-mail out, but also to make sure the good e-mail arrive properly! + +[docs-dkim-dmarc-spf]: ../best-practices/dkim_dmarc_spf.md +[rspamd-actions-config]: https://github.com/docker-mailserver/docker-mailserver/blob/master/target/rspamd/local.d/actions.conf +[rspamc-docs-bayes]: https://rspamd.com/doc/configuration/statistic.html + +### Workers + The proxy worker operates in [self-scan mode][rspamd-docs-proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). DMS does not set a default password for the controller worker. You may want to do that yourself. In setups where you already have an authentication provider in front of the Rspamd webpage, you may want to [set the `secure_ip ` option to `"0.0.0.0/0"` for the controller worker](#with-the-help-of-a-custom-file) to disable password authentication inside Rspamd completely. @@ -39,9 +83,9 @@ DMS does not set a default password for the controller worker. You may want to d ### Persistence with Redis -When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist it's data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. +When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. -Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS rspamd config_). +Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). ### Web Interface @@ -77,13 +121,13 @@ You can find a list of all Rspamd modules [on their website][rspamd-docs-modules #### Disabled By Default -DMS disables certain modules (clickhouse, elastic, neural, reputation, spamassassin, url_redirector, metric_exporter) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. +DMS disables certain modules (`clickhouse`, `elastic`, `neural`, `reputation`, `spamassassin`, `url_redirector`, `metric_exporter`) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. #### Anti-Virus (ClamAV) You can choose to enable ClamAV, and Rspamd will then use it to check for viruses. Just set the environment variable `ENABLE_CLAMAV=1`. -#### RBLs (Realtime Blacklists) / DNSBLs (DNS-based Blacklists) +#### RBLs (Real-time Blacklists) / DNSBLs (DNS-based Blacklists) The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. As a consequence, Rspamd will perform DNS lookups to a variety of blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][rbl-vs-dnsbl]\]. @@ -105,12 +149,15 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. +!!! question "What is the [`local.d` directory and how does it compare to `override.d`][rspamd-docs-config-directories]?" + !!! warning "Clashing Overrides" Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. [rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory +[rspamd-docs-config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories ### With the Help of a Custom File @@ -127,7 +174,7 @@ where `COMMAND` can be: 3. `set-option-for-module`: sets the value for option `ARGUMENT2` to `ARGUMENT3` inside module `ARGUMENT1` 4. `set-option-for-controller`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the controller worker 5. `set-option-for-proxy`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the proxy worker -6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behaviour][rspamd-docs-basic-options] to value `ARGUMENT2` +6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][rspamd-docs-basic-options] to value `ARGUMENT2` 7. `add-line`: this will add the complete line after `ARGUMENT1` (with all characters) to the file `/etc/rspamd/override.d/` !!! example "An Example Is [Shown Down Below](#adjusting-and-extending-the-very-basic-configuration)" diff --git a/docs/content/examples/use-cases/imap-folders.md b/docs/content/examples/use-cases/imap-folders.md index ad5f3ed3..ce90355a 100644 --- a/docs/content/examples/use-cases/imap-folders.md +++ b/docs/content/examples/use-cases/imap-folders.md @@ -61,7 +61,7 @@ Take care to test localized names work well as well. This information is provided by the community. - It presently lacks references to confirm the behaviour. If any information is incorrect please let us know! :smile: + It presently lacks references to confirm the behavior. If any information is incorrect please let us know! :smile: [docs-config-overrides-dovecot]: ../../config/advanced/override-defaults/dovecot.md#override-configuration diff --git a/docs/content/faq.md b/docs/content/faq.md index f666b102..0985c0a9 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -12,7 +12,7 @@ Mails are stored in `/var/mail/${domain}/${username}`. Since `v9.0.0` it is poss ### How are IMAP mailboxes (_aka IMAP Folders_) set up? -`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](../examples/use-cases/imap-folders) for more information. +`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](./examples/use-cases/imap-folders.md) for more information. ### How do I update DMS? @@ -128,7 +128,7 @@ find "${PWD}/docker-data/dms-backups/" -type f -mtime +30 -delete ### I Want to Know More About the Ports -See [this part of the documentation](../config/security/understanding-the-ports/) for further details and best practice advice, **especially regarding security concerns**. +See [this part of the documentation](./config/security/understanding-the-ports.md) for further details and best practice advice, **especially regarding security concerns**. ### How can I configure my email client? @@ -210,7 +210,7 @@ baduser@example.com devnull ### What kind of SSL certificates can I use? -Both RSA and ECDSA certs are supported. You can provide your own cert files manually, or mount a `letsencrypt` generated directory (_with alternative support for Traefik's `acme.json`_). Check out the [`SSL_TYPE` documentation](../config/environment/#ssl_type) for more details. +Both RSA and ECDSA certs are supported. You can provide your own cert files manually, or mount a `letsencrypt` generated directory (_with alternative support for Traefik's `acme.json`_). Check out the [`SSL_TYPE` documentation](./config/environment.md#ssl_type) for more details. ### I just moved from my old mail server to DMS, but "it doesn't work"? diff --git a/docs/content/index.md b/docs/content/index.md index ff1214b1..3739423c 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -24,9 +24,9 @@ If you're completely new to mail servers or you want to read up on them, check o There is also a script - [`setup.sh`][github-file-setupsh] - supplied with this project. It supports you in configuring and administrating your server. Information on how to get it and how to use it is available [on a dedicated page][docs-setupsh]. -[docs-introduction]: ./introduction/ -[docs-usage]: ./usage/ -[docs-examples]: ./examples/tutorials/basic-installation/ +[docs-introduction]: ./introduction.md +[docs-usage]: ./usage.md +[docs-examples]: ./examples/tutorials/basic-installation.md [github-file-setupsh]: https://github.com/docker-mailserver/docker-mailserver/blob/master/setup.sh [docs-setupsh]: ./config/setup.sh/ @@ -59,10 +59,10 @@ You might also want to check out: DMS employs a variety of tests. If you want to know more about our test suite, view our [testing docs][docs-tests]. -[docs-tests]: ./contributing/tests/ +[docs-tests]: ./contributing/tests.md ### Contributing We are always happy to welcome new contributors. For guidelines and entrypoints please have a look at the [Contributing section][docs-contributing]. -[docs-contributing]: ./contributing/issues-and-pull-requests/ +[docs-contributing]: ./contributing/issues-and-pull-requests.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index aaaaf51b..8a6a24b0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -86,8 +86,8 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.tilde - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: extend_pygments_lang: - name: yml From 2d59aac5a1f3058287e7563884917cbcca9a5c53 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:54:27 +1300 Subject: [PATCH 123/125] chore: Add maintenance comment for `sed` usage (#3789) This is a more explicit reminder for any future contributors that get thrown off by the usage of `sed` here and may be inclined to change it. Add a link to reference a comment where it's already been explored what the alternative `sed` invocations available are. --- target/scripts/startup/setup.d/security/misc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 170f46fb..36d8905f 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -70,6 +70,8 @@ function __setup__security__spamassassin() { if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring SpamAssassin' + # Maintainers should take care in attempting to change these sed commands. Alternatives were already explored: + # https://github.com/docker-mailserver/docker-mailserver/pull/3767#issuecomment-1885989591 # shellcheck disable=SC2016 sed -i -r 's|^\$sa_tag_level_deflt (.*);|\$sa_tag_level_deflt = '"${SA_TAG}"';|g' /etc/amavis/conf.d/20-debian_defaults From 437114c5dd7928037deea1a2e31535b5a20746ad Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:46:22 +1300 Subject: [PATCH 124/125] tests: Revise `process_check_restart.bats` (#3780) --- CHANGELOG.md | 1 + .../process_check_restart.bats | 37 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b871f0ab..cfdfd314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Tests**: + - Revised testing of service process management (supervisord) to be more robust ([#3780](https://github.com/docker-mailserver/docker-mailserver/pull/3780)) - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747) & [#3772](https://github.com/docker-mailserver/docker-mailserver/pull/3772)): - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. - `swaks` version is now the latest from Github releases instead of the Debian package. diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 4b01454e..0a54a60f 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -11,19 +11,16 @@ function teardown() { _default_teardown ; } # Process matching notes: # opendkim (/usr/sbin/opendkim) - x2 of the same process are found running (1 is the parent) # opendmarc (/usr/sbin/opendmarc) -# master (/usr/lib/postfix/sbin/master) - Postfix main process (Can take a few seconds running to be ready) -# NOTE: pgrep or pkill used with `--full` would also match `/usr/sbin/amavisd-new (master)` -# -# amavi (/usr/sbin/amavi) - Matches three processes, the main process is `/usr/sbin/amavisd-new (master)` -# NOTE: `amavisd-new` can only be matched with `--full`, regardless pkill would return `/usr/sbin/amavi` +# postfix (/usr/lib/postfix/sbin/master) - Postfix main process (two ancestors, launched via pidproxy python3 script) # +# amavisd-new (usr/sbin/amavisd-new) # clamd (/usr/sbin/clamd) # dovecot (/usr/sbin/dovecot) # fetchmail (/usr/bin/fetchmail) -# fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - Started by fail2ban-wrapper.sh +# fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - NOTE: python3 is due to the shebang # mta-sts-daemon (/usr/bin/bin/python3 /usr/bin/mta-sts-daemon) -# postgrey (postgrey) - NOTE: This process lacks path information to match with `--full` in pgrep / pkill -# postsrsd (/usr/sbin/postsrsd) - NOTE: Also matches the wrapper: `/bin/bash /usr/local/bin/postsrsd-wrapper.sh` +# postgrey (postgrey) - NOTE: This process command uses perl via shebang, but unlike python3 the context is missing +# postsrsd (/usr/sbin/postsrsd) # saslauthd (/usr/sbin/saslauthd) - x5 of the same process are found running (1 is a parent of 4) # Delays: @@ -31,17 +28,16 @@ function teardown() { _default_teardown ; } # dovecot + fail2ban, take approx 1 sec to kill properly # opendkim + opendmarc can take up to 6 sec to kill properly # clamd + postsrsd sometimes take 1-3 sec to restart after old process is killed. -# postfix + fail2ban (due to Wrapper scripts) can delay a restart by up to 5 seconds from usage of sleep. # These processes should always be running: CORE_PROCESS_LIST=( - master + postfix ) # These processes can be toggled via ENV: # NOTE: clamd handled in separate test case ENV_PROCESS_LIST=( - amavi + amavisd-new dovecot fail2ban-server fetchmail @@ -89,7 +85,7 @@ ENV_PROCESS_LIST=( done } -# Average time: 23 seconds (29 with wrapper scripts) +# Average time: 23 seconds @test "(enabled ENV) should restart processes when killed" { export CONTAINER_NAME=${CONTAINER2_NAME} local CONTAINER_ARGS_ENV_CUSTOM=( @@ -153,14 +149,18 @@ function _should_restart_when_killed() { # Wait until process has been running for at least MIN_PROCESS_AGE: # (this allows us to more confidently check the process was restarted) _run_until_success_or_timeout 30 _check_if_process_is_running "${PROCESS}" "${MIN_PROCESS_AGE}" - # NOTE: refute_output doesn't have output to compare to when a run failure is due to a timeout + # NOTE: refute_output will not have any output to compare against if a `run` failure is caused by a timeout assert_success assert_output --partial "${PROCESS}" # Should kill the process successfully: # (which should then get restarted by supervisord) - _run_in_container pkill --echo "${PROCESS}" - assert_output --partial "${PROCESS}" + # NOTE: The process name from `pkill --echo` does not always match the equivalent processs name from `pgrep --list-full`. + # The oldest process returned (if multiple) should be the top-level process launched by supervisord, + # the PID will verify the target process was killed correctly: + local PID=$(_exec_in_container pgrep --full --oldest "${PROCESS}") + _run_in_container pkill --echo --full "${PROCESS}" + assert_output --partial "killed (pid ${PID})" assert_success # Wait until original process is not running: @@ -176,13 +176,16 @@ function _should_restart_when_killed() { assert_output --partial "${PROCESS}" } -# NOTE: CONTAINER_NAME is implicit; it should have be set prior to calling. +# NOTE: CONTAINER_NAME is implicit; it should have been set prior to calling. function _check_if_process_is_running() { local PROCESS=${1} local MIN_SECS_RUNNING [[ -n ${2:-} ]] && MIN_SECS_RUNNING=('--older' "${2}") - local IS_RUNNING=$(docker exec "${CONTAINER_NAME}" pgrep --list-full "${MIN_SECS_RUNNING[@]}" "${PROCESS}") + # `--list-full` provides information for matching against (full process path) + # `--full` allows matching the process against the full path (required if a process is not the exec command, such as started by python3 command without a shebang) + # `--oldest` should select the parent process when there are multiple results, typically the command defined in `supervisor-app.conf` + local IS_RUNNING=$(_exec_in_container pgrep --full --list-full "${MIN_SECS_RUNNING[@]}" --oldest "${PROCESS}") # When no matches are found, nothing is returned. Provide something we can assert on (helpful for debugging): if [[ ! ${IS_RUNNING} =~ ${PROCESS} ]]; then From 9cdbef2b369fb4fb0f1b4e534da8703daf92abc9 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Thu, 18 Jan 2024 10:41:55 +0100 Subject: [PATCH 125/125] setup/dkim: chown created dkim directories and keys to config user (#3783) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 2 ++ target/bin/open-dkim | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfdfd314..3ecf1251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ All notable changes to this project will be documented in this file. The format ### Fixes +- **Setup:** + - `setup` CLI - `setup dkim domain` now creates the keys files with the user owning the key directory ([#3783](https://github.com/docker-mailserver/docker-mailserver/pull/3783)) - **Dovecot:** - During container startup for Dovecot Sieve, `.sievec` source files compiled to `.svbin` now have their `mtime` adjusted post setup to ensure it is always older than the associated `.svbin` file. This avoids superfluous error logs for sieve scripts that don't actually need to be compiled again ([#3779](https://github.com/docker-mailserver/docker-mailserver/pull/3779)) - **Internal:** diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 86fbfb81..808ef8cc 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -144,6 +144,9 @@ while read -r DKIM_DOMAIN; do --directory="/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}" fi + # fix permissions to use the same user:group as /tmp/docker-mailserver/opendkim/keys + chown -R "$(stat -c '%U:%G' /tmp/docker-mailserver/opendkim/keys)" "/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}" + # write to KeyTable if necessary KEYTABLEENTRY="${SELECTOR}._domainkey.${DKIM_DOMAIN} ${DKIM_DOMAIN}:${SELECTOR}:/etc/opendkim/keys/${DKIM_DOMAIN}/${SELECTOR}.private" if [[ ! -f "/tmp/docker-mailserver/opendkim/KeyTable" ]]; then