From 303ca82fb925ab9a0e0aa8bf68a2caa402afe0aa Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 01:02:19 +1300 Subject: [PATCH 01/17] docs(fix): New external link icon workaround for mkdocs-material 9.5.5 (#3823) This is the easiest to maintain workaround now available. Upstream continues to reject the value such a feature for accessibility. --- docs/content/assets/css/customizations.css | 8 ++++++-- docs/mkdocs.yml | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/content/assets/css/customizations.css b/docs/content/assets/css/customizations.css index 49ede009..320da152 100644 --- a/docs/content/assets/css/customizations.css +++ b/docs/content/assets/css/customizations.css @@ -16,12 +16,16 @@ If you want to append instead, switch `::before` to `::after`. src: url('../fonts/external-link.woff') format('woff'); } -/* Matches the two nav link classes that start with `http` `href` values, regular docs pages use relative URLs instead. */ -.md-tabs__link[href^="http"]::before, .md-nav__link[href^="http"]::before { +/* + Since mkdocs-material 9.5.5 broke support in our docs from DMS v13.3.1, we now use our own class name, + which has been included for the two external nav links in mkdocs.yml via workaround (insert HTML). +*/ +.icon-external-link::before { display: inline-block; /* treat similar to text */ font-family: 'external-link'; content:'\0041'; /* represents "A" which our font renders as an icon instead of the "A" glyph */ font-size: 80%; /* icon is a little too big by default, scale it down */ + margin-right: 4px; } /* ============================================================================================================= */ diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 0d34b407..15d2c3b2 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -173,5 +173,5 @@ nav: - 'General Information': contributing/general.md - 'Tests': contributing/tests.md - 'Issues and Pull Requests': contributing/issues-and-pull-requests.md - - 'DockerHub': https://hub.docker.com/r/mailserver/docker-mailserver/ - - 'GHCR': https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver + - 'DockerHub': https://hub.docker.com/r/mailserver/docker-mailserver/ + - 'GHCR': https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver From 22c6daee32ad0769002564054654a4e989c9eb10 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 01:21:24 +1300 Subject: [PATCH 02/17] chore: Revise improper restart message (#3826) Improved guidance. --- target/scripts/startup/check-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index 61f21d1c..cf1c40f5 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -19,7 +19,7 @@ function _check_improper_restart() { if [[ -f /CONTAINER_START ]]; then _log 'warn' 'This container was (likely) improperly restarted which can result in undefined behavior' - _log 'warn' 'Please destroy the container properly and then start DMS again' + _log 'warn' "Please use 'docker compose up --force-recreate' or equivalent (view our troubleshooting docs)" fi } From 47f8d50bebaa0b1f1cb85d45c1d9210d9ce80a49 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:28:26 +1300 Subject: [PATCH 03/17] fix: Ensure configs are sanitized for parsing (#3819) * chore: Detect missing final newline in configs read These lines will be not be processed by `read`, emit a warning to raise awareness. * fix: Ensure parsed config has final newline appended (when possible) This functionality was handled in `accounts.sh` via a similar sed command (that the linked references also offer). `printf` is better for this, no shellcheck comment required either. We additionally don't attempt to modify files that are read-only. * fix: Ensure parsed configs have CRLF to LF corrected (where possible) Likewise, this runtime fix was only covering two config files. It now applies to all callers of this method. * fix: Sanitize `postfix-master.cf` via helper This feature should have been using the helper to avoid user error from their config updates accidentally introducing subtle breakage implicitly (due to CRLF or missing final newline). * tests: Add test cases for new helpers * tests: `rm` is redundant when using `BATS_TEST_TMPDIR` This temporary directory is created and removed implicitly. Even after a test failure. * chore: Remove old `postfix-virtual.cf` migration logic This was introduced in 2018, there should be no one needing to rely on this anymore? * tests: Remove comment on sed failure concern * chore: Add entry to `CHANGELOG.md` * Apply suggestions from code review Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 4 ++ target/scripts/helpers/accounts.sh | 14 ------- target/scripts/helpers/aliases.sh | 5 --- target/scripts/helpers/utils.sh | 35 ++++++++++++++++ target/scripts/startup/setup.d/postfix.sh | 22 +++++----- test/helper/setup.bash | 1 - .../set3/scripts/helper_functions.bats | 42 +++++++++++++++++++ 7 files changed, 92 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f07561..9f440e31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,10 @@ The most noteworthy change of this release is the update of the container's base - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) +### Fixes + +- DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) + ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) ### Fixes diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 7499a2ec..31ded04a 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -19,16 +19,9 @@ function _create_accounts() { _create_masters if [[ -f ${DATABASE_ACCOUNTS} ]]; then - _log 'trace' "Checking file line endings" - sed -i 's|\r||g' "${DATABASE_ACCOUNTS}" - _log 'trace' "Regenerating postfix user list" echo "# WARNING: this file is auto-generated. Modify ${DATABASE_ACCOUNTS} to edit the user list." > /etc/postfix/vmailbox - # checking that ${DATABASE_ACCOUNTS} ends with a newline - # shellcheck disable=SC1003 - sed -i -e '$a\' "${DATABASE_ACCOUNTS}" - chown dovecot:dovecot "${DOVECOT_USERDB_FILE}" chmod 640 "${DOVECOT_USERDB_FILE}" @@ -158,15 +151,8 @@ function _create_masters() { local DATABASE_DOVECOT_MASTERS='/tmp/docker-mailserver/dovecot-masters.cf' if [[ -f ${DATABASE_DOVECOT_MASTERS} ]]; then - _log 'trace' "Checking file line endings" - sed -i 's|\r||g' "${DATABASE_DOVECOT_MASTERS}" - _log 'trace' "Regenerating dovecot masters list" - # checking that ${DATABASE_DOVECOT_MASTERS} ends with a newline - # shellcheck disable=SC1003 - sed -i -e '$a\' "${DATABASE_DOVECOT_MASTERS}" - chown dovecot:dovecot "${DOVECOT_MASTERDB_FILE}" chmod 640 "${DOVECOT_MASTERDB_FILE}" diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 04a56da3..b0f2fa1a 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -12,11 +12,6 @@ function _handle_postfix_virtual_config() { local DATABASE_VIRTUAL=/tmp/docker-mailserver/postfix-virtual.cf if [[ -f ${DATABASE_VIRTUAL} ]]; then - # fixing old virtual user file - if grep -q ",$" "${DATABASE_VIRTUAL}"; then - sed -i -e "s|, |,|g" -e "s|,$||g" "${DATABASE_VIRTUAL}" - fi - cp -f "${DATABASE_VIRTUAL}" /etc/postfix/virtual else _log 'debug' "'${DATABASE_VIRTUAL}' not provided - no mail alias/forward created" diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index f7095bf3..c848e13e 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -17,9 +17,44 @@ function _escape_for_sed() { # Returns input after filtering out lines that are: # empty, white-space, comments (`#` as the first non-whitespace character) function _get_valid_lines_from_file() { + _convert_crlf_to_lf_if_necessary "${1}" + _append_final_newline_if_missing "${1}" + grep --extended-regexp --invert-match "^\s*$|^\s*#" "${1}" || true } +# This is to sanitize configs from users that unknowingly introduced CRLF: +function _convert_crlf_to_lf_if_necessary() { + if [[ $(file "${1}") =~ 'CRLF' ]]; then + _log 'warn' "File '${1}' contains CRLF line-endings" + + if [[ -w ${1} ]]; then + _log 'debug' 'Converting CRLF to LF' + sed -i 's|\r||g' "${1}" + else + _log 'warn' "File '${1}' is not writable - cannot change CRLF to LF" + fi + fi +} + +# This is to sanitize configs from users that unknowingly removed the end-of-file LF: +function _append_final_newline_if_missing() { + # Correctly detect a missing final newline and fix it: + # https://stackoverflow.com/questions/38746/how-to-detect-file-ends-in-newline#comment82380232_25749716 + # https://unix.stackexchange.com/questions/31947/how-to-add-a-newline-to-the-end-of-a-file/441200#441200 + # https://unix.stackexchange.com/questions/159557/how-to-non-invasively-test-for-write-access-to-a-file + if [[ $(tail -c1 "${1}" | wc -l) -eq 0 ]]; then + # Avoid fixing when the destination is read-only: + if [[ -w ${1} ]]; then + printf '\n' >> "${1}" + + _log 'info' "File '${1}' was missing a final newline - this has been fixed" + else + _log 'warn' "File '${1}' is missing a final newline - it is not writable, hence it was not fixed - the last line will not be processed!" + fi + fi +} + # Provide the name of an environment variable to this function # and it will return its value stored in /etc/dms-settings function _get_dms_env_value() { diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 5aec8636..05052faa 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -109,8 +109,9 @@ function _setup_postfix_late() { function __postfix__setup_override_configuration() { __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values' - if [[ -f /tmp/docker-mailserver/postfix-main.cf ]]; then - cat /tmp/docker-mailserver/postfix-main.cf >>/etc/postfix/main.cf + local OVERRIDE_CONFIG_POSTFIX_MAIN='/tmp/docker-mailserver/postfix-main.cf' + if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MAIN} ]]; then + cat "${OVERRIDE_CONFIG_POSTFIX_MAIN}" >>/etc/postfix/main.cf _adjust_mtime_for_postfix_maincf # do not directly output to 'main.cf' as this causes a read-write-conflict @@ -118,20 +119,19 @@ function __postfix__setup_override_configuration() { mv /tmp/postfix-main-new.cf /etc/postfix/main.cf _adjust_mtime_for_postfix_maincf - __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '/tmp/docker-mailserver/postfix-main.cf'" + __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MAIN}'" else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-main.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MAIN}' was not provided" fi - if [[ -f /tmp/docker-mailserver/postfix-master.cf ]]; then + local OVERRIDE_CONFIG_POSTFIX_MASTER='/tmp/docker-mailserver/postfix-master.cf' + if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MASTER} ]]; then while read -r LINE; do - if [[ ${LINE} =~ ^[0-9a-z] ]]; then - postconf -P "${LINE}" - fi - done < /tmp/docker-mailserver/postfix-master.cf - __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '/tmp/docker-mailserver/postfix-master.cf'" + [[ ${LINE} =~ ^[0-9a-z] ]] && postconf -P "${LINE}" + done < <(_get_valid_lines_from_file "${OVERRIDE_CONFIG_POSTFIX_MASTER}") + __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MASTER}'" else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-master.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MASTER}' was not provided" fi } diff --git a/test/helper/setup.bash b/test/helper/setup.bash index 0dd57bd6..ed2e4e32 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -102,7 +102,6 @@ function _init_with_defaults() { # The config volume cannot be read-only as some data needs to be written at container startup # - # - two sed failures (unknown lines) # - dovecot-quotas.cf (setup-stack.sh:_setup_dovecot_quotas) # - postfix-aliases.cf (setup-stack.sh:_setup_postfix_aliases) # TODO: Check how many tests need write access. Consider using `docker create` + `docker cp` for easier cleanup. diff --git a/test/tests/parallel/set3/scripts/helper_functions.bats b/test/tests/parallel/set3/scripts/helper_functions.bats index 332de448..518f8717 100644 --- a/test/tests/parallel/set3/scripts/helper_functions.bats +++ b/test/tests/parallel/set3/scripts/helper_functions.bats @@ -70,3 +70,45 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/ assert_failure assert_output --partial "ENV var name must be provided to _env_var_expect_integer" } + +@test '(utils.sh) _convert_crlf_to_lf_if_necessary' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh + source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh + source "${SOURCE_BASE_PATH}/utils.sh" + + # Create a temporary file in the BATS test-case folder: + local TMP_DMS_CONFIG=$(mktemp -p "${BATS_TEST_TMPDIR}" -t 'dms_XXX.cf') + # A file with mixed line-endings including CRLF: + echo -en 'line one\nline two\r\n' > "${TMP_DMS_CONFIG}" + + # Confirm CRLF detected: + run file "${TMP_DMS_CONFIG}" + assert_output --partial 'CRLF' + + # Helper method detects and fixes: + _convert_crlf_to_lf_if_necessary "${TMP_DMS_CONFIG}" + run file "${TMP_DMS_CONFIG}" + refute_output --partial 'CRLF' +} + +@test '(utils.sh) _append_final_newline_if_missing' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh + source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh + source "${SOURCE_BASE_PATH}/utils.sh" + + # Create a temporary file in the BATS test-case folder: + local TMP_DMS_CONFIG=$(mktemp -p "${BATS_TEST_TMPDIR}" -t 'dms_XXX.cf') + # A file missing a final newline: + echo -en 'line one\nline two' > "${TMP_DMS_CONFIG}" + + # Confirm missing newline: + run bash -c "tail -c 1 '${TMP_DMS_CONFIG}' | wc -l" + assert_output '0' + + # Helper method detects and fixes: + _append_final_newline_if_missing "${TMP_DMS_CONFIG}" + run bash -c "tail -c 1 '${TMP_DMS_CONFIG}' | wc -l" + assert_output '1' +} From 487867285bd1c7c6d3157afde45adce08d3c0409 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:32:18 +1300 Subject: [PATCH 04/17] docs: UX Improvement - Better distinguish side nav page categories (#3835) --- docs/content/assets/css/customizations.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/content/assets/css/customizations.css b/docs/content/assets/css/customizations.css index 320da152..25cb0274 100644 --- a/docs/content/assets/css/customizations.css +++ b/docs/content/assets/css/customizations.css @@ -102,3 +102,8 @@ div.md-content article.md-content__inner a.toclink code { .highlight.no-copy .md-clipboard { display: none; } /* ============================================================================================================= */ + +/* Make the left-sidebar nav categories better distinguished from page links (bold text) */ +.md-nav__item--nested > .md-nav__link { + font-weight: 700; +} From 7d9eb1e4a7ddd7b78b7bda2928b802af990af5ec Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:32:49 +1300 Subject: [PATCH 05/17] docs: Add context to `sender-cleanup` in Postfix `master.cf` (#3834) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/postfix/master.cf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/postfix/master.cf b/target/postfix/master.cf index e5b955a4..7d2b21d6 100644 --- a/target/postfix/master.cf +++ b/target/postfix/master.cf @@ -46,6 +46,8 @@ pickup fifo n - n 60 1 pickup -o content_filter= -o receive_override_options=no_header_body_checks +# This relates to submission(s) services defined above: +# https://www.postfix.org/BUILTIN_FILTER_README.html#mx_submission sender-cleanup unix n - n - 0 cleanup -o syslog_name=postfix/sender-cleanup -o header_checks=pcre:/etc/postfix/maps/sender_header_filter.pcre From ba27edc801c0b94cfb7d975dd2fb918a75ac9ea3 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:07:46 +0100 Subject: [PATCH 06/17] Rspamd: only declare Rspamd variables when not already declared (#3837) * only declare Rspamd vars when not already declared * update CHANGELOG * Update CHANGELOG.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 1 + target/scripts/helpers/rspamd.sh | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f440e31..a9c03f79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The most noteworthy change of this release is the update of the container's base ### Fixes - DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) +- Variables related to Rspamd are declared as `readonly`, which would cause warnings in the log when being re-declared; we now guard against this issue ([#3837](https://github.com/docker-mailserver/docker-mailserver/pull/3837)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh index 2f4dcc46..8d1fd668 100644 --- a/target/scripts/helpers/rspamd.sh +++ b/target/scripts/helpers/rspamd.sh @@ -15,14 +15,19 @@ function __do_as_rspamd_user() { # they cannot be modified. Use this function when you require common directory # names, file names, etc. function _rspamd_get_envs() { - readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' - readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + # If the variables are already set, we cannot set them again as they are declared + # with `readonly`. Checking whether one is declared suffices, because either all + # are declared at once, or none. + if [[ ! -v RSPAMD_LOCAL_D ]]; then + readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' + readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" - readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" + readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" - readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" + readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" + fi } # Parses `RSPAMD_DMS_CUSTOM_COMMANDS_F` and executed the directives given by the file. From 9ac11021e1c33d87ec706f3f76b6c3197f1f000d Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Fri, 26 Jan 2024 14:40:29 +0100 Subject: [PATCH 07/17] setup-stack: fix error when RSPAMD_DMS_DKIM_D is not set (#3827) * setup-stack: fix error when RSPAMD_DMS_DKIM_D is not set prevent messages like this chown: cannot access '': No such file or directory when RSPAMD_DMS_DKIM_D has no value * Update target/scripts/startup/setup-stack.sh --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/scripts/startup/setup-stack.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index c3c54cc3..24ef1581 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -95,8 +95,9 @@ function _setup_apply_fixes_after_configuration() { _log 'debug' 'Removing files and directories from older versions' rm -rf /var/mail-state/spool-postfix/{dev,etc,lib,pid,usr,private/auth} + _rspamd_get_envs # /tmp/docker-mailserver/rspamd/dkim - _log 'debug' "Ensuring ${RSPAMD_DMS_DKIM_D} is owned by '_rspamd:_rspamd'" + _log 'debug' "Ensuring '${RSPAMD_DMS_DKIM_D}' is owned by '_rspamd:_rspamd'" chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" } From a8ccd54da57111f6e619c987da1ca9d807fa4fcc Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 01:50:01 +1300 Subject: [PATCH 08/17] ci: `docs-preview-deploy.yml` - Switch to official `download-artifact` action (#3838) v4 of the official action now supports this use-case. --- .github/workflows/docs-preview-deploy.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 93819efa..6f6904d5 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -23,14 +23,13 @@ jobs: # Restore workflow context # # ======================== # - # The official Github Action for downloading artifacts does not support multi-workflow + # Retrieve the artifact uploaded from `docs-preview-prepare.yml` workflow run that triggered this deployment - name: 'Download build artifact' - uses: dawidd6/action-download-artifact@v3 + uses: actions/download-artifact@v4 with: + name: preview-build github_token: ${{ secrets.GITHUB_TOKEN }} run_id: ${{ github.event.workflow_run.id }} - workflow: docs-preview-prepare.yml - name: preview-build - name: 'Extract build artifact' run: tar -xf artifact.tar.zst From f27629be4e24a1fc288e947662f8294743063d5f Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 01:51:37 +1300 Subject: [PATCH 09/17] docs: Minor revisions to `README.md` (#3839) --- README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e81d21ad..cfd453ff 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,27 @@ [ci::status]: https://img.shields.io/github/actions/workflow/status/docker-mailserver/docker-mailserver/default_on_push.yml?branch=master&color=blue&label=CI&logo=github&logoColor=white&style=for-the-badge [ci::github]: https://github.com/docker-mailserver/docker-mailserver/actions -[docker::pulls]: https://img.shields.io/docker/pulls/mailserver/docker-mailserver.svg?style=for-the-badge&logo=docker&logoColor=white +[docker::pulls]: https://img.shields.io/docker/pulls/mailserver/docker-mailserver.svg?style=for-the-badge&logo=docker&logoColor=white&color=blue [docker::hub]: https://hub.docker.com/r/mailserver/docker-mailserver/ -[documentation::badge]: https://img.shields.io/badge/DOCUMENTATION-GH%20PAGES-0078D4?style=for-the-badge&logo=git&logoColor=white +[documentation::badge]: https://img.shields.io/badge/DOCUMENTATION-GH%20PAGES-0078D4?style=for-the-badge&logo=googledocs&logoColor=white [documentation::web]: https://docker-mailserver.github.io/docker-mailserver/latest/ ## :page_with_curl: About -A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. Originally created by @tomav, this project is now maintained by volunteers since January 2021. +A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). +- Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. +- Originally created by [@tomav](https://github.com/tomav), this project is now maintained by volunteers since January 2021. -## :bulb: Documentation +## -We provide a [dedicated documentation][documentation::web] hosted on GitHub Pages. Make sure to read it as it contains all the information necessary to set up and configure your mail server. The documentation is crafted with Markdown & [MkDocs Material](https://squidfunk.github.io/mkdocs-material/). +> [!TIP] +> Be sure to read [our documentation][documentation::web]. It provides guidance on initial setup of your mail server. -## :boom: Issues - -If you have issues, please search through [the documentation][documentation::web] **for your version** before opening an issue. The issue tracker is for issues, not for personal support. Make sure the version of the documentation matches the image version you're using! +> [!IMPORTANT] +> If you have issues, please search through [the documentation][documentation::web] **for your version** before opening an issue. +> +> The issue tracker is for issues, not for personal support. +> Make sure the version of the documentation matches the image version you're using! ## :link: Links to Useful Resources From 4a05d7bb7caeb67347034f5280f41f2a42170410 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 10:23:49 +1300 Subject: [PATCH 10/17] docs: Add Debian 12 breaking change for `opendmarc` package (#3841) --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9c03f79..e785cac0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,11 @@ The most noteworthy change of this release is the update of the container's base - The script now uses `/etc/os-release` to determine the release name of Debian - Removed custom installations of Fail2Ban, getmail6 and Rspamd - Updated packages lists and added comments for maintainability +- OpenDMARC upgrade: `v1.4.0` => `v1.4.2` ([#3841](https://github.com/docker-mailserver/docker-mailserver/pull/3841)) + - Previous versions of OpenDMARC would place incoming mail from domains announcing `p=quarantaine` (_that fail the DMARC check_) into the [Postfix "hold" queue](https://www.postfix.org/QSHAPE_README.html#hold_queue) until administrative intervention. + - [OpenDMARC v1.4.2 has disabled that feature by default](https://github.com/trusteddomainproject/OpenDMARC/issues/105), but it can be enabled again by adding the setting `HoldQuarantinedMessages true` to [`/etc/opendmarc.conf`](https://github.com/docker-mailserver/docker-mailserver/blob/v13.3.1/target/opendmarc/opendmarc.conf) (_provided from DMS_). + - [Our `user-patches.sh` feature](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/override-defaults/user-patches/) provides a convenient approach to updating that config file. + - Please let us know if you disagree with the upstream default being carried with DMS, or the value of providing alternative configuration support within DMS. - **Postfix:** - Postfix upgrade from 3.5 to 3.7 ([#3403](https://github.com/docker-mailserver/docker-mailserver/pull/3403)) - `compatibility_level` was raised from `2` to `3.6` From 204825fa5a2476a0b185df85365b5cad6b73b570 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 20:41:19 +1300 Subject: [PATCH 11/17] ci(fix): `docs-preview-deploy.yml` - Use the correct setting names (#3843) --- .github/workflows/docs-preview-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 6f6904d5..3b6f3c67 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -28,8 +28,8 @@ jobs: uses: actions/download-artifact@v4 with: name: preview-build - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} - name: 'Extract build artifact' run: tar -xf artifact.tar.zst From 11c508cd11c9c850527c54b49631374052c67303 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 28 Jan 2024 11:15:47 +0000 Subject: [PATCH 12/17] docs: update `CONTRIBUTORS.md` (#3844) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Casper --- CONTRIBUTORS.md | 183 +++++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 88 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f86435f4..72784ab0 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -134,13 +134,6 @@ Thanks goes to these wonderful people ✨ - - - alinmear -
- alinmear - - ap-wtioit @@ -148,6 +141,13 @@ Thanks goes to these wonderful people ✨ ap-wtioit + + + alinmear +
+ alinmear + + dominikwinter @@ -815,13 +815,20 @@ Thanks goes to these wonderful people ✨ - - nilshoell + + auchri
- nilshoell + auchri + + + stephan-devop +
+ stephan-devop + + stigok @@ -856,15 +863,15 @@ Thanks goes to these wonderful people ✨
TechnicLab - + + thomasschmit
thomasschmit - - + Thiritin @@ -899,15 +906,15 @@ Thanks goes to these wonderful people ✨
Twist235 - + + k3it
k3it - - + Drakulix @@ -942,15 +949,15 @@ Thanks goes to these wonderful people ✨
Zepmann - - - - allddd -
- allddd - + + + nilshoell +
+ nilshoell + + nknapp @@ -985,15 +992,15 @@ Thanks goes to these wonderful people ✨
mrPjer - + + p3dda
p3dda - - + peter-hartmann @@ -1028,15 +1035,15 @@ Thanks goes to these wonderful people ✨
norrs - + + MightySCollins
MightySCollins - - + 501st-alpha1 @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
shyim - + + sjmudd
sjmudd - - + simonsystem @@ -1088,10 +1095,10 @@ Thanks goes to these wonderful people ✨ - - stephan-devop + + allddd
- stephan-devop + allddd @@ -1114,15 +1121,15 @@ Thanks goes to these wonderful people ✨
odinis - + + okamidash
okamidash - - + olaf-mandel @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
rhyst - + + rmlhuk
rmlhuk - - + rriski @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
squash - + + strarsis
strarsis - - + tamueller @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
worldworm - + + arcaine2
arcaine2 - - + awb99 @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
dimalo - + + eleith
eleith - - + ghnp5 @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
idaadi - + + ixeft
ixeft - - + jjtt @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
callmemagnus - + + marios88
marios88 - - + matrixes @@ -1395,13 +1402,6 @@ Thanks goes to these wonderful people ✨ mchamplain - - - auchri -
- auchri - - arkanovicz @@ -1539,10 +1539,10 @@ Thanks goes to these wonderful people ✨ - - thechubbypanda + + froks
- thechubbypanda + froks @@ -1696,6 +1696,13 @@ Thanks goes to these wonderful people ✨ crash7 + + + thechubbypanda +
+ thechubbypanda + + KCrawley @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
JustAnother1 - + + LeoWinterDE
LeoWinterDE - - + linhandev @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
madmath03 - + + maxemann96
maxemann96 - - + dragetd @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
mcchots - + + MohammedNoureldin
MohammedNoureldin - - + mpldr @@ -1839,13 +1846,6 @@ Thanks goes to these wonderful people ✨ radicand - - - froks -
- froks - - fkefer @@ -1911,6 +1911,13 @@ Thanks goes to these wonderful people ✨ Influencer + + + JamBalaya56562 +
+ JamBalaya56562 + + jcalfee @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
init-js - + + Jeidnx
Jeidnx - - + JiLleON @@ -1974,15 +1981,15 @@ Thanks goes to these wonderful people ✨
JOduMonT - + + Kaan88
Kaan88 - - + akkumar From 3b11a8305ee2e303b7e14faeb942e140d0434b3b Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 29 Jan 2024 10:35:19 +1300 Subject: [PATCH 13/17] docs: Remove ENV `ONE_DIR` (#3840) * docs: Better document DMS volumes * docs: Remove any mention of `ONE_DIR` ENV * chore: Remove `ONE_DIR` ENV from scripts Only `ONE_DIR=0` has any effect. As the actual feature is now dependent upon the `/var/mail-state` location existing. It is advised not mounting anything there instead if wanting to avoid runtime state consolidation. * docs: Adjust link ref convention This is more search friendly / organized to find references to all DMS volumes. * lint: Ensure final newline is present VSCode by default excludes this if the last line rendered is removed (rendered as a separate blank line). A separate setting can enforce adding the final newline upon save regardless. --- CHANGELOG.md | 3 + docs/content/config/advanced/kubernetes.md | 1 - .../config/advanced/optional-config.md | 62 +++++++++++++++++-- .../config/best-practices/dkim_dmarc_spf.md | 2 +- docs/content/config/environment.md | 13 ++-- docs/content/config/security/fail2ban.md | 4 +- docs/content/config/security/rspamd.md | 18 ++++-- docs/content/config/security/ssl.md | 4 +- .../tutorials/mailserver-behind-proxy.md | 2 - docs/content/faq.md | 39 ++++-------- docs/content/usage.md | 4 +- mailserver.env | 4 -- target/scripts/startup/setup.d/mail_state.sh | 6 +- .../startup/setup.d/security/rspamd.sh | 6 +- target/scripts/startup/variables-stack.sh | 1 - 15 files changed, 102 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e785cac0..41e6d5e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,9 @@ The most noteworthy change of this release is the update of the container's base ### Updates +- **Environment Variables:** + - `ONE_DIR` has been removed (legacy ENV) ([#3840](https://github.com/docker-mailserver/docker-mailserver/pull/3840)) + - It's only functionality remaining was to opt-out of run-time state consolidation with `ONE_DIR=0` (_when a volume was already mounted to `/var/mail-state`_). - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 8a47bffc..c277852a 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -40,7 +40,6 @@ data: POSTMASTER_ADDRESS: postmaster@example.com UPDATE_CHECK_INTERVAL: 10d POSTFIX_INET_PROTOCOLS: ipv4 - ONE_DIR: '1' ENABLE_CLAMAV: '1' ENABLE_POSTGREY: '0' ENABLE_FAIL2BAN: '1' diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 31090050..fb1ab29d 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -4,9 +4,64 @@ hide: - toc # Hide Table of Contents for this page --- -This is a list of all configuration files and directories which are optional or automatically generated in your [`docker-data/dms/config/`][docs-dms-config-volume] directory. +## Volumes -## Directories +DMS has several locations in the container which may be worth persisting externally via [Docker Volumes][docker-docs::volumes]. + +- Often you will want to prefer [bind mount volumes][docker-docs::volumes::bind-mount] for easy access to files at a local location on your filesystem. +- As a convention for our docs and example configs, the local location has the common prefix `docker-data/dms/` for grouping these related volumes. + +!!! info "Reference - Volmes for DMS" + + Our docs may refer to these DMS specific volumes only by name, or the host/container path for brevity. + + - [Config](#volumes-config): `docker-data/dms/config/` => `/tmp/docker-mailserver/` + - [Mail Storage](#volumes-mail): `docker-data/dms/mail-data/` => `/var/mail/` + - [State](#volumes-state): `docker-data/dms/mail-state/` => `/var/mail-state/` + - [Logs](#volumes-logs): `docker-data/dms/mail-logs/` => `/var/log/mail/` + +[docker-docs::volumes]: https://docs.docker.com/storage/volumes/ +[docker-docs::volumes::bind-mount]: https://docs.docker.com/storage/bind-mounts/ + +### Mail Storage Volume { #volumes-mail } + +This is the location where mail is delivered to your mailboxes. + +### State Volume { #volumes-state } + +Run-time specific state lives here, but so does some data you may want to keep if a failure event occurs (_crash, power loss_). + +!!! example "Examples of relevant data" + + - The Postfix queue (eg: mail pending delivery attempt) + - Fail2Ban blocks. + - ClamAV signature updates. + - Redis storage for Rspamd. + +!!! info "When a volume is mounted to `/var/mail-state/`" + + - Service run-time data is [consolidated into the `/var/mail-state/` directory][mail-state-folders]. Otherwise the original locations vary and would need to be mounted individually. + - The original locations are updated with symlinks to redirect to their new path in `/var/mail-state/` (_eg: `/var/lib/redis` => `/var/mail-state/lib-redis/`_). + + Supported services: Postfix, Dovecot, Fail2Ban, Amavis, PostGrey, ClamAV, SpamAssassin, Rspamd & Redis, Fetchmail, Getmail, LogRotate, PostSRSd, MTA-STS. + +!!! tip + + Sometimes it is helpful to disable this volume when troubleshooting to verify if the data stored here is in a bad state (_eg: caused by a failure event_). + +[mail-state-folders]: https://github.com/docker-mailserver/docker-mailserver/blob/v13.3.1/target/scripts/startup/setup.d/mail_state.sh#L13-L33 + +### Logs Volume { #volumes-log } + +This can be a useful volume to persist for troubleshooting needs for the full set of log files. + +### Config Volume { #volumes-config } + +Most configuration files for Postfix, Dovecot, etc. are persisted here. + +This is a list of all configuration files and directories which are optional, automatically generated / updated by our `setup` CLI, or other internal scripts. + +#### Directories - **sieve-filter:** directory for sieve filter scripts. (Docs: [Sieve][docs-sieve]) - **sieve-pipe:** directory for sieve pipe scripts. (Docs: [Sieve][docs-sieve]) @@ -14,7 +69,7 @@ This is a list of all configuration files and directories which are optional or - **ssl:** SSL Certificate directory if `SSL_TYPE` is set to `self-signed` or `custom`. (Docs: [SSL][docs-ssl]) - **rspamd:** Override directory for custom settings when using Rspamd (Docs: [Rspamd][docs-rspamd-override-d]) -## Files +#### Files - **{user_email_address}.dovecot.sieve:** User specific Sieve filter file. (Docs: [Sieve][docs-sieve]) - **before.dovecot.sieve:** Global Sieve filter file, applied prior to the `${login}.dovecot.sieve` filter. (Docs: [Sieve][docs-sieve]) @@ -42,7 +97,6 @@ This is a list of all configuration files and directories which are optional or - **user-patches.sh:** this file will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. (Docs: [FAQ - How to adjust settings with the `user-patches.sh` script][docs-faq-userpatches]) - **rspamd/custom-commands.conf:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) -[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory [docs-accounts-quota]: ../../config/user-management.md#quotas [docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases [docs-dkim]: ../../config/best-practices/dkim_dmarc_spf.md#dkim diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index ed56504c..290ac5cd 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -345,7 +345,7 @@ volumes: ``` [docs-accounts-add]: ../user-management.md#adding-a-new-account -[docs-volumes-config]: ../advanced/optional-config.md +[docs-volumes-config]: ../advanced/optional-config.md#volumes-config [docs-env-opendkim]: ../environment.md#enable_opendkim [docs-env-rspamd]: ../environment.md#enable_rspamd [docs-rspamd-config-dropin]: ../security/rspamd.md#manually diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 1aa6799d..b1c3e2b6 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -45,11 +45,6 @@ Default: 5000 The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage managed by Dovecot_). -##### ONE_DIR - -- 0 => state in default directories. -- **1** => consolidate all states into a single directory (`/var/mail-state`) to allow persistence using docker volumes. See the [related FAQ entry][docs-faq-onedir] for more information. - ##### ACCOUNT_PROVISIONER Configures the provisioning source of user accounts (including aliases) for user queries and authentication by services managed by DMS (_Postfix and Dovecot_). @@ -648,10 +643,10 @@ Controls the spam score threshold for triggering an action on mail that has a hi - [It will be quarantined][amavis-docs::quarantine] regardless of the `SA_KILL` action to perform. - With `D_PASS` the delivered mail also appends an `X-Quarantine-ID` mail header. The ID value of this header is part of the quarantined file name. - If emails are quarantined, they are compressed and stored at a location dependent on the [`ONE_DIR`](#one_dir) setting: + If emails are quarantined, they are compressed and stored at a location: - - `ONE_DIR=1` (default): `/var/mail-state/lib-amavis/virusmails/` - - `ONE_DIR=0`: `/var/lib/amavis/virusmails/` + - Default: `/var/lib/amavis/virusmails/` + - When the [`/var/mail-state/` volume][docs::dms-volumes-state] is present: `/var/mail-state/lib-amavis/virusmails/` !!! tip @@ -1056,9 +1051,9 @@ you to replace both instead of just the envelope sender. - password for default relay user [docs-rspamd]: ./security/rspamd.md -[docs-faq-onedir]: ../faq.md#what-about-docker-datadmsmail-state-folder-varmail-state-internally [docs-tls]: ./security/ssl.md [docs-tls-letsencrypt]: ./security/ssl.md#lets-encrypt-recommended [docs-tls-manual]: ./security/ssl.md#bring-your-own-certificates [docs-tls-selfsigned]: ./security/ssl.md#self-signed-certificates [docs-accounts-quota]: ./user-management.md#quotas +[docs::dms-volumes-state]: ./advanced/optional-config.md#volumes-state diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index 04e9a681..375f440c 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -33,7 +33,7 @@ DMS will automatically ban IP addresses of hosts that have generated 6 failed at ### Custom Files -!!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" +!!! question "What is [`docker-data/dms/config/`][docs::dms-volumes-config]?" This following configuration files inside the `docker-data/dms/config/` volume will be copied inside the container during startup @@ -44,7 +44,7 @@ This following configuration files inside the `docker-data/dms/config/` volume w - with this file, you can adjust F2B behavior in general - there is an example provided [in our repository on GitHub][github-file-f2bconfig] -[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory +[docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config [github-file-f2bjail]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-jail.cf [github-file-f2bconfig]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-fail2ban.cf diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 283c2a95..0c457ec3 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -83,9 +83,15 @@ DMS does not set a default password for the controller worker. You may want to d ### Persistence with Redis -When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. +When Rspamd is enabled, we implicitly also start an instance of Redis in the container: -Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). +- Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_or the [`/var/mail-state/`][docs::dms-volumes-state] volume when present_). +- With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. + +Redis uses `/etc/redis/redis.conf` for configuration: + +- We adjust this file when enabling the internal Redis service. +- If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). ### Web Interface @@ -145,7 +151,7 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo ### Manually -!!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" +!!! question "What is [`docker-data/dms/config/`][docs::dms-volumes-config]?" If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. @@ -156,7 +162,6 @@ If you want to overwrite the default settings and / or provide your own settings Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. [rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories -[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory [rspamd-docs-config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories ### With the Help of a Custom File @@ -197,7 +202,7 @@ You can also have comments (the line starts with `#`) and blank lines in `custom You want to start using Rspamd? Rspamd is disabled by default, so you need to set the following environment variables: -```cf +```env ENABLE_RSPAMD=1 ENABLE_OPENDKIM=0 ENABLE_OPENDMARC=0 @@ -252,3 +257,6 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [Abusix]: https://abusix.com/ [abusix-rspamd-integration]: https://docs.abusix.com/abusix-mail-intelligence/gbG8EcJ3x3fSUv8cMZLiwA/getting-started/dmw9dcwSGSNQiLTssFAnBW#rspamd + +[docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config +[docs::dms-volumes-state]: ../advanced/optional-config.md#volumes-state diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index c0c615cc..10f7adc4 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -634,7 +634,7 @@ This setup only comes with one caveat: The domain has to be configured on anothe Use self-signed certificates only for testing purposes! -This feature requires you to provide the following files into your [`docker-data/dms/config/ssl/` directory][docs-optional-config] (_internal location: `/tmp/docker-mailserver/ssl/`_): +This feature requires you to provide the following files into your [`docker-data/dms/config/ssl/` directory][docs::dms-volumes-config] (_internal location: `/tmp/docker-mailserver/ssl/`_): - `-key.pem` - `-cert.pem` @@ -876,7 +876,7 @@ By default DMS uses [`ffdhe4096`][ffdhe4096-src] from [IETF RFC 7919][ietf::rfc: Despite this, if you must use non-standard DH parameters or you would like to swap `ffdhe4096` for a different group (eg `ffdhe2048`); Add your own PEM encoded DH params file via a volume to `/tmp/docker-mailserver/dhparams.pem`. This will replace DH params for both Dovecot and Postfix services during container startup. [docs-env::ssl-type]: ../environment.md#ssl_type -[docs-optional-config]: ../advanced/optional-config.md +[docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config [docs-faq-baredomain]: ../../faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname [github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/compose.yaml diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 7bdef5d6..99939542 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -125,5 +125,3 @@ service imap-login { !!! note Port `10993` is used here to avoid conflicts with internal systems like `postscreen` and `amavis` as they will exchange messages on the default port and obviously have a different origin then compared to the proxy. - -[docs-optionalconfig]: ../../config/advanced/optional-config.md diff --git a/docs/content/faq.md b/docs/content/faq.md index 0985c0a9..6b1782e3 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -360,20 +360,6 @@ DMS does not manage those concerns, verify they are not causing your delivery pr - [mail-tester](https://www.mail-tester.com/) can test your deliverability. - [helloinbox](https://www.helloinbox.email/) provides a checklist of things to improve your deliverability. -### Special Directories - -#### What About the `docker-data/dms/config/` Directory? - -This documentation and all example configuration files in the GitHub repository use `docker-data/dms/config/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/tmp/docker-mailserver/` inside the container. - -Most configuration files for Postfix, Dovecot, etc. are persisted here. [Optional configuration][docs-optional-configuration] is stored here as well. - -#### What About the `docker-data/dms/mail-state/` Directory? - -This documentation and all example configuration files in the GitHub repository use `docker-data/dms/mail-state/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/var/mail-state/` inside the container. - -When you run DMS with the ENV variable `ONE_DIR=1` (default), this directory will provide support to persist Fail2Ban blocks, ClamAV signature updates, and the like when the container is restarted or recreated. Service data is [relocated to the `mail-state` folder][mail-state-folders] for the following services: Postfix, Dovecot, Fail2Ban, Amavis, PostGrey, ClamAV, SpamAssassin, Rspamd & Redis. - ### SpamAssasin #### How can I manage my custom SpamAssassin rules? @@ -390,14 +376,15 @@ The default setup `@local_domains_acl = ( ".$mydomain" );` does not match subdom Put received spams in `.Junk/` imap folder using `SPAMASSASSIN_SPAM_TO_INBOX=1` and `MOVE_SPAM_TO_JUNK=1` and add a _user_ cron like the following: -```conf -# This assumes you're having `environment: ONE_DIR=1` in the `mailserver.env`, -# with a consolidated config in `/var/mail-state` -# -# m h dom mon dow command -# Everyday 2:00AM, learn spam from a specific user -0 2 * * * docker exec mailserver sa-learn --spam /var/mail/example.com/username/.Junk --dbpath /var/mail-state/lib-amavis/.spamassassin -``` +!!! example + + **NOTE:** This example assumes you have a [`/var/mail-state` volume][docs::dms-volumes-state] mounted. + + ```conf + # m h dom mon dow command + # Everyday 2:00AM, learn spam from a specific user + 0 2 * * * docker exec mailserver sa-learn --spam /var/mail/example.com/username/.Junk --dbpath /var/mail-state/lib-amavis/.spamassassin + ``` With `docker-compose` you can more easily use the internal instance of `cron` within DMS. This is less problematic than the simple solution shown above, because it decouples the learning from the host on which DMS is running, and avoids errors if the mail server is not running. @@ -405,6 +392,8 @@ The following configuration works nicely: ??? example + **NOTE:** This example assumes you have a [`/var/mail-state` volume][docs::dms-volumes-state] mounted. + Create a _system_ cron file: ```sh @@ -418,9 +407,6 @@ The following configuration works nicely: Edit the system cron file `nano ./docker-data/dms/cron/sa-learn`, and set an appropriate configuration: ```conf - # This assumes you're having `environment: ONE_DIR=1` in the env-mailserver, - # with a consolidated config in `/var/mail-state` - # # '> /dev/null' to send error notifications from 'stderr' to 'postmaster@example.com' # # m h dom mon dow user command @@ -495,10 +481,10 @@ $spam_quarantine_to = "quarantine\@example.com"; ``` [fail2ban-customize]: ./config/security/fail2ban.md +[docs::dms-volumes-state]: ./config/advanced/optional-config.md#volumes-state [docs-maintenance]: ./config/advanced/maintenance/update-and-cleanup.md [docs-override-postfix]: ./config/advanced/override-defaults/postfix.md [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md -[docs-optional-configuration]: ./config/advanced/optional-config.md [docs::env::sa_env]: ./config/environment.md#spamassassin [docs::env::sa_kill]: ./config/environment.md#sa_kill [github-comment-baredomain]: https://github.com/docker-mailserver/docker-mailserver/issues/3048#issuecomment-1432358353 @@ -510,4 +496,3 @@ $spam_quarantine_to = "quarantine\@example.com"; [github-issue-1639]: https://github.com/docker-mailserver/docker-mailserver/issues/1639 [github-issue-1792]: https://github.com/docker-mailserver/docker-mailserver/pull/1792 [hanscees-userpatches]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-user-patches.sh -[mail-state-folders]: https://github.com/docker-mailserver/docker-mailserver/blob/c7e498194546416fb7231cb03254e77e085d18df/target/scripts/startup/misc-stack.sh#L24-L33 diff --git a/docs/content/usage.md b/docs/content/usage.md index 0550c24e..30f6e822 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -2,9 +2,9 @@ title: Usage --- -This pages explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs-dms-config-volume] to `/tmp/docker-mailserver/` inside the container. +This pages explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs::dms-volumes-config] to `/tmp/docker-mailserver/` inside the container. -[docs-dms-config-volume]: ./faq.md#what-about-the-docker-datadmsconfig-directory +[docs::dms-volumes-config]: ./config/advanced/optional-config.md#volumes-config ## Preliminary Steps diff --git a/mailserver.env b/mailserver.env index 1d131696..c171faed 100644 --- a/mailserver.env +++ b/mailserver.env @@ -30,10 +30,6 @@ LOG_LEVEL=info # debug => Also show debug messages SUPERVISOR_LOGLEVEL= -# 0 => mail state in default directories -# 1 => consolidate all states into a single directory (`/var/mail-state`) to allow persistence using docker volumes -ONE_DIR=1 - # Support for deployment where these defaults are not compatible (eg: some NAS appliances): # /var/mail vmail User ID (default: 5000) DMS_VMAIL_UID= diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 9963bbcc..9c43fea4 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -7,7 +7,7 @@ function _setup_save_states() { STATEDIR='/var/mail-state' - if [[ ${ONE_DIR} -eq 1 ]] && [[ -d ${STATEDIR} ]]; then + if [[ -d ${STATEDIR} ]]; then _log 'debug' "Consolidating all state onto ${STATEDIR}" # Always enabled features: @@ -111,9 +111,7 @@ function _setup_save_states() { # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3625 chmod 730 "${STATEDIR}/spool-postfix/maildrop" chmod 710 "${STATEDIR}/spool-postfix/public" - elif [[ ${ONE_DIR} -eq 1 ]]; then - _log 'warn' "'ONE_DIR=1' but no volume was mounted to '${STATEDIR}'" else - _log 'debug' 'Not consolidating state (because it has been disabled)' + _log 'debug' "'${STATEDIR}' is not present; Not consolidating state" fi } diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 86786932..a05a798e 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -127,9 +127,9 @@ expand_keys = true; EOF - # Here we adjust the Redis default configuration that we supply to Redis - # when starting it. Note that `/var/lib/redis/` is linked to - # `/var/mail-state/redis/` (for persisting it) if `ONE_DIR=1`. + # Here we adjust the Redis default configuration that we supply to Redis when starting it. + # NOTE: `/var/lib/redis/` is symlinked to `/var/mail-state/redis/` when DMS is started + # with a volume mounted to `/var/mail-state/` for data persistence. sedfile -i -E \ -e 's|^(bind).*|\1 127.0.0.1|g' \ -e 's|^(daemonize).*|\1 no|g' \ diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 0b351a9e..d14aeb3f 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -141,7 +141,6 @@ function __environment_variables_general_setup() { VARS[LOGWATCH_INTERVAL]="${LOGWATCH_INTERVAL:=none}" VARS[LOGWATCH_RECIPIENT]="${LOGWATCH_RECIPIENT:=${REPORT_RECIPIENT}}" VARS[LOGWATCH_SENDER]="${LOGWATCH_SENDER:=${REPORT_SENDER}}" - VARS[ONE_DIR]="${ONE_DIR:=1}" VARS[PERMIT_DOCKER]="${PERMIT_DOCKER:=none}" VARS[PFLOGSUMM_RECIPIENT]="${PFLOGSUMM_RECIPIENT:=${REPORT_RECIPIENT}}" VARS[PFLOGSUMM_SENDER]="${PFLOGSUMM_SENDER:=${REPORT_SENDER}}" From afb00939391e409a7bd9759d7346256198262144 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:38:01 +0100 Subject: [PATCH 14/17] spam: use Sieve for rewriting subject with Rspamd & SA/Amavis (#3820) --- CHANGELOG.md | 10 ++ docs/content/config/environment.md | 35 +++--- docs/content/config/security/rspamd.md | 5 +- mailserver.env | 13 ++- target/rspamd/local.d/actions.conf | 8 +- target/scripts/start-mailserver.sh | 2 + target/scripts/startup/check-stack.sh | 8 ++ .../scripts/startup/setup.d/security/misc.sh | 61 +++++++++-- target/scripts/startup/variables-stack.sh | 8 +- test/helper/common.bash | 16 +++ .../parallel/set1/spam_virus/amavis.bats | 101 ++++++++++++------ .../parallel/set1/spam_virus/rspamd_dkim.bats | 9 +- .../parallel/set1/spam_virus/rspamd_full.bats | 89 +++++++++------ .../set1/spam_virus/rspamd_partly.bats | 9 +- .../set1/spam_virus/spam_junk_folder.bats | 6 +- .../set1/spam_virus/undef_spam_subject.bats | 61 ----------- 16 files changed, 262 insertions(+), 179 deletions(-) delete mode 100644 test/tests/parallel/set1/spam_virus/undef_spam_subject.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 41e6d5e9..d16c03ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,14 @@ The most noteworthy change of this release is the update of the container's base - DMS `main.cf` has renamed `postscreen_dnsbl_whitelist_threshold` to `postscreen_dnsbl_allowlist_threshold` as part of this change. - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first. - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_). +- **Environment Variables**: + - `SA_SPAM_SUBJECT` has been renamed into `SPAM_SUBJECT` to become anti-spam service agnostic. ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) + - As this functionality is now handled in Dovecot via a Sieve script instead of the respective anti-spam service during Postfix processing, this feature will only apply to mail stored in Dovecot. If you have relied on this feature in a different context, it will no longer be available. + - Rspamd previously handled this functionality via the `rewrite_subject` action which as now been disabled by default in favor of the new approach with `SPAM_SUBJECT`. + - `SA_SPAM_SUBJECT` is now deprecated and will log a warning if used. The value is copied as a fallback to `SPAM_SUBJECT`. + - The default has changed to not prepend any prefix to the subject unless configured to do so. If you relied on the implicit prefix, you will now need to provide one explicitly. + - `undef` was previously supported as an opt-out with `SA_SPAM_SUBJECT`. This is no longer valid, the equivalent opt-out value is now an empty value (_or rather the omission of this ENV being configured_). + - The feature to include [`_SCORE_` tag](https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING) in your value to be replaced by the associated spam score is no longer available. ### Updates @@ -45,6 +53,8 @@ The most noteworthy change of this release is the update of the container's base - It's only functionality remaining was to opt-out of run-time state consolidation with `ONE_DIR=0` (_when a volume was already mounted to `/var/mail-state`_). - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) +- **Rspamd**: + - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead in favor of being anti-spam service agnostic ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) ### Fixes diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index b1c3e2b6..c98016d1 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -51,7 +51,7 @@ Configures the provisioning source of user accounts (including aliases) for user !!! tip "OAuth2 Support" - Presently DMS supports OAuth2 only as an supplementary authentication method. + Presently DMS supports OAuth2 only as an supplementary authentication method. - A third-party service must provide a valid token for the user which Dovecot validates with the authentication service provider. To enable this feature reference the [OAuth2 configuration example guide][docs::auth::oauth2-config-guide]. - User accounts must be provisioned to receive mail via one of the supported `ACCOUNT_PROVISIONER` providers. @@ -354,6 +354,16 @@ Enable to treat received spam as "read" (_avoids notification to MUA client of n - `X-Spam: Yes` (_added by Rspamd_) - `X-Spam-Flag: YES` (_added by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) +##### SPAM_SUBJECT + +This variable defines a prefix for e-mails tagged with the `X-Spam: Yes` (Rspamd) or `X-Spam-Flag: YES` (SpamAssassin/Amavis) header. + +Default: empty (no prefix will be added to e-mails) + +??? example "Including trailing white-space" + + Add trailing white-space by quote wrapping the value: `SPAM_SUBJECT='[SPAM] '` + #### Rspamd ##### ENABLE_RSPAMD @@ -562,7 +572,7 @@ Changes the interval in which log files are rotated. - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk) - [`MARK_SPAM_AS_READ=1`](#mark_spam_as_read) - - [`SA_SPAM_SUBJECT`](#sa_spam_subject) + - [`SPAM_SUBJECT`](#spam_subject) ##### SA_TAG @@ -602,8 +612,8 @@ When a spam score is high enough, mark mail as spam (_Appends the mail header: ` !!! info "Interaction with other ENV" - - [`SA_SPAM_SUBJECT`](#sa_spam_subject) modifies the mail subject to better communicate spam mail to the user. - - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SA_SPAM_SUBJECT`. + - [`SPAM_SUBJECT`](#spam_subject) modifies the mail subject to better communicate spam mail to the user. + - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SPAM_SUBJECT`. ##### SA_KILL @@ -659,23 +669,6 @@ Controls the spam score threshold for triggering an action on mail that has a hi [amavis-docs::actions]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#actions [amavis-docs::quarantine]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#quarantine -##### SA_SPAM_SUBJECT - -Adds a prefix to the subject header when mail is marked as spam (_via [`SA_TAG2`](#sa_tag2)_). - -- **`'***SPAM*** '`** => A string value to use as a mail subject prefix. -- `undef` => Opt-out of modifying the subject for mail marked as spam. - -??? example "Including trailing white-space" - - Add trailing white-space by quote wrapping the value: `SA_SPAM_SUBJECT='[SPAM] '` - -??? example "Including the associated spam score" - - The [`_SCORE_` tag][sa-docs::score-tag] will be substituted with the SpamAssassin score: `SA_SPAM_SUBJECT=***SPAM(_SCORE_)***`. - -[sa-docs::score-tag]: https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING - ##### SA_SHORTCIRCUIT_BAYES_SPAM - **1** => will activate SpamAssassin short circuiting for bayes spam detection. diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 0c457ec3..4c466c5d 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -22,8 +22,9 @@ The following environment variables are related to Rspamd: 5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -8. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] -9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) +8. [`SPAM_SUBJECT`](../environment.md#spam_subject) +9. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] +10. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd. diff --git a/mailserver.env b/mailserver.env index c171faed..38b9adca 100644 --- a/mailserver.env +++ b/mailserver.env @@ -130,6 +130,11 @@ ENABLE_IMAP=1 # **0** => Disabled ENABLE_CLAMAV=0 +# Add the value as a prefix to the mail subject when spam is detected. +# NOTE: By default spam is delivered to a junk folder, reducing the value of a subject prefix for spam. +# NOTE: When not using Docker Compose, other CRI may not support quote-wrapping the value here to preserve any trailing white-space. +SPAM_SUBJECT= + # Enables Rspamd # **0** => Disabled # 1 => Enabled @@ -378,7 +383,7 @@ ENABLE_SPAMASSASSIN=0 # Note: only has an effect if `ENABLE_SPAMASSASSIN=1` ENABLE_SPAMASSASSIN_KAM=0 -# deliver spam messages to the inbox (tagged using SA_SPAM_SUBJECT) +# deliver spam messages to the inbox (tagged using SPAM_SUBJECT) SPAMASSASSIN_SPAM_TO_INBOX=1 # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required) @@ -396,12 +401,6 @@ SA_TAG2=6.31 # triggers spam evasive actions SA_KILL=10.0 -# add tag to subject if spam detected -# The value `undef` opts-out of this feature. The value shown below is the default. -# NOTE: By default spam is delivered to a junk folder, reducing the value of adding a subject prefix. -# NOTE: If not using Docker Compose, other CRI may require the single quotes removed. -#SA_SPAM_SUBJECT='***SPAM*** ' - # ----------------------------------------------- # --- Fetchmail Section ------------------------- # ----------------------------------------------- diff --git a/target/rspamd/local.d/actions.conf b/target/rspamd/local.d/actions.conf index fb4c15b9..6ecb6d4b 100644 --- a/target/rspamd/local.d/actions.conf +++ b/target/rspamd/local.d/actions.conf @@ -6,7 +6,11 @@ # and to be able to explain the impact on the whole system. greylist = 4; add_header = 6; -rewrite_subject = 7; reject = 11; -subject = "***SPAM*** %s" +# The value `null` disabled the action. A subject rewrite is handled by `SPAM_SUBJECT`: +# https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#spam_subject +# +# The reasoning for this can be found in +# https://github.com/docker-mailserver/docker-mailserver/issues/3804 +rewrite_subject = null; diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 56dfa1fb..f18abdf6 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -36,6 +36,7 @@ function _register_functions() { _register_check_function '_check_improper_restart' _register_check_function '_check_hostname' _register_check_function '_check_log_level' + _register_check_function '_check_spam_prefix' # ? >> Setup @@ -48,6 +49,7 @@ function _register_functions() { _register_setup_function '_setup_dovecot_sieve' _register_setup_function '_setup_dovecot_dhparam' _register_setup_function '_setup_dovecot_quota' + _register_setup_function '_setup_spam_subject' _register_setup_function '_setup_spam_to_junk' _register_setup_function '_setup_spam_mark_as_read' fi diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index cf1c40f5..cc98cfe2 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -52,3 +52,11 @@ function _check_log_level() { LOG_LEVEL="${DEFAULT_LOG_LEVEL}" fi } + +function _check_spam_prefix() { + # This check should be independent of ENABLE_POP3 and ENABLE_IMAP + if [[ ${MOVE_SPAM_TO_JUNK} -eq 0 ]] \ + && [[ -z ${SPAM_SUBJECT} ]]; then + _log 'warn' "'MOVE_SPAM_TO_JUNK=0' and 'SPAM_SUBJECT' is empty - make sure this is intended: spam e-mails might not be immediately recognizable in this configuration" + fi +} diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index ec32a1e6..aefeba20 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -81,13 +81,9 @@ function __setup__security__spamassassin() { # shellcheck disable=SC2016 sed -i -r 's|^\$sa_kill_level_deflt (.*);|\$sa_kill_level_deflt = '"${SA_KILL}"';|g' /etc/amavis/conf.d/20-debian_defaults - if [[ ${SA_SPAM_SUBJECT} == 'undef' ]]; then - # shellcheck disable=SC2016 - sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults - else - # shellcheck disable=SC2016 - sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = '"'${SA_SPAM_SUBJECT}'"';|g' /etc/amavis/conf.d/20-debian_defaults - fi + # disable rewriting the subject as this is handles by _setup_spam_subject (which uses Dovecot Sieve) + # shellcheck disable=SC2016 + sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults # activate short circuits when SA BAYES is certain it has spam or ham. if [[ ${SA_SHORTCIRCUIT_BAYES_SPAM} -eq 1 ]]; then @@ -245,6 +241,57 @@ function __setup__security__amavis() { fi } +# If `SPAM_SUBJECT` is not empty, we create a Sieve script that alters the `Subject` +# header, in order to prepend a user-defined string. +function _setup_spam_subject() { + if [[ -z ${SPAM_SUBJECT} ]] + then + _log 'debug' 'Spam subject is not set - no prefix will be added to spam e-mails' + else + _log 'debug' "Spam subject is set - the prefix '${SPAM_SUBJECT}' will be added to spam e-mails" + + _log 'trace' "Enabling '+editheader' Sieve extension" + # check whether sieve_global_extensions is disabled (and enabled it if so) + sed -i -E 's|#(sieve_global_extensions.*)|\1|' /etc/dovecot/conf.d/90-sieve.conf + # then append the extension + sedfile -i -E 's|(sieve_global_extensions.*)|\1 +editheader|' /etc/dovecot/conf.d/90-sieve.conf + + _log 'trace' "Adding global (before) Sieve script for subject rewrite" + # This directory contains Sieve scripts that are executed before user-defined Sieve + # scripts run. + local DOVECOT_SIEVE_GLOBAL_BEFORE_DIR='/usr/lib/dovecot/sieve-global/before' + local DOVECOT_SIEVE_FILE='spam_subject' + readonly DOVECOT_SIEVE_GLOBAL_BEFORE_DIR DOVECOT_SIEVE_FILE + + mkdir -p "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}" + # ref: https://superuser.com/a/1502589 + cat >"${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}.sieve" << EOF +require ["editheader","variables"]; + +if anyof (header :contains "X-Spam-Flag" "YES", + header :contains "X-Spam" "Yes") +{ + # Match the entire subject ... + if header :matches "Subject" "*" { + # ... to get it in a match group that can then be stored in a variable: + set "subject" "\${1}"; + } + + # We can't "replace" a header, but we can delete (all instances of) it and + # re-add (a single instance of) it: + deleteheader "Subject"; + + # Note that the header is added ":last" (so it won't appear before possible + # "Received" headers). + addheader :last "Subject" "${SPAM_SUBJECT}\${subject}"; +} +EOF + + sievec "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}.sieve" + chown dovecot:root "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}."{sieve,svbin} + fi +} + # We can use Sieve to move spam emails to the "Junk" folder. function _setup_spam_to_junk() { if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]]; then diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index d14aeb3f..219ce6c8 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -25,6 +25,12 @@ function __environment_variables_backwards_compatibility() { _log 'error' "The ENV for which LDAP host to connect to must include the URI scheme ('ldap://', 'ldaps://', 'ldapi://')" fi + if [[ -n ${SA_SPAM_SUBJECT:-} ]]; then + _log 'warn' "'SA_SPAM_SUBJECT' has been renamed to 'SPAM_SUBJECT' - this warning will block startup on v15.0.0" + _log 'info' "Copying value of 'SA_SPAM_SUBJECT' into 'SPAM_SUBJECT' if 'SPAM_SUBJECT' has not been set explicitly" + SPAM_SUBJECT=${SPAM_SUBJECT:-${SA_SPAM_SUBJECT}} + fi + # TODO this can be uncommented in a PR handling the HOSTNAME/DOMAINNAME issue # TODO see check_for_changes.sh and dns.sh # if [[ -n ${OVERRIDE_HOSTNAME:-} ]] @@ -67,7 +73,7 @@ function __environment_variables_general_setup() { VARS[RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE]="${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE:=6}" VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="10.0"} - VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} + VARS[SPAM_SUBJECT]=${SPAM_SUBJECT:=} VARS[SA_TAG]=${SA_TAG:="2.0"} VARS[SA_TAG2]=${SA_TAG2:="6.31"} VARS[SPAMASSASSIN_SPAM_TO_INBOX]="${SPAMASSASSIN_SPAM_TO_INBOX:=1}" diff --git a/test/helper/common.bash b/test/helper/common.bash index 3c54862c..35f41283 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -417,5 +417,21 @@ function _nc_wrapper() { _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}" } +# A simple wrapper for a test that checks whether a file exists. +# +# @param ${1} = the path to the file inside the container +function _file_exists_in_container() { + _run_in_container_bash "[[ -f ${1} ]]" + assert_success +} + +# A simple wrapper for a test that checks whether a file does not exist. +# +# @param ${1} = the path to the file (that should not exists) inside the container +function _file_does_not_exist_in_container() { + _run_in_container_bash "[[ -f ${1} ]]" + assert_failure +} + # ? << Miscellaneous helper functions # ! ------------------------------------------------------------------- diff --git a/test/tests/parallel/set1/spam_virus/amavis.bats b/test/tests/parallel/set1/spam_virus/amavis.bats index d7a59cb2..73c5b2ae 100644 --- a/test/tests/parallel/set1/spam_virus/amavis.bats +++ b/test/tests/parallel/set1/spam_virus/amavis.bats @@ -2,8 +2,9 @@ load "${REPOSITORY_ROOT}/test/helper/common" load "${REPOSITORY_ROOT}/test/helper/setup" BATS_TEST_NAME_PREFIX='[Amavis + SA] ' -CONTAINER1_NAME='dms-test_amavis_enabled' -CONTAINER2_NAME='dms-test_amavis_disabled' +CONTAINER1_NAME='dms-test_amavis-enabled-default' +CONTAINER2_NAME='dms-test_amavis-enabled-custom' +CONTAINER3_NAME='dms-test_amavis-disabled' function setup_file() { export CONTAINER_NAME @@ -12,13 +13,25 @@ function setup_file() { _init_with_defaults local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 - --env AMAVIS_LOGLEVEL=2 --env ENABLE_SPAMASSASSIN=1 ) _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' CONTAINER_NAME=${CONTAINER2_NAME} _init_with_defaults + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env AMAVIS_LOGLEVEL=2 + --env ENABLE_SPAMASSASSIN=1 + --env SA_TAG=-5.0 + --env SA_TAG2=2.0 + --env SA_KILL=3.0 + --env SPAM_SUBJECT='***SPAM*** ' + ) + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + + CONTAINER_NAME=${CONTAINER3_NAME} + _init_with_defaults local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=0 --env ENABLE_SPAMASSASSIN=0 @@ -27,11 +40,37 @@ function setup_file() { } function teardown_file() { - docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" "${CONTAINER3_NAME}" } -@test '(Amavis enabled) configuration should be correct' { +@test '(Amavis enabled - defaults) default Amavis config is correct' { export CONTAINER_NAME=${CONTAINER1_NAME} + local AMAVIS_DEFAULTS_FILE='/etc/amavis/conf.d/20-debian_defaults' + + _run_in_container grep 'sa_tag_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_success + assert_output --partial 'sa_tag_level_deflt = 2.0;' + + _run_in_container grep 'sa_tag2_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_success + # shellcheck disable=SC2016 + assert_output --partial '$sa_tag2_level_deflt = 6.31;' + + _run_in_container grep 'sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_success + # shellcheck disable=SC2016 + assert_output --partial '$sa_kill_level_deflt = 10.0;' + + # This feature is handled by our SPAM_SUBJECT ENV through a sieve script instead. + # Thus the feature here should always be disabled via the 'undef' value. + _run_in_container grep 'sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" + assert_success + # shellcheck disable=SC2016 + assert_output --partial '$sa_spam_subject_tag = undef;' +} + +@test '(Amavis enabled - custom) configuration should be correct' { + export CONTAINER_NAME=${CONTAINER2_NAME} _run_in_container postconf -h content_filter assert_success @@ -41,47 +80,47 @@ function teardown_file() { _run_in_container grep -F '127.0.0.1:10025' /etc/postfix/master.cf assert_success - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new.disabled ]]' - assert_failure - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new ]]' - assert_success + _file_does_not_exist_in_container /etc/cron.d/amavisd-new.disabled + _file_exists_in_container /etc/cron.d/amavisd-new } -@test '(Amavis enabled) SA integration should be active' { - export CONTAINER_NAME=${CONTAINER1_NAME} +@test '(Amavis enabled - custom) SA integration should be active' { + export CONTAINER_NAME=${CONTAINER2_NAME} # give Amavis just a bit of time to print out its full debug log run _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep 'SpamControl: init_pre_fork on SpamAssassin done' /var/log/mail/mail.log assert_success } -@test '(Amavis enabled) SA ENV should update Amavis config' { - export CONTAINER_NAME=${CONTAINER1_NAME} - +@test '(Amavis enabled - custom) ENV should update Amavis config' { + export CONTAINER_NAME=${CONTAINER2_NAME} local AMAVIS_DEFAULTS_FILE='/etc/amavis/conf.d/20-debian_defaults' - # shellcheck disable=SC2016 - _run_in_container grep '\$sa_tag_level_deflt' "${AMAVIS_DEFAULTS_FILE}" - assert_success - assert_output --partial '= 2.0' - # shellcheck disable=SC2016 - _run_in_container grep '\$sa_tag2_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + _run_in_container grep 'sa_tag_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success - assert_output --partial '= 6.31' + # shellcheck disable=SC2016 + assert_output --partial '$sa_tag_level_deflt = -5.0;' - # shellcheck disable=SC2016 - _run_in_container grep '\$sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + _run_in_container grep 'sa_tag2_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success - assert_output --partial '= 10.0' + # shellcheck disable=SC2016 + assert_output --partial '$sa_tag2_level_deflt = 2.0;' - # shellcheck disable=SC2016 - _run_in_container grep '\$sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" + _run_in_container grep 'sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success - assert_output --partial "= '***SPAM*** ';" + # shellcheck disable=SC2016 + assert_output --partial '$sa_kill_level_deflt = 3.0;' + + # This feature is handled by our SPAM_SUBJECT ENV through a sieve script instead. + # Thus the feature here should always be disabled via the 'undef' value. + _run_in_container grep 'sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" + assert_success + # shellcheck disable=SC2016 + assert_output --partial '$sa_spam_subject_tag = undef;' } @test '(Amavis disabled) configuration should be correct' { - export CONTAINER_NAME=${CONTAINER2_NAME} + export CONTAINER_NAME=${CONTAINER3_NAME} _run_in_container postconf -h content_filter assert_success @@ -91,8 +130,6 @@ function teardown_file() { _run_in_container grep -F '127.0.0.1:10025' /etc/postfix/master.cf assert_failure - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new.disabled ]]' - assert_success - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new ]]' - assert_failure + _file_exists_in_container /etc/cron.d/amavisd-new.disabled + _file_does_not_exist_in_container /etc/cron.d/amavisd-new } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 044e38fb..20939f03 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -62,8 +62,7 @@ function teardown_file() { _default_teardown ; } assert_output --partial "Supplying a default configuration (to '${SIGNING_CONF_FILE}')" refute_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" assert_output --partial "Finished DKIM key creation" - _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" - assert_success + _file_exists_in_container "${SIGNING_CONF_FILE}" _exec_in_container_bash "echo 'blabla' >${SIGNING_CONF_FILE}" local INITIAL_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") @@ -87,8 +86,7 @@ function teardown_file() { _default_teardown ; } assert_success _count_files_in_directory_in_container /tmp/docker-mailserver/rspamd/dkim/ 3 - _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" - assert_success + _file_exists_in_container "${SIGNING_CONF_FILE}" __check_path_in_signing_config "/tmp/docker-mailserver/rspamd/dkim/rsa-2048-mail-${DOMAIN_NAME}.private.txt" __check_selector_in_signing_config 'mail' @@ -241,8 +239,7 @@ function __check_rsa_keys() { function __check_key_files_are_present() { local BASE_FILE_NAME="${1:?Base file name must be supplied to __check_key_files_are_present}" for FILE in ${BASE_FILE_NAME}.{public.txt,public.dns.txt,private.txt}; do - _run_in_container_bash "[[ -f ${FILE} ]]" - assert_success + _file_exists_in_container "${FILE}" done } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 9e6d6222..9b14860b 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -29,6 +29,7 @@ function setup_file() { --env RSPAMD_GREYLISTING=1 --env RSPAMD_HFILTER=1 --env RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=7 + --env SPAM_SUBJECT='[POTENTIAL SPAM] ' ) cp -r "${TEST_TMP_CONFIG}"/rspamd_full/* "${TEST_TMP_CONFIG}/" @@ -43,7 +44,7 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - # We will send 4 emails: + # We will send 5 emails: # 1. The first one should pass just fine _send_email_with_msgid 'rspamd-test-email-pass' # 2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection) @@ -56,6 +57,9 @@ function setup_file() { # ref: https://rspamd.com/doc/gtube_patterns.html _send_email_with_msgid 'rspamd-test-email-header' \ --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" + # 5. The fifth one will have its subject rewritten, but now spam header is applied. + _send_email_with_msgid 'rspamd-test-email-rewrite_subject' \ + --body "ZJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" _run_in_container cat /var/log/mail.log assert_success @@ -73,11 +77,18 @@ function teardown_file() { _default_teardown ; } assert_output 'rspamd_milter = inet:localhost:11332' } +@test 'Rspamd base configuration is correct' { + _run_in_container rspamadm configdump actions + assert_success + assert_line 'greylist = 4;' + assert_line 'reject = 11;' + assert_line 'add_header = 6;' + refute_line --regexp 'rewrite_subject = [0-9]+;' +} + @test "contents of '/etc/rspamd/override.d/' are copied" { local OVERRIDE_D='/etc/rspamd/override.d' - - _run_in_container_bash "[[ -f ${OVERRIDE_D}/testmodule_complicated.conf ]]" - assert_success + _file_exists_in_container "${OVERRIDE_D}/testmodule_complicated.conf" } @test 'startup log shows all features as properly enabled' { @@ -89,6 +100,7 @@ function teardown_file() { _default_teardown ; } assert_line --partial 'Enabling greylisting' assert_line --partial 'Hfilter (group) module is enabled' assert_line --partial "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to" + assert_line --partial "Spam subject is set - the prefix '[POTENTIAL SPAM] ' will be added to spam e-mails" assert_line --partial "Found file '/tmp/docker-mailserver/rspamd/custom-commands.conf' - parsing and applying it" } @@ -114,7 +126,7 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_msgid 'rspamd-test-email-pass' assert_output --partial "stored mail into mailbox 'INBOX'" - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 } @test 'detects and rejects spam' { @@ -129,7 +141,7 @@ function teardown_file() { _default_teardown ; } refute_output --partial "stored mail into mailbox 'INBOX'" assert_failure - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 } @test 'detects and rejects virus' { @@ -144,14 +156,13 @@ function teardown_file() { _default_teardown ; } refute_output --partial "stored mail into mailbox 'INBOX'" assert_failure - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 } @test 'custom commands work correctly' { # check `testmodule1` which should be disabled local MODULE_PATH='/etc/rspamd/override.d/testmodule1.conf' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F '# documentation: https://rspamd.com/doc/modules/testmodule1.html' "${MODULE_PATH}" assert_success _run_in_container grep -F 'enabled = false;' "${MODULE_PATH}" @@ -161,8 +172,7 @@ function teardown_file() { _default_teardown ; } # check `testmodule2` which should be enabled and it should have extra options set MODULE_PATH='/etc/rspamd/override.d/testmodule2.conf' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F '# documentation: https://rspamd.com/doc/modules/testmodule2.html' "${MODULE_PATH}" assert_success _run_in_container grep -F 'enabled = true;' "${MODULE_PATH}" @@ -181,8 +191,7 @@ function teardown_file() { _default_teardown ; } # check whether adding a single line writes the line properly in `testmodule4.something` MODULE_PATH='/etc/rspamd/override.d/testmodule4.something' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" # shellcheck disable=SC2016 _run_in_container grep -F 'some very long line with "weird $charact"ers' "${MODULE_PATH}" assert_success @@ -193,37 +202,31 @@ function teardown_file() { _default_teardown ; } # check whether spaces in front of options are handles properly in `testmodule_complicated` MODULE_PATH='/etc/rspamd/override.d/testmodule_complicated.conf' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F ' anOption = anotherValue;' "${MODULE_PATH}" # check whether controller option was written properly MODULE_PATH='/etc/rspamd/override.d/worker-controller.inc' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F 'someOption = someValue42;' "${MODULE_PATH}" assert_success # check whether controller option was written properly MODULE_PATH='/etc/rspamd/override.d/worker-proxy.inc' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F 'abcdefg71 = RAAAANdooM;' "${MODULE_PATH}" assert_success # check whether basic options are written properly MODULE_PATH='/etc/rspamd/override.d/options.inc' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F 'OhMy = "PraiseBeLinters !";' "${MODULE_PATH}" assert_success } @test 'MOVE_SPAM_TO_JUNK works for Rspamd' { - _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve ]]' - assert_success - _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]' - assert_success + _file_exists_in_container /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve + _file_exists_in_container /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin _service_log_should_contain_string 'rspamd' 'S (add header)' _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' @@ -231,14 +234,38 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_msgid 'rspamd-test-email-header' assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1 } +@test 'Rewriting subject works when enforcing it via GTUBE' { + _service_log_should_contain_string 'rspamd' 'S (rewrite subject)' + _service_log_should_contain_string 'rspamd' 'rewrite subject "Gtube pattern"' + + _print_mail_log_for_msgid 'rspamd-test-email-rewrite_subject' + assert_output --partial "stored mail into mailbox 'INBOX'" + + # check that the inbox contains the subject-rewritten e-mail + _run_in_container_bash "grep --fixed-strings 'Subject: *** SPAM ***' /var/mail/localhost.localdomain/user1/new/*" + assert_success + + # check that the inbox contains the normal e-mail (that passes just fine) + _run_in_container_bash "grep --fixed-strings 'Subject: test' /var/mail/localhost.localdomain/user1/new/*" + assert_success +} + +@test 'SPAM_SUBJECT works' { + _file_exists_in_container /usr/lib/dovecot/sieve-global/before/spam_subject.sieve + _file_exists_in_container /usr/lib/dovecot/sieve-global/before/spam_subject.svbin + + # we only have one e-mail in the junk folder, hence using '*' is fine + _run_in_container_bash "grep --fixed-strings 'Subject: [POTENTIAL SPAM]' /var/mail/localhost.localdomain/user1/.Junk/new/*" + assert_success +} + @test 'RSPAMD_LEARN works' { for FILE in learn-{ham,spam}.{sieve,svbin}; do - _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" - assert_success + _file_exists_in_container "/usr/lib/dovecot/sieve-pipe/${FILE}" done _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf @@ -305,8 +332,7 @@ function teardown_file() { _default_teardown ; } @test 'hfilter group module is configured correctly' { local MODULE_FILE='/etc/rspamd/local.d/hfilter_group.conf' - _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" - assert_success + _file_exists_in_container "${MODULE_FILE}" _run_in_container grep '__TAG__HFILTER_HOSTNAME_UNKNOWN' "${MODULE_FILE}" assert_success @@ -315,8 +341,7 @@ function teardown_file() { _default_teardown ; } @test 'checks on authenticated users are disabled' { local MODULE_FILE='/etc/rspamd/local.d/settings.conf' - _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" - assert_success + _file_exists_in_container "${MODULE_FILE}" _run_in_container grep -E -A 6 'authenticated \{' "${MODULE_FILE}" assert_success diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats index 9fc8af31..b647f562 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -55,6 +55,7 @@ function teardown_file() { _default_teardown ; } assert_line --partial 'Intelligent learning of spam and ham is disabled' assert_line --partial 'Greylisting is disabled' assert_line --partial 'Disabling Hfilter (group) module' + assert_line --partial 'Spam subject is not set' } @test 'antivirus maximum size was not adjusted unnecessarily' { @@ -64,8 +65,7 @@ function teardown_file() { _default_teardown ; } @test 'learning is properly disabled' { for FILE in learn-{ham,spam}.{sieve,svbin}; do - _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" - assert_failure + _file_does_not_exist_in_container "/usr/lib/dovecot/sieve-pipe/${FILE}" done _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf @@ -83,14 +83,13 @@ function teardown_file() { _default_teardown ; } } @test 'hfilter group module configuration is deleted' { - _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' + _file_does_not_exist_in_container /etc/rspamd/local.d/hfilter_group.conf assert_failure } @test 'checks on authenticated users are enabled' { local MODULE_FILE='/etc/rspamd/local.d/settings.conf' - _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" - assert_success + _file_exists_in_container "${MODULE_FILE}" _run_in_container grep -E 'authenticated \{' "${MODULE_FILE}" assert_failure diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 15ec4fe1..0c873382 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -34,7 +34,7 @@ function teardown() { _default_teardown ; } local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT="SPAM: " + --env SPAM_SUBJECT="SPAM: " --env SPAMASSASSIN_SPAM_TO_INBOX=1 --env MOVE_SPAM_TO_JUNK=0 --env PERMIT_DOCKER=container @@ -55,7 +55,7 @@ function teardown() { _default_teardown ; } local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT="SPAM: " + --env SPAM_SUBJECT="SPAM: " --env SPAMASSASSIN_SPAM_TO_INBOX=1 --env MOVE_SPAM_TO_JUNK=1 --env PERMIT_DOCKER=container @@ -77,7 +77,7 @@ function teardown() { _default_teardown ; } local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT="SPAM: " + --env SPAM_SUBJECT="SPAM: " --env SPAMASSASSIN_SPAM_TO_INBOX=1 --env MARK_SPAM_AS_READ=1 --env PERMIT_DOCKER=container diff --git a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats b/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats deleted file mode 100644 index 35260d51..00000000 --- a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats +++ /dev/null @@ -1,61 +0,0 @@ -load "${REPOSITORY_ROOT}/test/helper/setup" -load "${REPOSITORY_ROOT}/test/helper/common" - -BATS_TEST_NAME_PREFIX='[Spam] (undefined subject) ' - -CONTAINER1_NAME='dms-test_spam-undef-subject_1' -CONTAINER2_NAME='dms-test_spam-undef-subject_2' -CONTAINER_NAME=${CONTAINER2_NAME} - -function teardown() { _default_teardown ; } - -@test "'SA_SPAM_SUBJECT=undef' should update Amavis config" { - export CONTAINER_NAME=${CONTAINER1_NAME} - local CUSTOM_SETUP_ARGUMENTS=( - --env ENABLE_AMAVIS=1 - --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT='undef' - ) - _init_with_defaults - _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - - _run_in_container_bash "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'" - assert_success -} - -# TODO: Unclear why some of these ENV are relevant for the test? -@test "Docker env variables are set correctly (custom)" { - export CONTAINER_NAME=${CONTAINER2_NAME} - - local CUSTOM_SETUP_ARGUMENTS=( - --env ENABLE_CLAMAV=1 - --env SPOOF_PROTECTION=1 - --env ENABLE_SPAMASSASSIN=1 - --env REPORT_RECIPIENT=user1@localhost.localdomain - --env REPORT_SENDER=report1@mail.my-domain.com - --env SA_TAG=-5.0 - --env SA_TAG2=2.0 - --env SA_KILL=3.0 - --env SA_SPAM_SUBJECT="SPAM: " - --env VIRUSMAILS_DELETE_DELAY=7 - --env ENABLE_SRS=1 - --env ENABLE_MANAGESIEVE=1 - --env PERMIT_DOCKER=host - # NOTE: ulimit required for `ENABLE_SRS=1` until running a newer `postsrsd` - --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" - ) - _init_with_defaults - _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - - _run_in_container_bash "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= -5.0'" - assert_success - - _run_in_container_bash "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'" - assert_success - - _run_in_container_bash "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'" - assert_success - - _run_in_container_bash "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= .SPAM: .'" - assert_success -} From 2018be7fdc04ce313667d365427fa9e79b4db2eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:08:59 +0100 Subject: [PATCH 15/17] chore(deps): Bump anchore/scan-action from 3.6.0 to 3.6.1 (#3848) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.6.0 to 3.6.1. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.6.0...v3.6.1) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b261de91..cb1a5994 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.6.0 + uses: anchore/scan-action@v3.6.1 id: scan with: image: mailserver-testing:ci From 4162d608e40370198662e7006ce6d5a81b94671c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 29 Jan 2024 22:10:03 +0100 Subject: [PATCH 16/17] Rspamd scripts: only correct permissions when directory exists (#3849) --- target/scripts/startup/setup-stack.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 24ef1581..81e1a98e 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -97,8 +97,10 @@ function _setup_apply_fixes_after_configuration() { _rspamd_get_envs # /tmp/docker-mailserver/rspamd/dkim - _log 'debug' "Ensuring '${RSPAMD_DMS_DKIM_D}' is owned by '_rspamd:_rspamd'" - chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" + if [[ -d ${RSPAMD_DMS_DKIM_D} ]]; then + _log 'debug' "Ensuring '${RSPAMD_DMS_DKIM_D}' is owned by '_rspamd:_rspamd'" + chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" + fi } function _run_user_patches() { From 244c455ca1b92371a5c29a47ce28588f3eb5baf7 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:11:45 +1300 Subject: [PATCH 17/17] fix: `packages.sh` - Download `jaq` via release `tag` not `latest` (#3852) As the filename includes the version / tag, we cannot rely on the latest URL to be stable. --- target/scripts/build/packages.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 24dae8a7..afff4cad 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -38,7 +38,8 @@ function _pre_installation_steps() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' _log 'trace' 'Installing jaq' - curl -sSfL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq + local JAQ_TAG='v1.3.0' + curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-${JAQ_TAG}-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq _log 'trace' 'Installing swaks' local SWAKS_VERSION='20240103.0'