From 303ca82fb925ab9a0e0aa8bf68a2caa402afe0aa Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 01:02:19 +1300 Subject: [PATCH 1/6] docs(fix): New external link icon workaround for mkdocs-material 9.5.5 (#3823) This is the easiest to maintain workaround now available. Upstream continues to reject the value such a feature for accessibility. --- docs/content/assets/css/customizations.css | 8 ++++++-- docs/mkdocs.yml | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/content/assets/css/customizations.css b/docs/content/assets/css/customizations.css index 49ede009..320da152 100644 --- a/docs/content/assets/css/customizations.css +++ b/docs/content/assets/css/customizations.css @@ -16,12 +16,16 @@ If you want to append instead, switch `::before` to `::after`. src: url('../fonts/external-link.woff') format('woff'); } -/* Matches the two nav link classes that start with `http` `href` values, regular docs pages use relative URLs instead. */ -.md-tabs__link[href^="http"]::before, .md-nav__link[href^="http"]::before { +/* + Since mkdocs-material 9.5.5 broke support in our docs from DMS v13.3.1, we now use our own class name, + which has been included for the two external nav links in mkdocs.yml via workaround (insert HTML). +*/ +.icon-external-link::before { display: inline-block; /* treat similar to text */ font-family: 'external-link'; content:'\0041'; /* represents "A" which our font renders as an icon instead of the "A" glyph */ font-size: 80%; /* icon is a little too big by default, scale it down */ + margin-right: 4px; } /* ============================================================================================================= */ diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 0d34b407..15d2c3b2 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -173,5 +173,5 @@ nav: - 'General Information': contributing/general.md - 'Tests': contributing/tests.md - 'Issues and Pull Requests': contributing/issues-and-pull-requests.md - - 'DockerHub': https://hub.docker.com/r/mailserver/docker-mailserver/ - - 'GHCR': https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver + - 'DockerHub': https://hub.docker.com/r/mailserver/docker-mailserver/ + - 'GHCR': https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver From 22c6daee32ad0769002564054654a4e989c9eb10 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 01:21:24 +1300 Subject: [PATCH 2/6] chore: Revise improper restart message (#3826) Improved guidance. --- target/scripts/startup/check-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index 61f21d1c..cf1c40f5 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -19,7 +19,7 @@ function _check_improper_restart() { if [[ -f /CONTAINER_START ]]; then _log 'warn' 'This container was (likely) improperly restarted which can result in undefined behavior' - _log 'warn' 'Please destroy the container properly and then start DMS again' + _log 'warn' "Please use 'docker compose up --force-recreate' or equivalent (view our troubleshooting docs)" fi } From 47f8d50bebaa0b1f1cb85d45c1d9210d9ce80a49 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:28:26 +1300 Subject: [PATCH 3/6] fix: Ensure configs are sanitized for parsing (#3819) * chore: Detect missing final newline in configs read These lines will be not be processed by `read`, emit a warning to raise awareness. * fix: Ensure parsed config has final newline appended (when possible) This functionality was handled in `accounts.sh` via a similar sed command (that the linked references also offer). `printf` is better for this, no shellcheck comment required either. We additionally don't attempt to modify files that are read-only. * fix: Ensure parsed configs have CRLF to LF corrected (where possible) Likewise, this runtime fix was only covering two config files. It now applies to all callers of this method. * fix: Sanitize `postfix-master.cf` via helper This feature should have been using the helper to avoid user error from their config updates accidentally introducing subtle breakage implicitly (due to CRLF or missing final newline). * tests: Add test cases for new helpers * tests: `rm` is redundant when using `BATS_TEST_TMPDIR` This temporary directory is created and removed implicitly. Even after a test failure. * chore: Remove old `postfix-virtual.cf` migration logic This was introduced in 2018, there should be no one needing to rely on this anymore? * tests: Remove comment on sed failure concern * chore: Add entry to `CHANGELOG.md` * Apply suggestions from code review Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 4 ++ target/scripts/helpers/accounts.sh | 14 ------- target/scripts/helpers/aliases.sh | 5 --- target/scripts/helpers/utils.sh | 35 ++++++++++++++++ target/scripts/startup/setup.d/postfix.sh | 22 +++++----- test/helper/setup.bash | 1 - .../set3/scripts/helper_functions.bats | 42 +++++++++++++++++++ 7 files changed, 92 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f07561..9f440e31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,10 @@ The most noteworthy change of this release is the update of the container's base - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) +### Fixes + +- DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) + ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) ### Fixes diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 7499a2ec..31ded04a 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -19,16 +19,9 @@ function _create_accounts() { _create_masters if [[ -f ${DATABASE_ACCOUNTS} ]]; then - _log 'trace' "Checking file line endings" - sed -i 's|\r||g' "${DATABASE_ACCOUNTS}" - _log 'trace' "Regenerating postfix user list" echo "# WARNING: this file is auto-generated. Modify ${DATABASE_ACCOUNTS} to edit the user list." > /etc/postfix/vmailbox - # checking that ${DATABASE_ACCOUNTS} ends with a newline - # shellcheck disable=SC1003 - sed -i -e '$a\' "${DATABASE_ACCOUNTS}" - chown dovecot:dovecot "${DOVECOT_USERDB_FILE}" chmod 640 "${DOVECOT_USERDB_FILE}" @@ -158,15 +151,8 @@ function _create_masters() { local DATABASE_DOVECOT_MASTERS='/tmp/docker-mailserver/dovecot-masters.cf' if [[ -f ${DATABASE_DOVECOT_MASTERS} ]]; then - _log 'trace' "Checking file line endings" - sed -i 's|\r||g' "${DATABASE_DOVECOT_MASTERS}" - _log 'trace' "Regenerating dovecot masters list" - # checking that ${DATABASE_DOVECOT_MASTERS} ends with a newline - # shellcheck disable=SC1003 - sed -i -e '$a\' "${DATABASE_DOVECOT_MASTERS}" - chown dovecot:dovecot "${DOVECOT_MASTERDB_FILE}" chmod 640 "${DOVECOT_MASTERDB_FILE}" diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 04a56da3..b0f2fa1a 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -12,11 +12,6 @@ function _handle_postfix_virtual_config() { local DATABASE_VIRTUAL=/tmp/docker-mailserver/postfix-virtual.cf if [[ -f ${DATABASE_VIRTUAL} ]]; then - # fixing old virtual user file - if grep -q ",$" "${DATABASE_VIRTUAL}"; then - sed -i -e "s|, |,|g" -e "s|,$||g" "${DATABASE_VIRTUAL}" - fi - cp -f "${DATABASE_VIRTUAL}" /etc/postfix/virtual else _log 'debug' "'${DATABASE_VIRTUAL}' not provided - no mail alias/forward created" diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index f7095bf3..c848e13e 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -17,9 +17,44 @@ function _escape_for_sed() { # Returns input after filtering out lines that are: # empty, white-space, comments (`#` as the first non-whitespace character) function _get_valid_lines_from_file() { + _convert_crlf_to_lf_if_necessary "${1}" + _append_final_newline_if_missing "${1}" + grep --extended-regexp --invert-match "^\s*$|^\s*#" "${1}" || true } +# This is to sanitize configs from users that unknowingly introduced CRLF: +function _convert_crlf_to_lf_if_necessary() { + if [[ $(file "${1}") =~ 'CRLF' ]]; then + _log 'warn' "File '${1}' contains CRLF line-endings" + + if [[ -w ${1} ]]; then + _log 'debug' 'Converting CRLF to LF' + sed -i 's|\r||g' "${1}" + else + _log 'warn' "File '${1}' is not writable - cannot change CRLF to LF" + fi + fi +} + +# This is to sanitize configs from users that unknowingly removed the end-of-file LF: +function _append_final_newline_if_missing() { + # Correctly detect a missing final newline and fix it: + # https://stackoverflow.com/questions/38746/how-to-detect-file-ends-in-newline#comment82380232_25749716 + # https://unix.stackexchange.com/questions/31947/how-to-add-a-newline-to-the-end-of-a-file/441200#441200 + # https://unix.stackexchange.com/questions/159557/how-to-non-invasively-test-for-write-access-to-a-file + if [[ $(tail -c1 "${1}" | wc -l) -eq 0 ]]; then + # Avoid fixing when the destination is read-only: + if [[ -w ${1} ]]; then + printf '\n' >> "${1}" + + _log 'info' "File '${1}' was missing a final newline - this has been fixed" + else + _log 'warn' "File '${1}' is missing a final newline - it is not writable, hence it was not fixed - the last line will not be processed!" + fi + fi +} + # Provide the name of an environment variable to this function # and it will return its value stored in /etc/dms-settings function _get_dms_env_value() { diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 5aec8636..05052faa 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -109,8 +109,9 @@ function _setup_postfix_late() { function __postfix__setup_override_configuration() { __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values' - if [[ -f /tmp/docker-mailserver/postfix-main.cf ]]; then - cat /tmp/docker-mailserver/postfix-main.cf >>/etc/postfix/main.cf + local OVERRIDE_CONFIG_POSTFIX_MAIN='/tmp/docker-mailserver/postfix-main.cf' + if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MAIN} ]]; then + cat "${OVERRIDE_CONFIG_POSTFIX_MAIN}" >>/etc/postfix/main.cf _adjust_mtime_for_postfix_maincf # do not directly output to 'main.cf' as this causes a read-write-conflict @@ -118,20 +119,19 @@ function __postfix__setup_override_configuration() { mv /tmp/postfix-main-new.cf /etc/postfix/main.cf _adjust_mtime_for_postfix_maincf - __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '/tmp/docker-mailserver/postfix-main.cf'" + __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MAIN}'" else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-main.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MAIN}' was not provided" fi - if [[ -f /tmp/docker-mailserver/postfix-master.cf ]]; then + local OVERRIDE_CONFIG_POSTFIX_MASTER='/tmp/docker-mailserver/postfix-master.cf' + if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MASTER} ]]; then while read -r LINE; do - if [[ ${LINE} =~ ^[0-9a-z] ]]; then - postconf -P "${LINE}" - fi - done < /tmp/docker-mailserver/postfix-master.cf - __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '/tmp/docker-mailserver/postfix-master.cf'" + [[ ${LINE} =~ ^[0-9a-z] ]] && postconf -P "${LINE}" + done < <(_get_valid_lines_from_file "${OVERRIDE_CONFIG_POSTFIX_MASTER}") + __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MASTER}'" else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-master.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MASTER}' was not provided" fi } diff --git a/test/helper/setup.bash b/test/helper/setup.bash index 0dd57bd6..ed2e4e32 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -102,7 +102,6 @@ function _init_with_defaults() { # The config volume cannot be read-only as some data needs to be written at container startup # - # - two sed failures (unknown lines) # - dovecot-quotas.cf (setup-stack.sh:_setup_dovecot_quotas) # - postfix-aliases.cf (setup-stack.sh:_setup_postfix_aliases) # TODO: Check how many tests need write access. Consider using `docker create` + `docker cp` for easier cleanup. diff --git a/test/tests/parallel/set3/scripts/helper_functions.bats b/test/tests/parallel/set3/scripts/helper_functions.bats index 332de448..518f8717 100644 --- a/test/tests/parallel/set3/scripts/helper_functions.bats +++ b/test/tests/parallel/set3/scripts/helper_functions.bats @@ -70,3 +70,45 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/ assert_failure assert_output --partial "ENV var name must be provided to _env_var_expect_integer" } + +@test '(utils.sh) _convert_crlf_to_lf_if_necessary' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh + source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh + source "${SOURCE_BASE_PATH}/utils.sh" + + # Create a temporary file in the BATS test-case folder: + local TMP_DMS_CONFIG=$(mktemp -p "${BATS_TEST_TMPDIR}" -t 'dms_XXX.cf') + # A file with mixed line-endings including CRLF: + echo -en 'line one\nline two\r\n' > "${TMP_DMS_CONFIG}" + + # Confirm CRLF detected: + run file "${TMP_DMS_CONFIG}" + assert_output --partial 'CRLF' + + # Helper method detects and fixes: + _convert_crlf_to_lf_if_necessary "${TMP_DMS_CONFIG}" + run file "${TMP_DMS_CONFIG}" + refute_output --partial 'CRLF' +} + +@test '(utils.sh) _append_final_newline_if_missing' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh + source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh + source "${SOURCE_BASE_PATH}/utils.sh" + + # Create a temporary file in the BATS test-case folder: + local TMP_DMS_CONFIG=$(mktemp -p "${BATS_TEST_TMPDIR}" -t 'dms_XXX.cf') + # A file missing a final newline: + echo -en 'line one\nline two' > "${TMP_DMS_CONFIG}" + + # Confirm missing newline: + run bash -c "tail -c 1 '${TMP_DMS_CONFIG}' | wc -l" + assert_output '0' + + # Helper method detects and fixes: + _append_final_newline_if_missing "${TMP_DMS_CONFIG}" + run bash -c "tail -c 1 '${TMP_DMS_CONFIG}' | wc -l" + assert_output '1' +} From 487867285bd1c7c6d3157afde45adce08d3c0409 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:32:18 +1300 Subject: [PATCH 4/6] docs: UX Improvement - Better distinguish side nav page categories (#3835) --- docs/content/assets/css/customizations.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/content/assets/css/customizations.css b/docs/content/assets/css/customizations.css index 320da152..25cb0274 100644 --- a/docs/content/assets/css/customizations.css +++ b/docs/content/assets/css/customizations.css @@ -102,3 +102,8 @@ div.md-content article.md-content__inner a.toclink code { .highlight.no-copy .md-clipboard { display: none; } /* ============================================================================================================= */ + +/* Make the left-sidebar nav categories better distinguished from page links (bold text) */ +.md-nav__item--nested > .md-nav__link { + font-weight: 700; +} From 7d9eb1e4a7ddd7b78b7bda2928b802af990af5ec Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:32:49 +1300 Subject: [PATCH 5/6] docs: Add context to `sender-cleanup` in Postfix `master.cf` (#3834) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/postfix/master.cf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/postfix/master.cf b/target/postfix/master.cf index e5b955a4..7d2b21d6 100644 --- a/target/postfix/master.cf +++ b/target/postfix/master.cf @@ -46,6 +46,8 @@ pickup fifo n - n 60 1 pickup -o content_filter= -o receive_override_options=no_header_body_checks +# This relates to submission(s) services defined above: +# https://www.postfix.org/BUILTIN_FILTER_README.html#mx_submission sender-cleanup unix n - n - 0 cleanup -o syslog_name=postfix/sender-cleanup -o header_checks=pcre:/etc/postfix/maps/sender_header_filter.pcre From ba27edc801c0b94cfb7d975dd2fb918a75ac9ea3 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:07:46 +0100 Subject: [PATCH 6/6] Rspamd: only declare Rspamd variables when not already declared (#3837) * only declare Rspamd vars when not already declared * update CHANGELOG * Update CHANGELOG.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 1 + target/scripts/helpers/rspamd.sh | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f440e31..a9c03f79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The most noteworthy change of this release is the update of the container's base ### Fixes - DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) +- Variables related to Rspamd are declared as `readonly`, which would cause warnings in the log when being re-declared; we now guard against this issue ([#3837](https://github.com/docker-mailserver/docker-mailserver/pull/3837)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh index 2f4dcc46..8d1fd668 100644 --- a/target/scripts/helpers/rspamd.sh +++ b/target/scripts/helpers/rspamd.sh @@ -15,14 +15,19 @@ function __do_as_rspamd_user() { # they cannot be modified. Use this function when you require common directory # names, file names, etc. function _rspamd_get_envs() { - readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' - readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + # If the variables are already set, we cannot set them again as they are declared + # with `readonly`. Checking whether one is declared suffices, because either all + # are declared at once, or none. + if [[ ! -v RSPAMD_LOCAL_D ]]; then + readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' + readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" - readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" + readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" - readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" + readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" + fi } # Parses `RSPAMD_DMS_CUSTOM_COMMANDS_F` and executed the directives given by the file.