Merge pull request #1596 from mwnx/relayhostmap-from-virtual

Populate relayhost_map from virtual accounts (and other fixes)

target/check-for-changes.sh

@@ -16,7 +16,6 @@ if [ ! -f postfix-accounts.cf ]; then
 fi
 
 # Verify checksum file exists; must be prepared by start-mailserver.sh
-CHKSUM_FILE=/tmp/docker-mailserver-config-chksum
 if [ ! -f $CHKSUM_FILE ]; then
    echo "${log_date} ${CHKSUM_FILE} is missing! Start script failed? Exit!"
    exit
@@ -32,12 +31,6 @@ fi
 PM_ADDRESS="${POSTMASTER_ADDRESS:=postmaster@${DOMAINNAME}}"
 echo "${log_date} Using postmaster address ${PM_ADDRESS}"
 
-# Create an array of files to monitor, must be the same as in start-mailserver.sh
-declare -a cf_files=()
-for file in postfix-accounts.cf postfix-virtual.cf postfix-aliases.cf dovecot-quotas.cf /etc/letsencrypt/acme.json "/etc/letsencrypt/live/$HOSTNAME/key.pem" "/etc/letsencrypt/live/$HOSTNAME/fullchain.pem"; do
-  [ -f "$file" ] && cf_files+=("$file")
-done
-
 # Wait to make sure server is up before we start
 sleep 10
 
@@ -48,10 +41,12 @@ while true; do
 log_date=$(date +"%Y-%m-%d %H:%M:%S ")
 
 # Get chksum and check it, no need to lock config yet
-chksum=$(sha512sum -c --ignore-missing $CHKSUM_FILE)
+monitored_files_checksums >"$CHKSUM_FILE.new"
 
-if [[ $chksum == *"FAIL"* ]]; then
+if ! cmp --silent -- "$CHKSUM_FILE" "$CHKSUM_FILE.new"; then
 	echo "${log_date} Change detected"
+  changed=$(grep -Fxvf "$CHKSUM_FILE" "$CHKSUM_FILE.new" | sed 's/^[^ ]\+  //')
+  mv "$CHKSUM_FILE.new" "$CHKSUM_FILE"
 
 	# Bug alert! This overwrites the alias set by start-mailserver.sh
 	# Take care that changes in one script are propagated to the other
@@ -63,13 +58,18 @@ if [[ $chksum == *"FAIL"* ]]; then
         (
           flock -e 200
 
-  if [[ $chksum == *"/etc/letsencrypt/acme.json: FAILED"* ]]; then
+  for file in $changed; do
-    for certdomain in $SSL_DOMAIN $HOSTNAME $DOMAINNAME; do
+    case $file in
-      if extractCertsFromAcmeJson "$certdomain"; then
+    /etc/letsencrypt/acme.json)
-        break
+      for certdomain in $SSL_DOMAIN $HOSTNAME $DOMAINNAME; do
-      fi
+        if extractCertsFromAcmeJson "$certdomain"; then
-    done
+          break
-  fi
+        fi
+      done
+      ;;
+    #TODO: Perform updates below conditionally as well.
+    esac
+  done
 
 	#regen postix aliases.
 	echo "root: ${PM_ADDRESS}" > /etc/aliases
@@ -81,6 +81,7 @@ if [[ $chksum == *"FAIL"* ]]; then
 	#regen postfix accounts.
 	echo -n > /etc/postfix/vmailbox
 	echo -n > /etc/dovecot/userdb
+
 	if [ -f /tmp/docker-mailserver/postfix-accounts.cf -a "$ENABLE_LDAP" != 1 ]; then
 		sed -i 's/\r//g' /tmp/docker-mailserver/postfix-accounts.cf
 		echo "# WARNING: this file is auto-generated. Modify config/postfix-accounts.cf to edit user list." > /etc/postfix/vmailbox
@@ -95,7 +96,6 @@ if [[ $chksum == *"FAIL"* ]]; then
 		if [ ! -z "$RELAY_HOST" ]; then
 			# keep old config
 			echo -n > /etc/postfix/sasl_passwd
-			echo -n > /etc/postfix/relayhost_map
 			if [ ! -z "$SASL_PASSWD" ]; then
 				echo "$SASL_PASSWD" >> /etc/postfix/sasl_passwd
 			fi
@@ -111,14 +111,6 @@ if [[ $chksum == *"FAIL"* ]]; then
 			if [ ! -z "$RELAY_USER" ] && [ ! -z "$RELAY_PASSWORD" ]; then
 				echo "[$RELAY_HOST]:$RELAY_PORT		$RELAY_USER:$RELAY_PASSWORD" >> /etc/postfix/sasl_passwd
 			fi
-			# add relay maps from file
-			if [ -f /tmp/docker-mailserver/postfix-relaymap.cf ]; then
-				(grep -v "^\s*$\|^\s*\#" /tmp/docker-mailserver/postfix-relaymap.cf || true) | while read line; do
-					if ! echo "$line" | grep -q -e "\s*#"; then
-						echo "$line" >> /etc/postfix/relayhost_map
-					fi
-				done
-			fi
 		fi
 
 		# Creating users
@@ -152,22 +144,15 @@ if [[ $chksum == *"FAIL"* ]]; then
 			# Copy user provided sieve file, if present
 			test -e /tmp/docker-mailserver/${login}.dovecot.sieve && cp /tmp/docker-mailserver/${login}.dovecot.sieve /var/mail/${domain}/${user}/.dovecot.sieve
 			echo ${domain} >> /tmp/vhost.tmp
-			# add domains to relayhost_map
-			if [ ! -z "$RELAY_HOST" ]; then
-				if ! grep -q -e "^@${domain}\s" /etc/postfix/relayhost_map; then
-					echo "@${domain}		[$RELAY_HOST]:$RELAY_PORT" >> /etc/postfix/relayhost_map
-				fi
-			fi
 		done
 	fi
+	if [ ! -z "$RELAY_HOST" ]; then
+		populate_relayhost_map
+	fi
 	if [ -f /etc/postfix/sasl_passwd ]; then
 		chown root:root /etc/postfix/sasl_passwd
 		chmod 0600 /etc/postfix/sasl_passwd
 	fi
-	if [ -f /etc/postfix/relayhost_map ]; then
-		chown root:root /etc/postfix/relayhost_map
-		chmod 0600 /etc/postfix/relayhost_map
-	fi
 	if [ -f postfix-virtual.cf ]; then
 	# regen postfix aliases
 	echo -n > /etc/postfix/virtual
@@ -211,9 +196,6 @@ if [[ $chksum == *"FAIL"* ]]; then
 		supervisorctl restart dovecot
 	fi
 
-	echo "${log_date} Update checksum"
-	sha512sum ${cf_files[@]/#/--tag } >$CHKSUM_FILE
-
         ) 200<postfix-accounts.cf # end lock
 fi
 

target/helper_functions.sh

@@ -73,3 +73,113 @@ for key, value in acme.items():
     return 1
   fi
 }
+
+declare -A DEFAULT_VARS
+DEFAULT_VARS["DMS_DEBUG"]="${DMS_DEBUG:="0"}"
+
+function notify () {
+  c_red="\e[0;31m"
+  c_green="\e[0;32m"
+  c_brown="\e[0;33m"
+  c_blue="\e[0;34m"
+  c_bold="\033[1m"
+  c_reset="\e[0m"
+
+  notification_type=$1
+  notification_msg=$2
+  notification_format=$3
+  msg=""
+
+  case "${notification_type}" in
+    'taskgrp')
+      msg="${c_bold}${notification_msg}${c_reset}"
+      ;;
+    'task')
+      if [[ ${DEFAULT_VARS["DMS_DEBUG"]} == 1 ]]; then
+        msg="  ${notification_msg}${c_reset}"
+      fi
+      ;;
+    'inf')
+      if [[ ${DEFAULT_VARS["DMS_DEBUG"]} == 1 ]]; then
+        msg="${c_green}  * ${notification_msg}${c_reset}"
+      fi
+      ;;
+    'started')
+      msg="${c_green} ${notification_msg}${c_reset}"
+      ;;
+    'warn')
+      msg="${c_brown}  * ${notification_msg}${c_reset}"
+      ;;
+    'err')
+      msg="${c_red}  * ${notification_msg}${c_reset}"
+      ;;
+    'fatal')
+      msg="${c_red}Error: ${notification_msg}${c_reset}"
+      ;;
+    *)
+      msg=""
+      ;;
+  esac
+
+  case "${notification_format}" in
+    'n')
+      options="-ne"
+      ;;
+    *)
+      options="-e"
+      ;;
+  esac
+
+  [[ ! -z "${msg}" ]] && echo $options "${msg}"
+}
+
+# setup /etc/postfix/relayhost_map
+# --
+# @domain1.com        [smtp.mailgun.org]:587
+# @domain2.com        [smtp.mailgun.org]:587
+# @domain3.com        [smtp.mailgun.org]:587
+function populate_relayhost_map() {
+  echo -n > /etc/postfix/relayhost_map
+  chown root:root /etc/postfix/relayhost_map
+  chmod 0600 /etc/postfix/relayhost_map
+
+  if [ -f /tmp/docker-mailserver/postfix-relaymap.cf ]; then
+    notify 'inf' "Adding relay mappings from postfix-relaymap.cf"
+    # Keep lines which are not a comment *and* have a destination.
+    sed -n '/^\s*[^#[:space:]]\S*\s\+\S/p' /tmp/docker-mailserver/postfix-relaymap.cf \
+        >> /etc/postfix/relayhost_map
+  fi
+  {
+    # Note: Won't detect domains when lhs has spaces (but who does that?!).
+    sed -n '/^\s*[^#[:space:]]/ s/^[^@|]*@\([^|]\+\)|.*$/\1/p' /tmp/docker-mailserver/postfix-accounts.cf
+    [ -f /tmp/docker-mailserver/postfix-virtual.cf ] &&
+      sed -n '/^\s*[^#[:space:]]/ s/^\s*[^@[:space:]]*@\(\S\+\)\s.*/\1/p' /tmp/docker-mailserver/postfix-virtual.cf
+  } | while read domain; do
+    if ! grep -q -e "^@${domain}\b" /etc/postfix/relayhost_map &&
+       ! grep -qs -e "^\s*@${domain}\s*$" /tmp/docker-mailserver/postfix-relaymap.cf; then
+      # Domain not already present *and* not ignored.
+      notify 'inf' "Adding relay mapping for ${domain}"
+      echo "@${domain}    [$RELAY_HOST]:$RELAY_PORT" >> /etc/postfix/relayhost_map
+    fi
+  done
+}
+
+# File storing the checksums of the monitored files.
+CHKSUM_FILE=/tmp/docker-mailserver-config-chksum
+
+# Compute checksums of monitored files.
+function monitored_files_checksums() {
+  (
+    cd /tmp/docker-mailserver
+    # (2>/dev/null to ignore warnings about files that don't exist)
+    exec sha512sum 2>/dev/null -- \
+           postfix-accounts.cf \
+           postfix-virtual.cf \
+           postfix-aliases.cf \
+           dovecot-quotas.cf \
+           /etc/letsencrypt/acme.json \
+           "/etc/letsencrypt/live/$HOSTNAME/key.pem" \
+           "/etc/letsencrypt/live/$HOSTNAME/fullchain.pem"
+  )
+  return 0
+}

target/start-mailserver.sh

@@ -29,7 +29,7 @@ DEFAULT_VARS["POSTFIX_MAILBOX_SIZE_LIMIT"]="${POSTFIX_MAILBOX_SIZE_LIMIT:="0"}"
 DEFAULT_VARS["POSTFIX_INET_PROTOCOLS"]="${POSTFIX_INET_PROTOCOLS:="all"}"
 DEFAULT_VARS["ENABLE_SASLAUTHD"]="${ENABLE_SASLAUTHD:="0"}"
 DEFAULT_VARS["SMTP_ONLY"]="${SMTP_ONLY:="0"}"
-DEFAULT_VARS["DMS_DEBUG"]="${DMS_DEBUG:="0"}"
+# DEFAULT_VARS["DMS_DEBUG"] defined in helper_functions.sh
 DEFAULT_VARS["OVERRIDE_HOSTNAME"]="${OVERRIDE_HOSTNAME}"
 DEFAULT_VARS["POSTSCREEN_ACTION"]="${POSTSCREEN_ACTION:="enforce"}"
 DEFAULT_VARS["SPOOF_PROTECTION"]="${SPOOF_PROTECTION:="0"}"
@@ -309,62 +309,6 @@ function _register_misc_function() {
 ##########################################################################
 
 
-function notify () {
-	c_red="\e[0;31m"
-	c_green="\e[0;32m"
-	c_brown="\e[0;33m"
-	c_blue="\e[0;34m"
-	c_bold="\033[1m"
-	c_reset="\e[0m"
-
-	notification_type=$1
-	notification_msg=$2
-	notification_format=$3
-	msg=""
-
-	case "${notification_type}" in
-		'taskgrp')
-			msg="${c_bold}${notification_msg}${c_reset}"
-			;;
-		'task')
-			if [[ ${DEFAULT_VARS["DMS_DEBUG"]} == 1 ]]; then
-				msg="  ${notification_msg}${c_reset}"
-			fi
-			;;
-		'inf')
-			if [[ ${DEFAULT_VARS["DMS_DEBUG"]} == 1 ]]; then
-				msg="${c_green}  * ${notification_msg}${c_reset}"
-			fi
-			;;
-		'started')
-			msg="${c_green} ${notification_msg}${c_reset}"
-			;;
-		'warn')
-			msg="${c_brown}  * ${notification_msg}${c_reset}"
-			;;
-		'err')
-			msg="${c_red}  * ${notification_msg}${c_reset}"
-			;;
-		'fatal')
-			msg="${c_red}Error: ${notification_msg}${c_reset}"
-			;;
-		*)
-			msg=""
-			;;
-	esac
-
-	case "${notification_format}" in
-		'n')
-			options="-ne"
-	  	;;
-		*)
-  		options="-e"
-			;;
-	esac
-
-	[[ ! -z "${msg}" ]] && echo $options "${msg}"
-}
-
 function defunc() {
 	notify 'fatal' "Please fix your configuration. Exiting..."
 	exit 1
@@ -500,19 +444,9 @@ function _setup_file_permissions() {
 function _setup_chksum_file() {
         notify 'task' "Setting up configuration checksum file"
 
-
         if [ -d /tmp/docker-mailserver ]; then
-          pushd /tmp/docker-mailserver
-
-          declare -a cf_files=()
-          for file in postfix-accounts.cf postfix-virtual.cf postfix-aliases.cf dovecot-quotas.cf /etc/letsencrypt/acme.json "/etc/letsencrypt/live/$HOSTNAME/key.pem" "/etc/letsencrypt/live/$HOSTNAME/fullchain.pem"; do
-            [ -f "$file" ] && cf_files+=("$file")
-          done
-
           notify 'inf' "Creating $CHKSUM_FILE"
-          sha512sum ${cf_files[@]/#/--tag } >$CHKSUM_FILE
+          monitored_files_checksums >"$CHKSUM_FILE"
-
-          popd
         else
           # We could just skip the file, but perhaps config can be added later?
           # If so it must be processed by the check for changes script
@@ -1349,36 +1283,7 @@ function _setup_postfix_relay_hosts() {
 	fi
 	# end /etc/postfix/sasl_passwd
 
-	# setup /etc/postfix/relayhost_map
+	populate_relayhost_map
-	# --
-	# @domain1.com        [smtp.mailgun.org]:587
-	# @domain2.com        [smtp.mailgun.org]:587
-	# @domain3.com        [smtp.mailgun.org]:587
-
-	echo -n > /etc/postfix/relayhost_map
-
-	if [ -f /tmp/docker-mailserver/postfix-relaymap.cf ]; then
-		notify 'inf' "Adding relay mappings from postfix-relaymap.cf"
-		while read line; do
-			if ! echo "$line" | grep -q -e "\s*#"; then
-				echo "$line" >> /etc/postfix/relayhost_map
-			fi
-		done < /tmp/docker-mailserver/postfix-relaymap.cf
-	fi
-	grep -v "^\s*$\|^\s*\#" /tmp/docker-mailserver/postfix-accounts.cf | while IFS=$'|' read login pass
-	do
-		domain=$(echo ${login} | cut -d @ -f2)
-		if ! grep -q -e "^@${domain}\b" /etc/postfix/relayhost_map; then
-			notify 'inf' "Adding relay mapping for ${domain}"
-			echo "@${domain}		[$RELAY_HOST]:$RELAY_PORT" >> /etc/postfix/relayhost_map
-		fi
-	done
-	# remove lines with no destination
-	sed -i '/^@\S*\s*$/d' /etc/postfix/relayhost_map
-
-	chown root:root /etc/postfix/relayhost_map
-	chmod 0600 /etc/postfix/relayhost_map
-	# end /etc/postfix/relayhost_map
 
 	postconf -e \
 		"smtp_sasl_auth_enable = yes" \

test/config/relay-hosts/postfix-virtual.cf

@@ -0,0 +1 @@
+@domain1.tld user1@domainone.tld

test/mail_ssl_letsencrypt.bats

@@ -118,7 +118,7 @@ function teardown_file() {
   assert_output --partial "Cert found in /etc/letsencrypt/acme.json for *.example.com"
   assert_output --partial "postfix: stopped"
   assert_output --partial "postfix: started"
-  assert_output --partial "Update checksum"
+  assert_output --partial "Change detected"
 
   run docker exec mail_lets_acme_json /bin/bash -c "cat /etc/letsencrypt/live/mail.my-domain.com/key.pem"
   assert_output "$(cat "`pwd`/test/config/letsencrypt/changed/key.pem")"

test/mail_with_relays.bats

@@ -9,8 +9,13 @@ function teardown() {
 }
 
 function setup_file() {
+    # We use a temporary config directory since we'll be dynamically editing
+    # it with setup.sh.
+    tmp_confdir=$(mktemp -d /tmp/docker-mailserver-config-relay-hosts-XXXXX)
+    cp -aT test/config/relay-hosts "$tmp_confdir"
+
     docker run -d --name mail_with_relays \
-            -v "`pwd`/test/config/relay-hosts":/tmp/docker-mailserver \
+            -v "$tmp_confdir":/tmp/docker-mailserver \
             -v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
             -e RELAY_HOST=default.relay.com \
             -e RELAY_PORT=2525 \
@@ -25,6 +30,7 @@ function setup_file() {
 
 function teardown_file() {
     docker rm -f mail_with_relays
+    rm -rf "$tmp_confdir"
 }
 
 @test "first" {
@@ -32,28 +38,60 @@ function teardown_file() {
 }
 
 @test "checking relay hosts: default mapping is added from env vars" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainone.tld\s\+\[default.relay.com\]:2525" | wc -l | grep 1'
+  run docker exec mail_with_relays grep -e domainone.tld /etc/postfix/relayhost_map
-  assert_success
+  assert_output -e '^@domainone.tld\s+\[default.relay.com\]:2525$'
+}
+
+@test "checking relay hosts: default mapping is added from env vars for virtual user entry" {
+  run docker exec mail_with_relays grep -e domain1.tld /etc/postfix/relayhost_map
+  assert_output -e '^@domain1.tld\s+\[default.relay.com\]:2525$'
+}
+
+@test "checking relay hosts: default mapping is added from env vars for new user entry" {
+  run docker exec mail_with_relays grep -e domainzero.tld /etc/postfix/relayhost_map
+  assert_output ''
+  run ./setup.sh -c mail_with_relays email add user0@domainzero.tld password123
+  for i in {1..10}; do
+    sleep 1
+    run docker exec mail_with_relays grep -e domainzero.tld /etc/postfix/relayhost_map
+    [[ $status == 0 ]] && break
+  done
+  assert_output -e '^@domainzero.tld\s+\[default.relay.com\]:2525$'
+}
+
+@test "checking relay hosts: default mapping is added from env vars for new virtual user entry" {
+  run docker exec mail_with_relays grep -e domain2.tld /etc/postfix/relayhost_map
+  assert_output ''
+  run ./setup.sh -c mail_with_relays alias add user2@domain2.tld user2@domaintwo.tld
+  for i in {1..10}; do
+    sleep 1
+    run docker exec mail_with_relays grep -e domain2.tld /etc/postfix/relayhost_map
+    [[ $status == 0 ]] && break
+  done
+  assert_output -e '^@domain2.tld\s+\[default.relay.com\]:2525$'
 }
 
 @test "checking relay hosts: custom mapping is added from file" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domaintwo.tld\s\+\[other.relay.com\]:587" | wc -l | grep 1'
+  run docker exec mail_with_relays grep -e domaintwo.tld /etc/postfix/relayhost_map
-  assert_success
+  assert_output -e '^@domaintwo.tld\s+\[other.relay.com\]:587$'
 }
 
 @test "checking relay hosts: ignored domain is not added" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainthree.tld\s\+\[any.relay.com\]:25" | wc -l | grep 0'
+  run docker exec mail_with_relays grep -e domainthree.tld /etc/postfix/relayhost_map
-  assert_success
+  assert_failure 1
+  assert_output ''
 }
 
 @test "checking relay hosts: auth entry is added" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^@domaintwo.tld\s\+smtp_user_2:smtp_password_2" | wc -l | grep 1'
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^@domaintwo.tld\s\+smtp_user_2:smtp_password_2" | wc -l'
   assert_success
+  assert_output 1
 }
 
 @test "checking relay hosts: default auth entry is added" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^\[default.relay.com\]:2525\s\+smtp_user:smtp_password" | wc -l | grep 1'
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^\[default.relay.com\]:2525\s\+smtp_user:smtp_password" | wc -l'
   assert_success
+  assert_output 1
 }
 
 @test "last" {

test/tests.bats

@@ -20,7 +20,7 @@ function wait_for_service() {
 
 function count_processed_changes() {
   containerName=$1
-  docker exec $containerName cat /var/log/supervisor/changedetector.log | grep "Update checksum" | wc -l
+  docker exec $containerName cat /var/log/supervisor/changedetector.log | grep "Change detected" | wc -l
 }
 
 #