Merge pull request #1217 from martin-schulze-vireso/feature/modulariz_tests

WIP: Modularize tests

Makefile

@@ -188,71 +188,6 @@ run:
 		--link ldap_for_mail:ldap \
 		-h mail.my-domain.com -t $(NAME)
 	sleep 15
-	docker run -d --name mail_with_imap \
-		-v "`pwd`/test/config":/tmp/docker-mailserver \
-		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
-		-e ENABLE_SASLAUTHD=1 \
-		-e SASLAUTHD_MECHANISMS=rimap \
-		-e SASLAUTHD_MECH_OPTIONS=127.0.0.1 \
-		-e POSTMASTER_ADDRESS=postmaster@localhost.localdomain \
-		-e DMS_DEBUG=0 \
-		-h mail.my-domain.com -t $(NAME)
-	sleep 15
-	docker run -d --name mail_postscreen \
-		-v "`pwd`/test/config":/tmp/docker-mailserver \
-		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
-		-e POSTSCREEN_ACTION=enforce \
-		--cap-add=NET_ADMIN \
-		-h mail.my-domain.com -t $(NAME)
-	sleep 15
-	docker run -d --name mail_lmtp_ip \
-		-v "`pwd`/test/config":/tmp/docker-mailserver \
-		-v "`pwd`/test/config/dovecot-lmtp":/etc/dovecot \
-		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
-		-e ENABLE_POSTFIX_VIRTUAL_TRANSPORT=1 \
-		-e POSTFIX_DAGENT=lmtp:127.0.0.1:24 \
-		-e DMS_DEBUG=0 \
-		-h mail.my-domain.com -t $(NAME)
-	sleep 30
-	docker run -d --name mail_with_postgrey \
-		-v "`pwd`/test/config":/tmp/docker-mailserver \
-		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
-		-e ENABLE_POSTGREY=1 \
-		-e POSTGREY_DELAY=15 \
-		-e POSTGREY_MAX_AGE=35 \
-		-e POSTGREY_AUTO_WHITELIST_CLIENTS=5 \
-		-e POSTGREY_TEXT="Delayed by postgrey" \
-		-e DMS_DEBUG=0 \
-		-h mail.my-domain.com -t $(NAME)
-	sleep 20
-	docker run -d --name mail_undef_spam_subject \
-		-v "`pwd`/test/config":/tmp/docker-mailserver \
-		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
-		-e ENABLE_SPAMASSASSIN=1 \
-		-e SA_SPAM_SUBJECT="undef" \
-		-h mail.my-domain.com -t $(NAME)
-	sleep 15
-	docker run -d --name mail_with_relays \
-		-v "`pwd`/test/config/relay-hosts":/tmp/docker-mailserver \
-		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
-		-e RELAY_HOST=default.relay.com \
-		-e RELAY_PORT=2525 \
-		-e RELAY_USER=smtp_user \
-		-e RELAY_PASSWORD=smtp_password \
-		--cap-add=SYS_PTRACE \
-		-e PERMIT_DOCKER=host \
-		-e DMS_DEBUG=0 \
-		-h mail.my-domain.com -t $(NAME)
-	sleep 15
-	docker run -d --name mail_with_default_relay \
-		-v "`pwd`/test/config/relay-hosts":/tmp/docker-mailserver \
-		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
-		-e DEFAULT_RELAY_HOST=default.relay.host.invalid:25 \
-		--cap-add=SYS_PTRACE \
-		-e PERMIT_DOCKER=host \
-		-e DMS_DEBUG=0 \
-		-h mail.my-domain.com -t $(NAME)
-	sleep 15
 
 generate-accounts-after-run:
 	docker run --rm -e MAIL_USER=added@localhost.localdomain -e MAIL_PASS=mypassword -t $(NAME) /bin/sh -c 'echo "$$MAIL_USER|$$(doveadm pw -s SHA512-CRYPT -u $$MAIL_USER -p $$MAIL_PASS)"' >> test/config/postfix-accounts.cf
@@ -285,7 +220,6 @@ fixtures:
 	docker exec mail_disabled_clamav_spamassassin /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt"
 	docker exec mail /bin/sh -c "sendmail root < /tmp/docker-mailserver-test/email-templates/root-email.txt"
 	# postfix virtual transport lmtp
-	docker exec mail_lmtp_ip /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt"
 	docker exec mail_privacy /bin/sh -c "openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/email-templates/send-privacy-email.txt"
 	docker exec mail_override_hostname /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt"
 	# Wait for mails to be analyzed
@@ -320,16 +254,9 @@ clean:
 		mail_manual_ssl \
 		ldap_for_mail \
 		mail_with_ldap \
-		mail_with_imap \
-		mail_lmtp_ip \
-		mail_with_postgrey \
-		mail_undef_spam_subject \
-		mail_postscreen \
 		mail_override_hostname \
 		mail_domainname \
-		mail_srs_domainname \
+		mail_srs_domainname
-		mail_with_relays \
-		mail_with_default_relay 
 
 	@if [ -d config.bak ]; then\
 		rm -rf config ;\

test/default_relay_host.bats

@@ -0,0 +1,26 @@
+load 'test_helper/common'
+
+function setup() {
+    docker run -d --name mail_with_default_relay \
+		-v "`pwd`/test/config/relay-hosts":/tmp/docker-mailserver \
+		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+		-e DEFAULT_RELAY_HOST=default.relay.host.invalid:25 \
+		--cap-add=SYS_PTRACE \
+		-e PERMIT_DOCKER=host \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t ${NAME}
+    wait_for_finished_setup_in_container mail_with_default_relay
+}
+
+function teardown() {
+    docker rm -f mail_with_default_relay
+}
+
+#
+# default relay host
+#
+
+@test "checking default relay host: default relay host is added to main.cf" {
+  run docker exec mail_with_default_relay /bin/sh -c 'grep -e "^relayhost = default.relay.host.invalid:25" /etc/postfix/main.cf | wc -l | grep 1'
+  assert_success
+}

test/mail_lmtp_ip.bats

@@ -0,0 +1,56 @@
+load 'test_helper/common'
+
+setup() {
+    run_setup_file_if_necessary
+}
+
+teardown() {
+    run_teardown_file_if_necessary
+}
+
+setup_file() {
+    docker run -d --name mail_lmtp_ip \
+		-v "`pwd`/test/config":/tmp/docker-mailserver \
+		-v "`pwd`/test/config/dovecot-lmtp":/etc/dovecot \
+		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+		-e ENABLE_POSTFIX_VIRTUAL_TRANSPORT=1 \
+		-e POSTFIX_DAGENT=lmtp:127.0.0.1:24 \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t ${NAME}
+    wait_for_finished_setup_in_container mail_lmtp_ip
+}
+
+
+teardown_file() {
+    docker rm -f mail_lmtp_ip
+}
+
+@test "first" {
+    skip 'only used to call setup_file from setup'
+}
+
+#
+# Postfix VIRTUAL_TRANSPORT
+#
+@test "checking postfix-lmtp: virtual_transport config is set" {
+  run docker exec mail_lmtp_ip /bin/sh -c "grep 'virtual_transport = lmtp:127.0.0.1:24' /etc/postfix/main.cf"
+  assert_success
+}
+
+@test "checking postfix-lmtp: delivers mail to existing account" {
+  # maybe we can move this into the setup to speed things up futher.
+  # this likely would need an async coroutine to avoid blocking the other tests while waiting for the server to come up
+  wait_for_smtp_port_in_container mail_lmtp_ip
+  run docker exec mail_lmtp_ip /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt"
+  assert_success
+  
+  # polling needs to avoid wc -l's unconditionally successful return status
+  repeat_until_success_or_timeout 60 docker exec mail_lmtp_ip /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)'"
+  run docker exec mail_lmtp_ip /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "last" {
+    skip 'only used to call teardown_file from teardown'
+}

test/mail_postscreen.bats

@@ -0,0 +1,62 @@
+load 'test_helper/common'
+
+setup() {
+    run_setup_file_if_necessary
+
+    # Getting mail container IP
+    MAIL_POSTSCREEN_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' mail_postscreen)
+}
+
+teardown() {
+    run_teardown_file_if_necessary
+}
+
+setup_file() {
+    docker run -d --name mail_postscreen \
+		-v "`pwd`/test/config":/tmp/docker-mailserver \
+		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+		-e POSTSCREEN_ACTION=enforce \
+		--cap-add=NET_ADMIN \
+		-h mail.my-domain.com -t ${NAME}
+
+    docker run --name mail_postscreen_sender \
+        -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \
+        -d ${NAME} \
+        tail -f /var/log/faillog
+
+    wait_for_smtp_port_in_container mail_postscreen
+}
+
+teardown_file() {
+    docker rm -f mail_postscreen mail_postscreen_sender
+}
+
+@test "first" {
+    skip 'only used to call setup_file from setup'
+}
+
+@test "checking postscreen: talk too fast" {
+  docker exec mail_postscreen_sender /bin/sh -c "nc $MAIL_POSTSCREEN_IP 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt"
+
+  repeat_until_success_or_timeout 10 run docker exec mail_postscreen grep 'COMMAND PIPELINING' /var/log/mail/mail.log
+  assert_success
+}
+
+@test "checking postscreen: positive test (respecting postscreen_greet_wait time and talking in turn)" {
+  for i in {1,2}; do
+    docker exec mail_postscreen_sender /bin/bash -c \
+    'exec 3<>/dev/tcp/'$MAIL_POSTSCREEN_IP'/25 && \
+    while IFS= read -r cmd; do \
+      head -1 <&3; \
+      [[ "$cmd" == "EHLO"* ]] && sleep 6; \
+      echo $cmd >&3; \
+    done < "/tmp/docker-mailserver-test/auth/smtp-auth-login.txt"'
+  done
+
+  repeat_until_success_or_timeout 10 run docker exec mail_postscreen grep 'PASS NEW ' /var/log/mail/mail.log
+  assert_success
+}
+
+@test "last" {
+    skip 'only used to call teardown_file from teardown'
+}

test/mail_undef_spam_subject.bats

@@ -0,0 +1,50 @@
+load 'test_helper/common'
+
+function setup() {
+    docker run -d --name mail_undef_spam_subject \
+            -v "`pwd`/test/config":/tmp/docker-mailserver \
+            -v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+            -e ENABLE_SPAMASSASSIN=1 \
+            -e SA_SPAM_SUBJECT="undef" \
+            -h mail.my-domain.com -t ${NAME}
+    CONTAINER=$(docker run -d \
+		-v "`pwd`/test/config":/tmp/docker-mailserver \
+		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+		-v "`pwd`/test/onedir":/var/mail-state \
+		-e ENABLE_CLAMAV=1 \
+		-e SPOOF_PROTECTION=1 \
+		-e ENABLE_SPAMASSASSIN=1 \
+		-e REPORT_RECIPIENT=user1@localhost.localdomain \
+		-e REPORT_SENDER=report1@mail.my-domain.com \
+		-e SA_TAG=-5.0 \
+		-e SA_TAG2=2.0 \
+		-e SA_KILL=3.0 \
+		-e SA_SPAM_SUBJECT="SPAM: " \
+		-e VIRUSMAILS_DELETE_DELAY=7 \
+		-e ENABLE_SRS=1 \
+		-e SASL_PASSWD="external-domain.com username:password" \
+		-e ENABLE_MANAGESIEVE=1 \
+		--cap-add=SYS_PTRACE \
+		-e PERMIT_DOCKER=host \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t ${NAME})
+    wait_for_finished_setup_in_container mail_undef_spam_subject
+    wait_for_finished_setup_in_container "$CONTAINER"
+}
+
+function teardown() {
+    docker rm -f mail_undef_spam_subject "$CONTAINER"
+}
+
+@test "checking spamassassin: docker env variables are set correctly (custom)" {
+  run docker exec "$CONTAINER" /bin/sh -c "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= -5.0'"
+  assert_success
+  run docker exec "$CONTAINER" /bin/sh -c "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'"
+  assert_success
+  run docker exec "$CONTAINER" /bin/sh -c "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'"
+  assert_success
+  run docker exec "$CONTAINER" /bin/sh -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= .SPAM: .'"
+  assert_success
+  run docker exec mail_undef_spam_subject /bin/sh -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'"
+  assert_success
+}

test/mail_with_imap.bats

@@ -0,0 +1,56 @@
+
+load 'test_helper/common'
+
+setup() {
+    run_setup_file_if_necessary
+}
+
+teardown() {
+    run_teardown_file_if_necessary
+}
+
+setup_file() {
+    docker run -d --name mail_with_imap \
+		-v "`pwd`/test/config":/tmp/docker-mailserver \
+		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+		-e ENABLE_SASLAUTHD=1 \
+		-e SASLAUTHD_MECHANISMS=rimap \
+		-e SASLAUTHD_MECH_OPTIONS=127.0.0.1 \
+		-e POSTMASTER_ADDRESS=postmaster@localhost.localdomain \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t ${NAME}
+    wait_for_smtp_port_in_container mail_with_imap
+}
+
+teardown_file() {
+    docker rm -f mail_with_imap
+}
+
+@test "first" {
+    skip 'only used to call setup_file from setup'
+}
+
+#
+# RIMAP
+#
+
+# dovecot
+@test "checking dovecot: ldap rimap connection and authentication works" {
+  run docker exec mail_with_imap /bin/sh -c "nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-auth.txt"
+  assert_success
+}
+
+# saslauthd
+@test "checking saslauthd: sasl rimap authentication works" {
+  run docker exec mail_with_imap bash -c "testsaslauthd -u user1@localhost.localdomain -p mypassword"
+  assert_success
+}
+
+@test "checking saslauthd: rimap smtp authentication" {
+  run docker exec mail_with_imap /bin/sh -c "nc -w 5 0.0.0.0 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt | grep 'Authentication successful'"
+  assert_success
+}
+
+@test "last" {
+    skip 'only used to call teardown_file from teardown'
+}

test/mail_with_postgrey.bats

@@ -0,0 +1,101 @@
+load 'test_helper/common'
+
+function setup() {
+    run_setup_file_if_necessary
+}
+
+function teardown() {
+    run_teardown_file_if_necessary
+}
+
+function setup_file() {
+    docker run -d --name mail_with_postgrey \
+		-v "`pwd`/test/config":/tmp/docker-mailserver \
+		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+		-e ENABLE_POSTGREY=1 \
+		-e POSTGREY_DELAY=15 \
+		-e POSTGREY_MAX_AGE=35 \
+		-e POSTGREY_AUTO_WHITELIST_CLIENTS=5 \
+		-e POSTGREY_TEXT="Delayed by postgrey" \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t ${NAME}
+    # using postfix availability as start indicator, this might be insufficient for postgrey
+    wait_for_smtp_port_in_container mail_with_postgrey
+}
+
+function teardown_file() {
+    docker rm -f mail_with_postgrey
+}
+
+@test "first" {
+  # this test must come first to reliably identify when to run setup_file
+}
+
+@test "checking postgrey: /etc/postfix/main.cf correctly edited" {
+  run docker exec mail_with_postgrey /bin/bash -c "grep 'bl.spamcop.net, check_policy_service inet:127.0.0.1:10023' /etc/postfix/main.cf | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "checking postgrey: /etc/default/postgrey correctly edited and has the default values" {
+  run docker exec mail_with_postgrey /bin/bash -c "grep '^POSTGREY_OPTS=\"--inet=127.0.0.1:10023 --delay=15 --max-age=35 --auto-whitelist-clients=5\"$' /etc/default/postgrey | wc -l"
+  assert_success
+  assert_output 1
+  run docker exec mail_with_postgrey /bin/bash -c "grep '^POSTGREY_TEXT=\"Delayed by postgrey\"$' /etc/default/postgrey | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "checking process: postgrey (postgrey server enabled)" {
+  run docker exec mail_with_postgrey /bin/bash -c "ps aux --forest | grep -v grep | grep 'postgrey'"
+  assert_success
+}
+
+@test "checking postgrey: there should be a log entry about a new greylisted e-mail user@external.tld in /var/log/mail/mail.log" {
+  #editing the postfix config in order to ensure that postgrey handles the test e-mail. The other spam checks at smtpd_recipient_restrictions would interfere with it.
+  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/permit_sasl_authenticated.*policyd-spf,$//g' /etc/postfix/main.cf"
+  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/reject_unauth_pipelining.*reject_unknown_recipient_domain,$//g' /etc/postfix/main.cf"
+  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/reject_rbl_client.*inet:127\.0\.0\.1:10023$//g' /etc/postfix/main.cf"
+  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/smtpd_recipient_restrictions =/smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:10023/g' /etc/postfix/main.cf"
+
+  run docker exec mail_with_postgrey /bin/sh -c "/etc/init.d/postfix reload"
+  run docker exec mail_with_postgrey /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt"
+  sleep 5 #ensure that the information has been written into the log
+  run docker exec mail_with_postgrey /bin/bash -c "grep -i 'action=greylist.*user@external\.tld' /var/log/mail/mail.log | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "checking postgrey: there should be a log entry about the retried and passed e-mail user@external.tld in /var/log/mail/mail.log" {
+  sleep 20 #wait 20 seconds so that postgrey would accept the message
+  run docker exec mail_with_postgrey /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt"
+  sleep 8
+  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=triplet found.*user@external\.tld' /var/log/mail/mail.log | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "checking postgrey: there should be a log entry about the whitelisted and passed e-mail user@whitelist.tld in /var/log/mail/mail.log" {
+  run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt"
+  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "checking postgrey: there should be a log entry about the whitelisted local and passed e-mail user@whitelistlocal.tld in /var/log/mail/mail.log" {
+  run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_local.txt"
+  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "checking postgrey: there should be a log entry about the whitelisted recipient user2@otherdomain.tld in /var/log/mail/mail.log" {
+  run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt"
+  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=recipient whitelist' /var/log/mail/mail.log | wc -l"
+  assert_success
+  assert_output 1
+}
+
+@test "last" {
+  # this test is only there to reliably mark the end for the teardown_file
+}

test/mail_with_postgrey_disabled_by_default.bats

@@ -0,0 +1,20 @@
+load 'test_helper/common'
+
+function setup() {
+    CONTAINER=$(docker run -d \
+		-v "`pwd`/test/config":/tmp/docker-mailserver \
+		-v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t ${NAME})
+    # using postfix availability as start indicator, this might be insufficient for postgrey
+    wait_for_smtp_port_in_container $CONTAINER
+}
+
+function teardown() {
+    docker rm -f $CONTAINER
+}
+
+@test "checking process: postgrey (disabled in default configuration)" {
+  run docker exec $CONTAINER /bin/bash -c "ps aux --forest | grep -v grep | grep 'postgrey'"
+  assert_failure
+}

test/mail_with_relays.bats

@@ -0,0 +1,61 @@
+load 'test_helper/common'
+
+function setup() {
+    run_setup_file_if_necessary
+}
+
+function teardown() {
+    run_teardown_file_if_necessary
+}
+
+function setup_file() {
+    docker run -d --name mail_with_relays \
+            -v "`pwd`/test/config/relay-hosts":/tmp/docker-mailserver \
+            -v "`pwd`/test/test-files":/tmp/docker-mailserver-test:ro \
+            -e RELAY_HOST=default.relay.com \
+            -e RELAY_PORT=2525 \
+            -e RELAY_USER=smtp_user \
+            -e RELAY_PASSWORD=smtp_password \
+            --cap-add=SYS_PTRACE \
+            -e PERMIT_DOCKER=host \
+            -e DMS_DEBUG=0 \
+            -h mail.my-domain.com -t ${NAME}
+        wait_for_finished_setup_in_container mail_with_relays
+}
+
+function teardown_file() {
+    docker rm -f mail_with_relays
+}
+
+@test "first" {
+  # this test must come first to reliably identify when to run setup_file
+}
+
+@test "checking relay hosts: default mapping is added from env vars" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainone.tld\s\+\[default.relay.com\]:2525" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking relay hosts: custom mapping is added from file" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domaintwo.tld\s\+\[other.relay.com\]:587" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking relay hosts: ignored domain is not added" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainthree.tld\s\+\[any.relay.com\]:25" | wc -l | grep 0'
+  assert_success
+}
+
+@test "checking relay hosts: auth entry is added" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^@domaintwo.tld\s\+smtp_user_2:smtp_password_2" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking relay hosts: default auth entry is added" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^\[default.relay.com\]:2525\s\+smtp_user:smtp_password" | wc -l | grep 1'
+  assert_success
+}
+
+@test "last" {
+  # this test is only there to reliably mark the end for the teardown_file
+}

test/permit_docker.bats

@@ -1,8 +1,6 @@
-load 'test_helper/bats-support/load'
+load 'test_helper/common'
-load 'test_helper/bats-assert/load'
 
 NON_DEFAULT_DOCKER_MAIL_NETWORK_NAME=non-default-docker-mail-network
-NAME=tvial/docker-mailserver:testing
 setup() {
     docker network create --driver bridge ${NON_DEFAULT_DOCKER_MAIL_NETWORK_NAME}
 	docker network create --driver bridge ${NON_DEFAULT_DOCKER_MAIL_NETWORK_NAME}2
@@ -31,7 +29,7 @@ setup() {
 		-t ${NAME}
 
     # wait until postfix is up
-    repeat_until_success_or_timeout 60 docker exec mail_smtponly_second_network /bin/sh -c "nc -z 0.0.0.0 25"
+    wait_for_smtp_port_in_container mail_smtponly_second_network
 }
 
 teardown() {
@@ -41,20 +39,6 @@ teardown() {
     docker network rm ${NON_DEFAULT_DOCKER_MAIL_NETWORK_NAME} ${NON_DEFAULT_DOCKER_MAIL_NETWORK_NAME}2
 }
 
-function repeat_until_success_or_timeout {
-    TIMEOUT=$1
-    STARTTIME=$SECONDS
-    shift 1
-    until "$@"
-    do
-        sleep 5
-        if [[ $(($SECONDS - $STARTTIME )) -gt $TIMEOUT ]]; then
-            echo "Timed out on command: $@"
-            exit 1
-        fi
-    done
-}
-
 @test "checking PERMIT_DOCKER: connected-networks" {
   ipnet1=$(docker network inspect --format '{{(index .IPAM.Config 0).Subnet}}' non-default-docker-mail-network)
   ipnet2=$(docker network inspect --format '{{(index .IPAM.Config 0).Subnet}}' non-default-docker-mail-network2)

test/test_helper/common.bash

@@ -0,0 +1,49 @@
+load 'test_helper/bats-support/load'
+load 'test_helper/bats-assert/load'
+
+NAME=tvial/docker-mailserver:testing
+
+# default timeout is 60 seconds
+TEST_TIMEOUT_IN_SECONDS=${TEST_TIMEOUT_IN_SECONDS-60}
+
+function repeat_until_success_or_timeout {
+    if ! [[ "$1" =~ ^[0-9]+$ ]]; then
+        echo "First parameter for timeout must be an integer, recieved \"$1\""
+        exit 1
+    fi
+    TIMEOUT=$1
+    STARTTIME=$SECONDS
+    shift 1
+    until "$@"
+    do
+        sleep 5
+        if [[ $(($SECONDS - $STARTTIME )) -gt $TIMEOUT ]]; then
+            echo "Timed out on command: $@"
+            exit 1
+        fi
+    done
+}
+
+# @param $1 name of the postfix container
+function wait_for_smtp_port_in_container() {
+    repeat_until_success_or_timeout $TEST_TIMEOUT_IN_SECONDS docker exec $1 /bin/sh -c "nc -z 0.0.0.0 25"
+}
+
+# @param $1 name of the postfix container
+function wait_for_finished_setup_in_container() {
+    repeat_until_success_or_timeout $TEST_TIMEOUT_IN_SECONDS sh -c "docker logs $1 | grep 'Starting mail server'"
+}
+
+# use in setup() in conjunction with a `@test "first" {}` to trigger setup_file reliably
+function run_setup_file_if_necessary() {
+    if [ "$BATS_TEST_NAME" == 'test_first' ]; then
+        setup_file
+    fi
+}
+
+# use in teardown() in conjunction with a `@test "last" {}` to trigger teardown_file reliably
+function run_teardown_file_if_necessary() {
+    if [ "$BATS_TEST_NAME" == 'test_last' ]; then
+        teardown_file
+    fi
+}

test/tests.bats

@@ -134,82 +134,6 @@ function count_processed_changes() {
   assert_success
 }
 
-
-#
-# postgrey
-#
-
-@test "checking process: postgrey (disabled in default configuration)" {
-  run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep 'postgrey'"
-  assert_failure
-}
-
-@test "checking postgrey: /etc/postfix/main.cf correctly edited" {
-  run docker exec mail_with_postgrey /bin/bash -c "grep 'bl.spamcop.net, check_policy_service inet:127.0.0.1:10023' /etc/postfix/main.cf | wc -l"
-  assert_success
-  assert_output 1
-}
-
-@test "checking postgrey: /etc/default/postgrey correctly edited and has the default values" {
-  run docker exec mail_with_postgrey /bin/bash -c "grep '^POSTGREY_OPTS=\"--inet=127.0.0.1:10023 --delay=15 --max-age=35 --auto-whitelist-clients=5\"$' /etc/default/postgrey | wc -l"
-  assert_success
-  assert_output 1
-  run docker exec mail_with_postgrey /bin/bash -c "grep '^POSTGREY_TEXT=\"Delayed by postgrey\"$' /etc/default/postgrey | wc -l"
-  assert_success
-  assert_output 1
-}
-
-@test "checking process: postgrey (postgrey server enabled)" {
-  run docker exec mail_with_postgrey /bin/bash -c "ps aux --forest | grep -v grep | grep 'postgrey'"
-  assert_success
-}
-
-@test "checking postgrey: there should be a log entry about a new greylisted e-mail user@external.tld in /var/log/mail/mail.log" {
-  #editing the postfix config in order to ensure that postgrey handles the test e-mail. The other spam checks at smtpd_recipient_restrictions would interfere with it.
-  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/permit_sasl_authenticated.*policyd-spf,$//g' /etc/postfix/main.cf"
-  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/reject_unauth_pipelining.*reject_unknown_recipient_domain,$//g' /etc/postfix/main.cf"
-  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/reject_rbl_client.*inet:127\.0\.0\.1:10023$//g' /etc/postfix/main.cf"
-  run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/smtpd_recipient_restrictions =/smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:10023/g' /etc/postfix/main.cf"
-
-  run docker exec mail_with_postgrey /bin/sh -c "/etc/init.d/postfix reload"
-  run docker exec mail_with_postgrey /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt"
-  sleep 5 #ensure that the information has been written into the log
-  run docker exec mail_with_postgrey /bin/bash -c "grep -i 'action=greylist.*user@external\.tld' /var/log/mail/mail.log | wc -l"
-  assert_success
-  assert_output 1
-}
-
-@test "checking postgrey: there should be a log entry about the retried and passed e-mail user@external.tld in /var/log/mail/mail.log" {
-  sleep 20 #wait 20 seconds so that postgrey would accept the message
-  run docker exec mail_with_postgrey /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt"
-  sleep 8
-  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=triplet found.*user@external\.tld' /var/log/mail/mail.log | wc -l"
-  assert_success
-  assert_output 1
-}
-
-@test "checking postgrey: there should be a log entry about the whitelisted and passed e-mail user@whitelist.tld in /var/log/mail/mail.log" {
-  run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt"
-  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l"
-  assert_success
-  assert_output 1
-}
-
-@test "checking postgrey: there should be a log entry about the whitelisted local and passed e-mail user@whitelistlocal.tld in /var/log/mail/mail.log" {
-  run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_local.txt"
-  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l"
-  assert_success
-  assert_output 1
-}
-
-@test "checking postgrey: there should be a log entry about the whitelisted recipient user2@otherdomain.tld in /var/log/mail/mail.log" {
-  run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt"
-  run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=recipient whitelist' /var/log/mail/mail.log | wc -l"
-  assert_success
-  assert_output 1
-}
-
-
 #
 # imap
 #
@@ -509,19 +433,6 @@ function count_processed_changes() {
   assert_success
 }
 
-@test "checking spamassassin: docker env variables are set correctly (custom)" {
-  run docker exec mail /bin/sh -c "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= -5.0'"
-  assert_success
-  run docker exec mail /bin/sh -c "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'"
-  assert_success
-  run docker exec mail /bin/sh -c "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'"
-  assert_success
-  run docker exec mail /bin/sh -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= .SPAM: .'"
-  assert_success
-  run docker exec mail_undef_spam_subject /bin/sh -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'"
-  assert_success
-}
-
 @test "checking spamassassin: all registered domains should see spam headers" {
   run docker exec mail /bin/sh -c "grep -ir 'X-Spam-' /var/mail/localhost.localdomain/user1/new"
   assert_success
@@ -961,39 +872,6 @@ function count_processed_changes() {
   assert_failure
 }
 
-#
-# postscreen
-#
-
-@test "checking postscreen" {
-  # Getting mail container IP
-  MAIL_POSTSCREEN_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' mail_postscreen)
-
-  # talk too fast:
-
-  docker exec fail-auth-mailer /bin/sh -c "nc $MAIL_POSTSCREEN_IP 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt"
-  sleep 5
-
-  run docker exec mail_postscreen grep 'COMMAND PIPELINING' /var/log/mail/mail.log
-  assert_success
-
-  # positive test. (respecting postscreen_greet_wait time and talking in turn):
-  for i in {1,2}; do
-    docker exec fail-auth-mailer /bin/bash -c \
-    'exec 3<>/dev/tcp/'$MAIL_POSTSCREEN_IP'/25 && \
-    while IFS= read -r cmd; do \
-      head -1 <&3; \
-      [[ "$cmd" == "EHLO"* ]] && sleep 6; \
-      echo $cmd >&3; \
-    done < "/tmp/docker-mailserver-test/auth/smtp-auth-login.txt"'
-  done
-
-  sleep 5
-
-  run docker exec mail_postscreen grep 'PASS NEW ' /var/log/mail/mail.log
-  assert_success
-}
-
 #
 # fetchmail
 #
@@ -1652,42 +1530,6 @@ function count_processed_changes() {
   assert_success
 }
 
-
-#
-# RIMAP
-#
-
-# dovecot
-@test "checking dovecot: ldap rimap connection and authentication works" {
-  run docker exec mail_with_imap /bin/sh -c "nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-auth.txt"
-  assert_success
-}
-
-# saslauthd
-@test "checking saslauthd: sasl rimap authentication works" {
-  run docker exec mail_with_imap bash -c "testsaslauthd -u user1@localhost.localdomain -p mypassword"
-  assert_success
-}
-
-@test "checking saslauthd: rimap smtp authentication" {
-  run docker exec mail_with_imap /bin/sh -c "nc -w 5 0.0.0.0 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt | grep 'Authentication successful'"
-  assert_success
-}
-
-#
-# Postfix VIRTUAL_TRANSPORT
-#
-@test "checking postfix-lmtp: virtual_transport config is set" {
-  run docker exec mail_lmtp_ip /bin/sh -c "grep 'virtual_transport = lmtp:127.0.0.1:24' /etc/postfix/main.cf"
-  assert_success
-}
-
-@test "checking postfix-lmtp: delivers mail to existing account" {
-  run docker exec mail_lmtp_ip /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | wc -l"
-  assert_success
-  assert_output 1
-}
-
 #
 # Pflogsumm delivery check
 #
@@ -1816,44 +1658,6 @@ function count_processed_changes() {
   assert_success
 }
 
-#
-# default relay host
-#
-
-@test "checking default relay host: default relay host is added to main.cf" {
-  run docker exec mail_with_default_relay /bin/sh -c 'grep -e "^relayhost = default.relay.host.invalid:25" /etc/postfix/main.cf | wc -l | grep 1'
-  assert_success
-}
-
-#
-# relay hosts
-#
-
-@test "checking relay hosts: default mapping is added from env vars" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainone.tld\s\+\[default.relay.com\]:2525" | wc -l | grep 1'
-  assert_success
-}
-
-@test "checking relay hosts: custom mapping is added from file" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domaintwo.tld\s\+\[other.relay.com\]:587" | wc -l | grep 1'
-  assert_success
-}
-
-@test "checking relay hosts: ignored domain is not added" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainthree.tld\s\+\[any.relay.com\]:25" | wc -l | grep 0'
-  assert_success
-}
-
-@test "checking relay hosts: auth entry is added" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^@domaintwo.tld\s\+smtp_user_2:smtp_password_2" | wc -l | grep 1'
-  assert_success
-}
-
-@test "checking relay hosts: default auth entry is added" {
-  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^\[default.relay.com\]:2525\s\+smtp_user:smtp_password" | wc -l | grep 1'
-  assert_success
-}
-
 #
 # root mail delivery
 #