Merge pull request #1447 from erik-wramner/next_to_master

Merge next branch with Debian Buster into master.

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:stretch-slim
+FROM debian:buster-slim
 
 ARG VCS_REF
 ARG VCS_VERSION
@@ -29,11 +29,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Packages
 # hadolint ignore=DL3015
-RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /etc/apt/sources.list.d/stretch-bp.list && \
+RUN \
   apt-get update -q --fix-missing && \
   apt-get -y install postfix && \
-  # TODO installing postfix with --no-install-recommends makes "checking ssl: generated default cert works correctly" fail
   apt-get -y install --no-install-recommends \
+    altermime \
     amavisd-new \
     apt-transport-https \
     arj \
@@ -56,6 +56,7 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
     iptables \
     locales \
     logwatch \
+    lhasa \
     libdate-manip-perl \
     liblz4-tool \
     libmail-spf-perl \
@@ -77,7 +78,8 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
     postsrsd \
     pyzor \
     razor \
-    ripole \
+    # TODO not present in buster?
+    #ripole \
     rpm2cpio \
     rsyslog \
     sasl2-bin \
@@ -88,14 +90,15 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
     unzip \
     whois \
     xz-utils \
-    zoo \
+    # TODO not present in buster?
-    && \
+    #zoo \
+    #&& \
   # use Dovecot community repo to react faster on security updates
-  curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \
+  #curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \
-  gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \
+  #gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \
-  echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \
+  #echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \
-  apt-get update -q --fix-missing && \
+  #apt-get update -q --fix-missing && \
-  apt-get -y install --no-install-recommends \
+  #apt-get -y install --no-install-recommends \
     dovecot-core \
     dovecot-imapd \
     dovecot-ldap \
@@ -116,17 +119,6 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
   rm -f /etc/postsrsd.secret && \
   rm -f /etc/cron.daily/00logwatch
 
-# install filebeat for logging
-RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \
-  echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \
-  apt-get update -q --fix-missing && \
-  apt-get -y install --no-install-recommends \
-    filebeat \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
-
-COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl
-
 RUN echo "0 */6 * * * clamav /usr/bin/freshclam --quiet" > /etc/cron.d/clamav-freshclam && \
   chmod 644 /etc/clamav/freshclam.conf && \
   freshclam && \
@@ -189,7 +181,8 @@ RUN sed -i -r 's/#(@|   \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_fil
 # Configure Fail2ban
 COPY target/fail2ban/jail.conf /etc/fail2ban/jail.conf
 COPY target/fail2ban/filter.d/dovecot.conf /etc/fail2ban/filter.d/dovecot.conf
-RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf && mkdir /var/run/fail2ban
+COPY target/fail2ban/filter.d/postfix-sasl.conf /etc/fail2ban/filter.d/postfix-sasl.conf
+RUN mkdir /var/run/fail2ban
 
 # Enables Pyzor and Razor
 RUN su - amavis -c "razor-admin -create && \
@@ -251,6 +244,11 @@ COPY target/supervisor/conf.d/* /etc/supervisor/conf.d/
 
 WORKDIR /
 
+# Switch iptables and ip6tables to legacy for fail2ban
+RUN update-alternatives --set iptables /usr/sbin/iptables-legacy \
+ && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
+
 EXPOSE 25 587 143 465 993 110 995 4190
 
 CMD ["supervisord", "-c", "/etc/supervisor/supervisord.conf"]

README.md

@@ -9,8 +9,8 @@ Easy to deploy and upgrade.
 
 ## ANNOUNCEMENT
 
-Some time VERY SOON we will merge the next branch based on Debian Buster to master.
+At this point we have merged the next branch based on Debian Buster into master.
-That means the docker image latest will change. The change may break things!
+That means the docker image latest uses Buster. The change may break things!
 
 The following possibly breaking changes are known:
 - Filebeat is removed and should be handled by another container, see [Wiki](https://github.com/tomav/docker-mailserver/wiki/).
@@ -122,7 +122,7 @@ If you got any problems with SPF and/or forwarding mails, give [SRS](https://git
 
 Your config folder will be mounted in `/tmp/docker-mailserver/`. To understand how things work on boot, please have a look at [start-mailserver.sh](https://github.com/tomav/docker-mailserver/blob/master/target/start-mailserver.sh)
 
-`restart: always` ensures that the mail server container (and ELK container when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit.
+`restart: always` ensures that the mail server container (and Filebeat/ELK containers when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit.
 
 #### Exposed ports
 * 25 receiving email from other mailservers

config/filebeat.docker.yml

@@ -0,0 +1,16 @@
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      hints.default_config.enabled: false
+
+processors:
+- add_cloud_metadata: ~
+
+output.logstash:
+    hosts: ["127.0.0.1:5044"]

docker-compose.elk.yml.dist

@@ -8,6 +8,10 @@ services:
     container_name: ${CONTAINER_NAME}
     links:
     - elk
+    labels:
+    - "co.elastic.logs/enabled=true"
+    - "co.elastic.logs/module=system"
+    - "co.elastic.logs/fileset.stdout=syslog"
     ports:
     - "25:25"
     - "143:143"
@@ -24,13 +28,27 @@ services:
     - NET_ADMIN
     - SYS_PTRACE
     restart: always
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.6.1
+    user: root
+    volumes:
+    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
+    command: ["filebeat", "-e", "--strict.perms=false"]
+    restart: always
   elk:
-     build: elk
+     build:
+       context: elk
+       args:
+       - MAXMIND_LICENSE
      ports:
      - "5601:5601"
      - "9200:9200"
      - "5044:5044"
      - "5000:5000"
+     env_file:
+     - elk/.env
      restart: always
 
 volumes:

docker-compose.filebeat.yml.dist

@@ -0,0 +1,42 @@
+version: '2'
+services:
+  mail:
+    image: tvial/docker-mailserver:latest
+    hostname: ${HOSTNAME}
+    domainname: ${DOMAINNAME}
+    container_name: ${CONTAINER_NAME}
+    ports:
+    - "25:25"
+    - "143:143"
+    - "587:587"
+    - "993:993"
+    labels:
+    - "co.elastic.logs/enabled=true"
+    - "co.elastic.logs/module=system"
+    - "co.elastic.logs/fileset.stdout=syslog"
+    volumes:
+    - maildata:/var/mail
+    - mailstate:/var/mail-state
+    - maillogs:/var/log/mail
+    - ./config/:/tmp/docker-mailserver/
+    env_file:
+    - .env
+    - env-mailserver
+    cap_add:
+    - NET_ADMIN
+    - SYS_PTRACE
+    restart: always
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.6.1
+    user: root
+    volumes:
+    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
+    command: ["filebeat", "-e", "--strict.perms=false"]
+    restart: always
+volumes:
+  maildata:
+    driver: local
+  maillogs:
+    driver: local

elk/.env.dist

@@ -0,0 +1 @@
+MAXMIND_LICENSE=

elk/Dockerfile

@@ -1,4 +1,4 @@
-FROM sebp/elk:720
+FROM sebp/elk:761
 
 RUN mkdir /etc/logstash/patterns.d
 #postfix grok and filter
@@ -12,8 +12,11 @@ RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/pa
 RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf
 # FIXME: may be a cron job?
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ARG MAXMIND_LICENSE
 RUN mkdir  -p /usr/share/GeoIP && \
- curl -L http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat 
+curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \
+| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat
 
 WORKDIR ${LOGSTASH_HOME}
 RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip

elk/docker-compose.yml.dist

@@ -0,0 +1,16 @@
+version: '2'
+
+services:
+  elk:
+     build:
+       context: .
+       args:
+       - MAXMIND_LICENSE
+     ports:
+     - "5601:5601"
+     - "9200:9200"
+     - "5044:5044"
+     - "5000:5000"
+     env_file:
+     - .env
+     restart: always

target/amavis/conf.d/49-docker-mailserver

@@ -0,0 +1,12 @@
+use strict;
+
+# Override options set in earlier files, use 50-user to override these
+
+# Bounce spam, the default option for buster is D_PASS to deliver
+$final_spam_destiny = D_BOUNCE;
+
+# Higher log level to get expected messages at startup
+$log_level = 2;
+
+#------------ Do not modify anything below this line -------------
+1;  # ensure a defined return

target/fail2ban/filter.d/postfix-sasl.conf

@@ -0,0 +1,19 @@
+# Fail2Ban filter for postfix authentication failures
+
+[INCLUDES]
+
+before = common.conf
+
+[Definition]
+
+_daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
+
+failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$
+
+ignoreregex = authentication failed: Connection lost to authentication server$
+
+[Init]
+
+journalmatch = _SYSTEMD_UNIT=postfix.service
+
+ignoreregex =

target/filebeat.yml.tmpl

@@ -1,13 +0,0 @@
-output:
-  logstash:
-    enabled: true
-    hosts:
-      - $ELK_HOST:$ELK_PORT
-
-filebeat:
-  prospectors:
-    -
-      paths:
-        - /var/log/mail/mail.log
-      document_type: syslog
-

target/start-mailserver.sh

@@ -96,10 +96,6 @@ function register_functions() {
 	_register_setup_function "_setup_default_vars"
 	_register_setup_function "_setup_file_permissions"
 
-	if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
-		_register_setup_function "_setup_elk_forwarder"
-	fi
-
 	if [ "$SMTP_ONLY" != 1 ]; then
 		_register_setup_function "_setup_dovecot"
                 _register_setup_function "_setup_dovecot_dhparam"
@@ -212,10 +208,6 @@ function register_functions() {
 	_register_start_daemon "_start_daemons_cron"
 	_register_start_daemon "_start_daemons_rsyslog"
 
-	if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
-		_register_start_daemon "_start_daemons_filebeat"
-	fi
-
 	if [ "$SMTP_ONLY" != 1 ]; then
 		_register_start_daemon "_start_daemons_dovecot"
 	fi
@@ -1465,18 +1457,6 @@ function _setup_security_stack() {
 	fi
 }
 
-function _setup_elk_forwarder() {
-	notify 'task' 'Setting up Elk forwarder'
-
-	ELK_PORT=${ELK_PORT:="5044"}
-	ELK_HOST=${ELK_HOST:="elk"}
-	notify 'inf' "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)"
-	cat /etc/filebeat/filebeat.yml.tmpl \
-		| sed "s@\$ELK_HOST@$ELK_HOST@g" \
-		| sed "s@\$ELK_PORT@$ELK_PORT@g" \
-		> /etc/filebeat/filebeat.yml
-}
-
 function _setup_logrotate() {
 	notify 'inf' "Setting up logrotate"
 
@@ -1763,11 +1743,6 @@ function _start_daemons_dovecot() {
 	#fi
 }
 
-function _start_daemons_filebeat() {
-	notify 'task' 'Starting filebeat' 'n'
-    supervisorctl start filebeat
-}
-
 function _start_daemons_fetchmail() {
 	notify 'task' 'Starting fetchmail' 'n'
 	/usr/local/bin/setup-fetchmail

target/supervisor/conf.d/supervisor-app.conf

@@ -65,14 +65,6 @@ stdout_logfile=/var/log/supervisor/%(program_name)s.log
 stderr_logfile=/var/log/supervisor/%(program_name)s.log
 command=/usr/sbin/dovecot -F -c /etc/dovecot/dovecot.conf
 
-[program:filebeat]
-startsecs=0
-autostart=false
-autorestart=true
-stdout_logfile=/var/log/supervisor/%(program_name)s.log
-stderr_logfile=/var/log/supervisor/%(program_name)s.log
-command=/usr/bin/filebeat -c /etc/filebeat/filebeat.yml
-
 [program:clamav]
 startsecs=0
 autostart=false

test/tests.bats

@@ -217,10 +217,8 @@ function count_processed_changes() {
 }
 
 @test "checking smtp: delivers mail to existing account" {
-  #run docker exec mail /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | sed 's/.* to=</</g' | sed 's/, relay.*//g' | sort | uniq -c | tr -s \" \" | tr '\n' ';'"
   run docker exec mail /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | sed 's/.* to=</</g' | sed 's/, relay.*//g' | sort | uniq -c | tr -s \" \""
   assert_success
-  #assert_output " 1 <added@localhost.localdomain>; 6 <user1@localhost.localdomain>; 1 <user1@localhost.localdomain>, orig_to=<postmaster@my-domain.com>; 1 <user1@localhost.localdomain>, orig_to=<root>; 1 <user1~test@localhost.localdomain>; 2 <user2@otherdomain.tld>;"
   cat <<'EOF' | assert_output
  1 <added@localhost.localdomain>
  6 <user1@localhost.localdomain>
@@ -284,7 +282,7 @@ EOF
 }
 
 @test "checking smtp: redirects mail to external aliases" {
-  run docker exec mail /bin/sh -c "grep -- '-> <external1@otherdomain.tld>' /var/log/mail/mail.log | wc -l"
+  run docker exec mail /bin/sh -c "grep -- '-> <external1@otherdomain.tld>' /var/log/mail/mail.log* | grep RelayedInbound | wc -l"
   assert_success
   assert_output 2
 }
@@ -785,9 +783,37 @@ EOF
 }
 
 @test "checking system: amavis decoders installed and available" {
-  run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq | tr '\n' ';'"
+  run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log*|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq"
   assert_success
-  assert_output "7z;Z;arc;arj;bz2;cab;cpio;deb;doc;exe;gz;iso;jar;kmz;lha;lrz;lz4;lzma;lzo;mail;rar;rpm;swf;tar;tnef;xz;zip;zoo;"
+  # Support for doc and zoo removed in buster
+  cat <<'EOF' | assert_output
+7z
+Z
+arc
+arj
+bz2
+cab
+cpio
+deb
+exe
+gz
+iso
+jar
+kmz
+lha
+lrz
+lz4
+lzma
+lzo
+mail
+rar
+rpm
+swf
+tar
+tnef
+xz
+zip
+EOF
 }
 
 
@@ -1307,6 +1333,6 @@ EOF
 #
 
 @test "checking that the container stops cleanly" {
-  run docker stop -t 60 mail
+  run docker stop -t 60 mail_override_hostname
   assert_success
 }