fixing merge conflict

Dockerfile

@@ -41,10 +41,16 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -q --fix-missing && \
     && \
   curl -sk http://neuro.debian.net/lists/trusty.de-m.libre > /etc/apt/sources.list.d/neurodebian.sources.list && \
   apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \
-  apt-get update -q --fix-missing && apt-get -y upgrade fail2ban && \
+  curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \
+  echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \
+  apt-get update -q --fix-missing && apt-get -y upgrade fail2ban filebeat && \
   apt-get autoclean && rm -rf /var/lib/apt/lists/* && \
   rm -rf /usr/share/locale/* && rm -rf /usr/share/man/* && rm -rf /usr/share/doc/*
 
+# Enables Clamav
+RUN (echo "0 0,6,12,18 * * * /usr/bin/freshclam --quiet" ; crontab -l) | crontab -
+RUN chmod 644 /etc/clamav/freshclam.conf && freshclam
+
 # Configures Dovecot
 RUN sed -i -e 's/include_try \/usr\/share\/dovecot\/protocols\.d/include_try \/etc\/dovecot\/protocols\.d/g' /etc/dovecot/dovecot.conf
 RUN sed -i -e 's/#mail_plugins = \$mail_plugins/mail_plugins = \$mail_plugins sieve/g' /etc/dovecot/conf.d/15-lda.conf
@@ -58,16 +64,13 @@ RUN sed -i -r 's/^(CRON)=0/\1=1/g' /etc/default/spamassassin
 RUN sed -i -r 's/#(@|   \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_filter_mode
 RUN adduser clamav amavis && adduser amavis clamav
 RUN useradd -u 5000 -d /home/docker -s /bin/bash -p $(echo docker | openssl passwd -1 -stdin) docker
+RUN (echo "0 4 * * * find /var/lib/amavis/virusmails/ -type f -mtime +\$VIRUSMAILS_DELETE_DELAY -delete" ; crontab -l) | crontab -
 
 # Configure Fail2ban
 COPY target/fail2ban/jail.conf /etc/fail2ban/jail.conf
 COPY target/fail2ban/filter.d/dovecot.conf /etc/fail2ban/filter.d/dovecot.conf
 RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf
 
-# Enables Clamav
-RUN (crontab; echo "0 0,6,12,18 * * * /usr/bin/freshclam --quiet") | sort - | uniq - | crontab -
-RUN chmod 644 /etc/clamav/freshclam.conf && freshclam
-
 # Enables Pyzor and Razor
 USER amavis
 RUN razor-admin -create && razor-admin -register && pyzor discover
@@ -114,3 +117,8 @@ RUN chmod +x /usr/local/bin/*
 EXPOSE 25 587 143 993 110 995 4190
 
 CMD /usr/local/bin/start-mailserver.sh
+
+
+ADD target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl
+
+

Makefile

@@ -23,9 +23,9 @@ run:
 		-e SA_TAG=1.0 \
 		-e SA_TAG2=2.0 \
 		-e SA_KILL=3.0 \
+		-e VIRUSMAILS_DELETE_DELAY=7 \
 		-e SASL_PASSWD="external-domain.com username:password" \
 		-e ENABLE_MANAGESIEVE=1 \
-		-e ONE_DIR=1 \
 		-e PERMIT_DOCKER=host\
 		-h mail.my-domain.com -t $(NAME)
 	sleep 20
@@ -145,7 +145,7 @@ clean:
 		rm -f config/postfix-accounts.cf ;\
 		mv config/postfix-accounts.cf.bak config/postfix-accounts.cf ;\
 	fi
-	-rm -rf test/onedir \
+	-sudo rm -rf test/onedir \
 		test/config/empty \
 		test/config/without-accounts \
 		test/config/without-virtual

README.md

@@ -222,3 +222,7 @@ Set different options for mynetworks option (can be overwrite in postfix-main.cf
 ##### KOPANO_DAGENT
   - => Specify the dns-name/ip-address where the kopano-dagent can be reached
 
+##### VIRUSMAILS_DELETE_DELAY
+
+Set how many days a virusmail will stay on the server before being deleted
+  - **empty** => 7 days

config/postfix-ldap-aliases.cf

@@ -1,10 +1,9 @@
-server_host = 192.168.0.100
-search_base = ou=Users,dc=example,dc=local
+server_host = localhost
+search_base = ou=Users,dc=example,dc=com
 version = 3
-bind = yes
-bind_dn = cn=kopano,ou=Users,dc=example,dc=local
-bind_pw = secret
 scope = sub
-query_filter = (&(objectClass=user)(otherMailbox=%s))
+query_filter = (&(objectClass=posixAccount)(kopanoAliases=%s))
 result_attribute = mail
+bind_dn =
+bind_pw =
 bind = yes

config/postfix-ldap-groups.cf

@@ -1,10 +1,9 @@
-server_host = 192.168.0.100
-search_base = ou=groups,dc=example,dc=local
+server_host = localhost
+search_base = ou=Groups,dc=exampple,dc=com
 version = 3
-bind = yes
-bind_dn = cn=kopano,ou=Users,dc=example,dc=local
-bind_pw = secret
-query_filter = (&(objectclass=group)(mail=%s))
+query_filter = (&(objectclass=kopano-group)(mail=%s))
 leaf_result_attribute = mail
 special_result_attribute = member
+bind_dn =
+bind_pw =
 bind = yes

config/postfix-ldap-users.cf

@@ -1,10 +1,9 @@
-server_host = 192.168.0.100
-search_base = ou=Users,dc=example,dc=local
+server_host = localhost
+search_base = ou=Users,dc=example,dc=com
 version = 3
-bind = yes
-bind_dn = cn=kopano,ou=Users,dc=example,dc=local
-bind_pw = secret
 scope = sub
-query_filter = (&(objectClass=user)(mail=%s))
+query_filter = (&(objectClass=posixAccount)(mail=%s))
 result_attribute = mail
+bind_dn =
+bind_pw =
 bind = yes

docker-compose.elk.yml.dist

@@ -0,0 +1,34 @@
+version: '2'
+
+services:
+  mail:
+    image: tvial/docker-mailserver:v2
+    hostname: mail
+    domainname: domain.com
+    container_name: mail
+    links:
+    - elk
+    ports:
+    - "25:25"
+    - "143:143"
+    - "587:587"
+    - "993:993"
+    volumes:
+    - maildata:/var/mail
+    - ./config/:/tmp/docker-mailserver/
+    environment:
+    - ENABLE_FAIL2BAN=1
+    - ENABLE_ELK_FORWARDER=1
+    cap_add:
+    - NET_ADMIN
+  elk:
+     build: elk
+     ports:
+     - "5601:5601"
+     - "9200:9200"
+     - "5044:5044"
+     - "5000:5000"
+
+volumes:
+  maildata:
+    driver: local

elk/02-beats-input.conf

@@ -0,0 +1,6 @@
+input {
+  beats {
+    port => 5044
+    ssl => false
+  }
+}

elk/10-syslog.conf

@@ -0,0 +1,14 @@
+filter {
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+      add_field => [ "program", "%{syslog_program}" ]
+    }
+    syslog_pri { }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+    }
+}
+
+

elk/16-amavis.conf

@@ -0,0 +1,23 @@
+filter {
+    # grok log lines by program name
+    if [program] == 'amavis' {
+        grok {
+            patterns_dir   => "/etc/logstash/patterns.d"
+            match          => [ "message", "%{AMAVIS}" ]
+            tag_on_failure => [ "_grok_amavis_nomatch" ]
+            add_tag        => [ "_grok_amavis_success" ]
+        }
+    }
+
+    # Do some data type conversions
+    mutate {
+        convert => [
+            # list of integer fields
+            "amavis_size", "integer",
+            "amavis_duration", "integer",
+
+            # list of float fields
+            "amavis_hits", "float"
+        ]
+    }
+}

elk/Dockerfile

@@ -0,0 +1,23 @@
+FROM sebp/elk
+
+RUN mkdir /etc/logstash/patterns.d
+#postfix grok and filter
+RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok
+RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf
+# custom amavis grok and filter
+ADD amavis.grok  /etc/logstash/patterns.d
+ADD 16-amavis.conf /etc/logstash/conf.d
+# dovecot grok and filter
+RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok
+RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf
+# FIXME: may be a cron job? 
+RUN mkdir  -p /usr/share/GeoIP && \
+ curl -L http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat 
+
+WORKDIR ${LOGSTASH_HOME}
+RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip
+
+# override beats input 
+ADD 02-beats-input.conf /etc/logstash/conf.d/
+# override syslog
+ADD 10-syslog.conf /etc/logstash/conf.d/

elk/amavis.grok

@@ -0,0 +1 @@
+AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms

target/filebeat.yml.tmpl

@@ -0,0 +1,13 @@
+output:
+  logstash:
+    enabled: true
+    hosts:
+      - $ELK_HOST:$ELK_PORT
+
+filebeat:
+  prospectors:
+    -
+      paths:
+        - /var/log/mail/mail.log
+      document_type: syslog
+

target/start-mailserver.sh

@@ -5,6 +5,12 @@ die () {
   exit 1
 }
 
+#
+# Default variables
+#
+
+echo "export VIRUSMAILS_DELETE_DELAY=${VIRUSMAILS_DELETE_DELAY:="7"}" >> /root/.bashrc
+
 #
 # Users
 #
@@ -442,10 +448,22 @@ if [ "$ONE_DIR" = 1 -a -d $statedir ]; then
     fi
   done
 fi
+if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
+ELK_PORT=${ELK_PORT:="5044"}
+ELK_HOST=${ELK_HOST:="elk"}
+echo "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)"
+cat /etc/filebeat/filebeat.yml.tmpl \
+	| sed "s@\$ELK_HOST@$ELK_HOST@g" \
+	| sed "s@\$ELK_PORT@$ELK_PORT@g" \
+	 > /etc/filebeat/filebeat.yml
+fi
 
 echo "Starting daemons"
 cron
 /etc/init.d/rsyslog start
+if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
+/etc/init.d/filebeat start
+fi
 
 # Enable Managesieve service by setting the symlink
 # to the configuration file Dovecot will actually find

test/tests.bats

@@ -538,9 +538,34 @@
 #
 
 @test "checking system: freshclam cron is enabled" {
-  run docker exec mail crontab -l
+  run docker exec mail bash -c "crontab -l | grep '/usr/bin/freshclam'"
+  [ "$status" -eq 0 ]
+}
+
+@test "checking amavis: virusmail wiper cron exists" {
+  run docker exec mail bash -c "crontab -l | grep '/var/lib/amavis/virusmails/'"
+  [ "$status" -eq 0 ]
+}
+
+@test "checking amavis: VIRUSMAILS_DELETE_DELAY override works as expected" {
+  run docker run -ti --rm -e VIRUSMAILS_DELETE_DELAY=2 `docker inspect --format '{{ .Config.Image }}' mail` /bin/bash -c 'echo $VIRUSMAILS_DELETE_DELAY | grep 2' 
+  [ "$status" -eq 0 ]
+}
+
+@test "checking amavis: old virusmail is wipped by cron" {
+  docker exec mail bash -c 'touch -d "`date --date=2000-01-01`" /var/lib/amavis/virusmails/should-be-deleted'
+  run docker exec -ti mail bash -c 'find /var/lib/amavis/virusmails/ -type f -mtime +$VIRUSMAILS_DELETE_DELAY -delete'
+  [ "$status" -eq 0 ]
+  run docker exec mail bash -c 'ls -la /var/lib/amavis/virusmails/ | grep should-be-deleted'
+  [ "$status" -eq 1 ]
+}
+
+@test "checking amavis: recent virusmail is not wipped by cron" {
+  docker exec mail bash -c 'touch -d "`date`"  /var/lib/amavis/virusmails/should-not-be-deleted'
+  run docker exec -ti mail bash -c 'find /var/lib/amavis/virusmails/ -type f -mtime +$VIRUSMAILS_DELETE_DELAY -delete'
+  [ "$status" -eq 0 ]
+  run docker exec mail bash -c 'ls -la /var/lib/amavis/virusmails/ | grep should-not-be-deleted'
   [ "$status" -eq 0 ]
-  [ "$output" = "0 0,6,12,18 * * * /usr/bin/freshclam --quiet" ]
 }
 
 @test "checking system: /var/log/mail/mail.log is error free" {