diff --git a/Dockerfile b/Dockerfile index 78bf6d64..b915ff3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,10 +41,16 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -q --fix-missing && \ && \ curl -sk http://neuro.debian.net/lists/trusty.de-m.libre > /etc/apt/sources.list.d/neurodebian.sources.list && \ apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ - apt-get update -q --fix-missing && apt-get -y upgrade fail2ban && \ + curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ + echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ + apt-get update -q --fix-missing && apt-get -y upgrade fail2ban filebeat && \ apt-get autoclean && rm -rf /var/lib/apt/lists/* && \ rm -rf /usr/share/locale/* && rm -rf /usr/share/man/* && rm -rf /usr/share/doc/* +# Enables Clamav +RUN (echo "0 0,6,12,18 * * * /usr/bin/freshclam --quiet" ; crontab -l) | crontab - +RUN chmod 644 /etc/clamav/freshclam.conf && freshclam + # Configures Dovecot RUN sed -i -e 's/include_try \/usr\/share\/dovecot\/protocols\.d/include_try \/etc\/dovecot\/protocols\.d/g' /etc/dovecot/dovecot.conf RUN sed -i -e 's/#mail_plugins = \$mail_plugins/mail_plugins = \$mail_plugins sieve/g' /etc/dovecot/conf.d/15-lda.conf @@ -58,16 +64,13 @@ RUN sed -i -r 's/^(CRON)=0/\1=1/g' /etc/default/spamassassin RUN sed -i -r 's/#(@| \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_filter_mode RUN adduser clamav amavis && adduser amavis clamav RUN useradd -u 5000 -d /home/docker -s /bin/bash -p $(echo docker | openssl passwd -1 -stdin) docker +RUN (echo "0 4 * * * find /var/lib/amavis/virusmails/ -type f -mtime +\$VIRUSMAILS_DELETE_DELAY -delete" ; crontab -l) | crontab - # Configure Fail2ban COPY target/fail2ban/jail.conf /etc/fail2ban/jail.conf COPY target/fail2ban/filter.d/dovecot.conf /etc/fail2ban/filter.d/dovecot.conf RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf -# Enables Clamav -RUN (crontab; echo "0 0,6,12,18 * * * /usr/bin/freshclam --quiet") | sort - | uniq - | crontab - -RUN chmod 644 /etc/clamav/freshclam.conf && freshclam - # Enables Pyzor and Razor USER amavis RUN razor-admin -create && razor-admin -register && pyzor discover @@ -114,3 +117,8 @@ RUN chmod +x /usr/local/bin/* EXPOSE 25 587 143 993 110 995 4190 CMD /usr/local/bin/start-mailserver.sh + + +ADD target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl + + diff --git a/Makefile b/Makefile index ec431d2b..ac3abe43 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ run: -e SA_TAG=1.0 \ -e SA_TAG2=2.0 \ -e SA_KILL=3.0 \ + -e VIRUSMAILS_DELETE_DELAY=7 \ -e SASL_PASSWD="external-domain.com username:password" \ -e ENABLE_MANAGESIEVE=1 \ - -e ONE_DIR=1 \ -e PERMIT_DOCKER=host\ -h mail.my-domain.com -t $(NAME) sleep 20 @@ -145,7 +145,7 @@ clean: rm -f config/postfix-accounts.cf ;\ mv config/postfix-accounts.cf.bak config/postfix-accounts.cf ;\ fi - -rm -rf test/onedir \ + -sudo rm -rf test/onedir \ test/config/empty \ test/config/without-accounts \ test/config/without-virtual diff --git a/README.md b/README.md index e623b29a..ca9cee00 100644 --- a/README.md +++ b/README.md @@ -222,3 +222,7 @@ Set different options for mynetworks option (can be overwrite in postfix-main.cf ##### KOPANO_DAGENT - => Specify the dns-name/ip-address where the kopano-dagent can be reached +##### VIRUSMAILS_DELETE_DELAY + +Set how many days a virusmail will stay on the server before being deleted + - **empty** => 7 days diff --git a/config/postfix-ldap-aliases.cf b/config/postfix-ldap-aliases.cf index 7fb66cf8..a0389705 100644 --- a/config/postfix-ldap-aliases.cf +++ b/config/postfix-ldap-aliases.cf @@ -1,10 +1,9 @@ -server_host = 192.168.0.100 -search_base = ou=Users,dc=example,dc=local +server_host = localhost +search_base = ou=Users,dc=example,dc=com version = 3 -bind = yes -bind_dn = cn=kopano,ou=Users,dc=example,dc=local -bind_pw = secret scope = sub -query_filter = (&(objectClass=user)(otherMailbox=%s)) +query_filter = (&(objectClass=posixAccount)(kopanoAliases=%s)) result_attribute = mail +bind_dn = +bind_pw = bind = yes diff --git a/config/postfix-ldap-groups.cf b/config/postfix-ldap-groups.cf index c6ede4e7..f91fcc4c 100644 --- a/config/postfix-ldap-groups.cf +++ b/config/postfix-ldap-groups.cf @@ -1,10 +1,9 @@ -server_host = 192.168.0.100 -search_base = ou=groups,dc=example,dc=local +server_host = localhost +search_base = ou=Groups,dc=exampple,dc=com version = 3 -bind = yes -bind_dn = cn=kopano,ou=Users,dc=example,dc=local -bind_pw = secret -query_filter = (&(objectclass=group)(mail=%s)) +query_filter = (&(objectclass=kopano-group)(mail=%s)) leaf_result_attribute = mail special_result_attribute = member +bind_dn = +bind_pw = bind = yes diff --git a/config/postfix-ldap-users.cf b/config/postfix-ldap-users.cf index af4280e4..b817a580 100644 --- a/config/postfix-ldap-users.cf +++ b/config/postfix-ldap-users.cf @@ -1,10 +1,9 @@ -server_host = 192.168.0.100 -search_base = ou=Users,dc=example,dc=local +server_host = localhost +search_base = ou=Users,dc=example,dc=com version = 3 -bind = yes -bind_dn = cn=kopano,ou=Users,dc=example,dc=local -bind_pw = secret scope = sub -query_filter = (&(objectClass=user)(mail=%s)) +query_filter = (&(objectClass=posixAccount)(mail=%s)) result_attribute = mail +bind_dn = +bind_pw = bind = yes diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist new file mode 100644 index 00000000..75ffa10e --- /dev/null +++ b/docker-compose.elk.yml.dist @@ -0,0 +1,34 @@ +version: '2' + +services: + mail: + image: tvial/docker-mailserver:v2 + hostname: mail + domainname: domain.com + container_name: mail + links: + - elk + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + volumes: + - maildata:/var/mail + - ./config/:/tmp/docker-mailserver/ + environment: + - ENABLE_FAIL2BAN=1 + - ENABLE_ELK_FORWARDER=1 + cap_add: + - NET_ADMIN + elk: + build: elk + ports: + - "5601:5601" + - "9200:9200" + - "5044:5044" + - "5000:5000" + +volumes: + maildata: + driver: local diff --git a/elk/02-beats-input.conf b/elk/02-beats-input.conf new file mode 100644 index 00000000..a00d3f5b --- /dev/null +++ b/elk/02-beats-input.conf @@ -0,0 +1,6 @@ +input { + beats { + port => 5044 + ssl => false + } +} diff --git a/elk/10-syslog.conf b/elk/10-syslog.conf new file mode 100644 index 00000000..ae03326a --- /dev/null +++ b/elk/10-syslog.conf @@ -0,0 +1,14 @@ +filter { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{host}" ] + add_field => [ "program", "%{syslog_program}" ] + } + syslog_pri { } + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } +} + + diff --git a/elk/16-amavis.conf b/elk/16-amavis.conf new file mode 100644 index 00000000..ad60eff6 --- /dev/null +++ b/elk/16-amavis.conf @@ -0,0 +1,23 @@ +filter { + # grok log lines by program name + if [program] == 'amavis' { + grok { + patterns_dir => "/etc/logstash/patterns.d" + match => [ "message", "%{AMAVIS}" ] + tag_on_failure => [ "_grok_amavis_nomatch" ] + add_tag => [ "_grok_amavis_success" ] + } + } + + # Do some data type conversions + mutate { + convert => [ + # list of integer fields + "amavis_size", "integer", + "amavis_duration", "integer", + + # list of float fields + "amavis_hits", "float" + ] + } +} diff --git a/elk/Dockerfile b/elk/Dockerfile new file mode 100644 index 00000000..1ffeb104 --- /dev/null +++ b/elk/Dockerfile @@ -0,0 +1,23 @@ +FROM sebp/elk + +RUN mkdir /etc/logstash/patterns.d +#postfix grok and filter +RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok +RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf +# custom amavis grok and filter +ADD amavis.grok /etc/logstash/patterns.d +ADD 16-amavis.conf /etc/logstash/conf.d +# dovecot grok and filter +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf +# FIXME: may be a cron job? +RUN mkdir -p /usr/share/GeoIP && \ + curl -L http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat + +WORKDIR ${LOGSTASH_HOME} +RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip + +# override beats input +ADD 02-beats-input.conf /etc/logstash/conf.d/ +# override syslog +ADD 10-syslog.conf /etc/logstash/conf.d/ diff --git a/elk/amavis.grok b/elk/amavis.grok new file mode 100644 index 00000000..36713188 --- /dev/null +++ b/elk/amavis.grok @@ -0,0 +1 @@ +AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms diff --git a/target/filebeat.yml.tmpl b/target/filebeat.yml.tmpl new file mode 100644 index 00000000..d2d84ed4 --- /dev/null +++ b/target/filebeat.yml.tmpl @@ -0,0 +1,13 @@ +output: + logstash: + enabled: true + hosts: + - $ELK_HOST:$ELK_PORT + +filebeat: + prospectors: + - + paths: + - /var/log/mail/mail.log + document_type: syslog + diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index 5ed87092..df9c2de1 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -5,6 +5,12 @@ die () { exit 1 } +# +# Default variables +# + +echo "export VIRUSMAILS_DELETE_DELAY=${VIRUSMAILS_DELETE_DELAY:="7"}" >> /root/.bashrc + # # Users # @@ -442,10 +448,22 @@ if [ "$ONE_DIR" = 1 -a -d $statedir ]; then fi done fi +if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then +ELK_PORT=${ELK_PORT:="5044"} +ELK_HOST=${ELK_HOST:="elk"} +echo "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)" +cat /etc/filebeat/filebeat.yml.tmpl \ + | sed "s@\$ELK_HOST@$ELK_HOST@g" \ + | sed "s@\$ELK_PORT@$ELK_PORT@g" \ + > /etc/filebeat/filebeat.yml +fi echo "Starting daemons" cron /etc/init.d/rsyslog start +if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then +/etc/init.d/filebeat start +fi # Enable Managesieve service by setting the symlink # to the configuration file Dovecot will actually find diff --git a/test/tests.bats b/test/tests.bats index c9989ea5..d3b79d74 100644 --- a/test/tests.bats +++ b/test/tests.bats @@ -538,9 +538,34 @@ # @test "checking system: freshclam cron is enabled" { - run docker exec mail crontab -l + run docker exec mail bash -c "crontab -l | grep '/usr/bin/freshclam'" + [ "$status" -eq 0 ] +} + +@test "checking amavis: virusmail wiper cron exists" { + run docker exec mail bash -c "crontab -l | grep '/var/lib/amavis/virusmails/'" + [ "$status" -eq 0 ] +} + +@test "checking amavis: VIRUSMAILS_DELETE_DELAY override works as expected" { + run docker run -ti --rm -e VIRUSMAILS_DELETE_DELAY=2 `docker inspect --format '{{ .Config.Image }}' mail` /bin/bash -c 'echo $VIRUSMAILS_DELETE_DELAY | grep 2' + [ "$status" -eq 0 ] +} + +@test "checking amavis: old virusmail is wipped by cron" { + docker exec mail bash -c 'touch -d "`date --date=2000-01-01`" /var/lib/amavis/virusmails/should-be-deleted' + run docker exec -ti mail bash -c 'find /var/lib/amavis/virusmails/ -type f -mtime +$VIRUSMAILS_DELETE_DELAY -delete' + [ "$status" -eq 0 ] + run docker exec mail bash -c 'ls -la /var/lib/amavis/virusmails/ | grep should-be-deleted' + [ "$status" -eq 1 ] +} + +@test "checking amavis: recent virusmail is not wipped by cron" { + docker exec mail bash -c 'touch -d "`date`" /var/lib/amavis/virusmails/should-not-be-deleted' + run docker exec -ti mail bash -c 'find /var/lib/amavis/virusmails/ -type f -mtime +$VIRUSMAILS_DELETE_DELAY -delete' + [ "$status" -eq 0 ] + run docker exec mail bash -c 'ls -la /var/lib/amavis/virusmails/ | grep should-not-be-deleted' [ "$status" -eq 0 ] - [ "$output" = "0 0,6,12,18 * * * /usr/bin/freshclam --quiet" ] } @test "checking system: /var/log/mail/mail.log is error free" {