From e4bab5b99600e28d55febef52e7d09bf1d93d8ff Mon Sep 17 00:00:00 2001 From: Pablo Castorino Date: Thu, 29 Sep 2016 22:52:05 +0200 Subject: [PATCH 1/4] add ELK support (#331) * add support to forward logs to ELK stack. * from docker elk customize image with * https://github.com/whyscream/postfix-grok-patterns * custom imput * override syslog filter. * fix typo. * Explicit forwarder vars and messages. * add amavis grok * add dovecot grok * add geoip db * add logstash geoip plugin * add custom amavis grok from @tomav. * switch to filebeats input * refactor syslog filter * add filebeat * add template config * replace rsyslog with filebeat. --- Dockerfile | 9 ++++++++- docker-compose.elk.yml.dist | 34 ++++++++++++++++++++++++++++++++++ elk/02-beats-input.conf | 6 ++++++ elk/10-syslog.conf | 14 ++++++++++++++ elk/Dockerfile | 26 ++++++++++++++++++++++++++ elk/amavis.grok | 11 +++++++++++ target/filebeat.yml.tmpl | 13 +++++++++++++ target/start-mailserver.sh | 12 ++++++++++++ 8 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 docker-compose.elk.yml.dist create mode 100644 elk/02-beats-input.conf create mode 100644 elk/10-syslog.conf create mode 100644 elk/Dockerfile create mode 100644 elk/amavis.grok create mode 100644 target/filebeat.yml.tmpl diff --git a/Dockerfile b/Dockerfile index 37a24ecd..77346926 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -q --fix-missing && \ && \ curl -sk http://neuro.debian.net/lists/trusty.de-m.libre > /etc/apt/sources.list.d/neurodebian.sources.list && \ apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9 && \ - apt-get update -q --fix-missing && apt-get -y upgrade fail2ban && \ + curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ + echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ + apt-get update -q --fix-missing && apt-get -y upgrade fail2ban filebeat && \ apt-get autoclean && rm -rf /var/lib/apt/lists/* && \ rm -rf /usr/share/locale/* && rm -rf /usr/share/man/* && rm -rf /usr/share/doc/* @@ -112,3 +114,8 @@ RUN chmod +x /usr/local/bin/* EXPOSE 25 587 143 993 110 995 4190 CMD /usr/local/bin/start-mailserver.sh + + +ADD target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl + + diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist new file mode 100644 index 00000000..75ffa10e --- /dev/null +++ b/docker-compose.elk.yml.dist @@ -0,0 +1,34 @@ +version: '2' + +services: + mail: + image: tvial/docker-mailserver:v2 + hostname: mail + domainname: domain.com + container_name: mail + links: + - elk + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + volumes: + - maildata:/var/mail + - ./config/:/tmp/docker-mailserver/ + environment: + - ENABLE_FAIL2BAN=1 + - ENABLE_ELK_FORWARDER=1 + cap_add: + - NET_ADMIN + elk: + build: elk + ports: + - "5601:5601" + - "9200:9200" + - "5044:5044" + - "5000:5000" + +volumes: + maildata: + driver: local diff --git a/elk/02-beats-input.conf b/elk/02-beats-input.conf new file mode 100644 index 00000000..a00d3f5b --- /dev/null +++ b/elk/02-beats-input.conf @@ -0,0 +1,6 @@ +input { + beats { + port => 5044 + ssl => false + } +} diff --git a/elk/10-syslog.conf b/elk/10-syslog.conf new file mode 100644 index 00000000..ae03326a --- /dev/null +++ b/elk/10-syslog.conf @@ -0,0 +1,14 @@ +filter { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{host}" ] + add_field => [ "program", "%{syslog_program}" ] + } + syslog_pri { } + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } +} + + diff --git a/elk/Dockerfile b/elk/Dockerfile new file mode 100644 index 00000000..dc62c270 --- /dev/null +++ b/elk/Dockerfile @@ -0,0 +1,26 @@ +FROM sebp/elk + +RUN mkdir /etc/logstash/patterns.d +#postfix grok and filter +RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok +RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf +# custom amavis grok and filter +ADD amavis.grok /etc/logstash/patterns.d +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-amavis.conf > /etc/logstash/conf.d/16-filter-amavis.conf +# dovecot grok and filter +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok +RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf +# FIXME: may be a cron job? +RUN mkdir -p /usr/share/GeoIP && \ + curl -L http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat + +WORKDIR ${LOGSTASH_HOME} +RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip + +# override beats input +ADD 02-beats-input.conf /etc/logstash/conf.d/ +# override syslog +ADD 10-syslog.conf /etc/logstash/conf.d/ + + + diff --git a/elk/amavis.grok b/elk/amavis.grok new file mode 100644 index 00000000..4bc74859 --- /dev/null +++ b/elk/amavis.grok @@ -0,0 +1,11 @@ +MAVIS_MESSAGEID Message-ID: <%{DATA:amavis_message-id}> +AMAVIS_SIZE size: %{POSINT:amavis_size} +AMAVIS_TESTS Tests: \[%{DATA:amavis_tests}\] +AMAVIS_FROM From: %{DATA:amavis_header_from} +AMAVIS_HITS Hits: %{NUMBER:amavis_hits} +AMAVIS_QUARANTINE quarantine: %{NOTSPACE:amavis_quarantine} +AMAVIS_SUBJECT Subject: "%{DATA:amavis_subject}" +AMAVIS_KV ((%{AMAVIS_MESSAGEID}|%{AMAVIS_SIZE}|%{AMAVIS_TESTS}|%{AMAVIS_FROM}|%{AMAVIS_HITS}|%{AMAVIS_QUARANTINE}|%{AMAVIS_SUBJECT}|%{DATA}), )* + +AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms + diff --git a/target/filebeat.yml.tmpl b/target/filebeat.yml.tmpl new file mode 100644 index 00000000..d2d84ed4 --- /dev/null +++ b/target/filebeat.yml.tmpl @@ -0,0 +1,13 @@ +output: + logstash: + enabled: true + hosts: + - $ELK_HOST:$ELK_PORT + +filebeat: + prospectors: + - + paths: + - /var/log/mail/mail.log + document_type: syslog + diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index 81611b72..5d3aeae0 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -354,10 +354,22 @@ if [ "$ONE_DIR" = 1 -a -d $statedir ]; then fi done fi +if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then +ELK_PORT=${ELK_PORT:="5044"} +ELK_HOST=${ELK_HOST:="elk"} +echo "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)" +cat /etc/filebeat/filebeat.yml.tmpl \ + | sed "s@\$ELK_HOST@$ELK_HOST@g" \ + | sed "s@\$ELK_PORT@$ELK_PORT@g" \ + > /etc/filebeat/filebeat.yml +fi echo "Starting daemons" cron /etc/init.d/rsyslog start +if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then +/etc/init.d/filebeat start +fi # Enable Managesieve service by setting the symlink # to the configuration file Dovecot will actually find From a97c8075ee4c0af9b3b83a071eda240e45944102 Mon Sep 17 00:00:00 2001 From: Thomas VIAL Date: Fri, 30 Sep 2016 13:54:50 +0200 Subject: [PATCH 2/4] Fixes #340 - amavis_duration is now a number and not a string anmymore (#341) Fixes #340 - amavis_duration is now a number and not a string anymore --- elk/16-amavis.conf | 23 +++++++++++++++++++++++ elk/Dockerfile | 5 +---- elk/amavis.grok | 10 ---------- 3 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 elk/16-amavis.conf diff --git a/elk/16-amavis.conf b/elk/16-amavis.conf new file mode 100644 index 00000000..ad60eff6 --- /dev/null +++ b/elk/16-amavis.conf @@ -0,0 +1,23 @@ +filter { + # grok log lines by program name + if [program] == 'amavis' { + grok { + patterns_dir => "/etc/logstash/patterns.d" + match => [ "message", "%{AMAVIS}" ] + tag_on_failure => [ "_grok_amavis_nomatch" ] + add_tag => [ "_grok_amavis_success" ] + } + } + + # Do some data type conversions + mutate { + convert => [ + # list of integer fields + "amavis_size", "integer", + "amavis_duration", "integer", + + # list of float fields + "amavis_hits", "float" + ] + } +} diff --git a/elk/Dockerfile b/elk/Dockerfile index dc62c270..1ffeb104 100644 --- a/elk/Dockerfile +++ b/elk/Dockerfile @@ -6,7 +6,7 @@ RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/ma RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf # custom amavis grok and filter ADD amavis.grok /etc/logstash/patterns.d -RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-amavis.conf > /etc/logstash/conf.d/16-filter-amavis.conf +ADD 16-amavis.conf /etc/logstash/conf.d # dovecot grok and filter RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf @@ -21,6 +21,3 @@ RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filte ADD 02-beats-input.conf /etc/logstash/conf.d/ # override syslog ADD 10-syslog.conf /etc/logstash/conf.d/ - - - diff --git a/elk/amavis.grok b/elk/amavis.grok index 4bc74859..36713188 100644 --- a/elk/amavis.grok +++ b/elk/amavis.grok @@ -1,11 +1 @@ -MAVIS_MESSAGEID Message-ID: <%{DATA:amavis_message-id}> -AMAVIS_SIZE size: %{POSINT:amavis_size} -AMAVIS_TESTS Tests: \[%{DATA:amavis_tests}\] -AMAVIS_FROM From: %{DATA:amavis_header_from} -AMAVIS_HITS Hits: %{NUMBER:amavis_hits} -AMAVIS_QUARANTINE quarantine: %{NOTSPACE:amavis_quarantine} -AMAVIS_SUBJECT Subject: "%{DATA:amavis_subject}" -AMAVIS_KV ((%{AMAVIS_MESSAGEID}|%{AMAVIS_SIZE}|%{AMAVIS_TESTS}|%{AMAVIS_FROM}|%{AMAVIS_HITS}|%{AMAVIS_QUARANTINE}|%{AMAVIS_SUBJECT}|%{DATA}), )* - AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms - From 986dc97f1a695a350bb7d800a41097d78eef6503 Mon Sep 17 00:00:00 2001 From: Dennis Stumm Date: Fri, 30 Sep 2016 23:37:09 +0200 Subject: [PATCH 3/4] Fix rm erros in tests (#342) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 93441c23..b3dfcd4c 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ clean: rm -f config/postfix-accounts.cf ;\ mv config/postfix-accounts.cf.bak config/postfix-accounts.cf ;\ fi - -rm -rf test/onedir \ + -sudo rm -rf test/onedir \ test/config/empty \ test/config/without-accounts \ test/config/without-virtual From 08dc28e3041550646cf8b45ccfccada2a0ffde3c Mon Sep 17 00:00:00 2001 From: Thomas VIAL Date: Sat, 8 Oct 2016 19:02:47 +0200 Subject: [PATCH 4/4] Fixes #346 and added a virusmail wiper triggered by a CRON (#347) * Fixes #346 and added a virusmail wiper triggered by a CRON * Renamed env to something more explicit VIRUSMAILS_DELETE_DELAY --- Dockerfile | 9 +++++---- Makefile | 2 +- README.md | 5 +++++ target/start-mailserver.sh | 6 ++++++ test/tests.bats | 29 +++++++++++++++++++++++++++-- 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 77346926..98f220d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -q --fix-missing && \ apt-get autoclean && rm -rf /var/lib/apt/lists/* && \ rm -rf /usr/share/locale/* && rm -rf /usr/share/man/* && rm -rf /usr/share/doc/* +# Enables Clamav +RUN (echo "0 0,6,12,18 * * * /usr/bin/freshclam --quiet" ; crontab -l) | crontab - +RUN chmod 644 /etc/clamav/freshclam.conf && freshclam + # Configures Dovecot RUN sed -i -e 's/include_try \/usr\/share\/dovecot\/protocols\.d/include_try \/etc\/dovecot\/protocols\.d/g' /etc/dovecot/dovecot.conf RUN sed -i -e 's/#mail_plugins = \$mail_plugins/mail_plugins = \$mail_plugins sieve/g' /etc/dovecot/conf.d/15-lda.conf @@ -58,16 +62,13 @@ RUN sed -i -r 's/^(CRON)=0/\1=1/g' /etc/default/spamassassin RUN sed -i -r 's/#(@| \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_filter_mode RUN adduser clamav amavis && adduser amavis clamav RUN useradd -u 5000 -d /home/docker -s /bin/bash -p $(echo docker | openssl passwd -1 -stdin) docker +RUN (echo "0 4 * * * find /var/lib/amavis/virusmails/ -type f -mtime +\$VIRUSMAILS_DELETE_DELAY -delete" ; crontab -l) | crontab - # Configure Fail2ban COPY target/fail2ban/jail.conf /etc/fail2ban/jail.conf COPY target/fail2ban/filter.d/dovecot.conf /etc/fail2ban/filter.d/dovecot.conf RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf -# Enables Clamav -RUN (crontab; echo "0 0,6,12,18 * * * /usr/bin/freshclam --quiet") | sort - | uniq - | crontab - -RUN chmod 644 /etc/clamav/freshclam.conf && freshclam - # Enables Pyzor and Razor USER amavis RUN razor-admin -create && razor-admin -register && pyzor discover diff --git a/Makefile b/Makefile index b3dfcd4c..18545b96 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ run: -e SA_TAG=1.0 \ -e SA_TAG2=2.0 \ -e SA_KILL=3.0 \ + -e VIRUSMAILS_DELETE_DELAY=7 \ -e SASL_PASSWD="external-domain.com username:password" \ -e ENABLE_MANAGESIEVE=1 \ - -e ONE_DIR=1 \ -e PERMIT_DOCKER=host\ -h mail.my-domain.com -t $(NAME) sleep 20 diff --git a/README.md b/README.md index 80b65ea3..71ae5f8e 100644 --- a/README.md +++ b/README.md @@ -156,3 +156,8 @@ Set different options for mynetworks option (can be overwrite in postfix-main.cf - **empty** => localhost only - host => Add docker host (ipv4 only) - network => Add all docker containers (ipv4 only) + +##### VIRUSMAILS_DELETE_DELAY + +Set how many days a virusmail will stay on the server before being deleted + - **empty** => 7 days diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index 5d3aeae0..968ad9af 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -5,6 +5,12 @@ die () { exit 1 } +# +# Default variables +# + +echo "export VIRUSMAILS_DELETE_DELAY=${VIRUSMAILS_DELETE_DELAY:="7"}" >> /root/.bashrc + # # Users # diff --git a/test/tests.bats b/test/tests.bats index c9989ea5..d3b79d74 100644 --- a/test/tests.bats +++ b/test/tests.bats @@ -538,9 +538,34 @@ # @test "checking system: freshclam cron is enabled" { - run docker exec mail crontab -l + run docker exec mail bash -c "crontab -l | grep '/usr/bin/freshclam'" + [ "$status" -eq 0 ] +} + +@test "checking amavis: virusmail wiper cron exists" { + run docker exec mail bash -c "crontab -l | grep '/var/lib/amavis/virusmails/'" + [ "$status" -eq 0 ] +} + +@test "checking amavis: VIRUSMAILS_DELETE_DELAY override works as expected" { + run docker run -ti --rm -e VIRUSMAILS_DELETE_DELAY=2 `docker inspect --format '{{ .Config.Image }}' mail` /bin/bash -c 'echo $VIRUSMAILS_DELETE_DELAY | grep 2' + [ "$status" -eq 0 ] +} + +@test "checking amavis: old virusmail is wipped by cron" { + docker exec mail bash -c 'touch -d "`date --date=2000-01-01`" /var/lib/amavis/virusmails/should-be-deleted' + run docker exec -ti mail bash -c 'find /var/lib/amavis/virusmails/ -type f -mtime +$VIRUSMAILS_DELETE_DELAY -delete' + [ "$status" -eq 0 ] + run docker exec mail bash -c 'ls -la /var/lib/amavis/virusmails/ | grep should-be-deleted' + [ "$status" -eq 1 ] +} + +@test "checking amavis: recent virusmail is not wipped by cron" { + docker exec mail bash -c 'touch -d "`date`" /var/lib/amavis/virusmails/should-not-be-deleted' + run docker exec -ti mail bash -c 'find /var/lib/amavis/virusmails/ -type f -mtime +$VIRUSMAILS_DELETE_DELAY -delete' + [ "$status" -eq 0 ] + run docker exec mail bash -c 'ls -la /var/lib/amavis/virusmails/ | grep should-not-be-deleted' [ "$status" -eq 0 ] - [ "$output" = "0 0,6,12,18 * * * /usr/bin/freshclam --quiet" ] } @test "checking system: /var/log/mail/mail.log is error free" {