From 6b4d1eb4a1e1473dd43bb190d14bf782a29d0c3d Mon Sep 17 00:00:00 2001 From: ppatrik Date: Wed, 10 Nov 2021 20:51:54 +0100 Subject: [PATCH] Works for moving to central elasticsearch database. --- docker/elk/logstash/dist/logstash.conf | 29 +++++++++++++++++++------- docker/elk/logstash/docker-compose.yml | 1 + etc/compose/collector.yml | 1 + etc/compose/elk_collector | 8 +++++++ etc/compose/industrial.yml | 1 + etc/compose/medical.yml | 1 + etc/compose/nextgen.yml | 1 + etc/compose/sensor.yml | 16 ++++++++++++++ etc/compose/standard.yml | 1 + 9 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 etc/compose/elk_collector diff --git a/docker/elk/logstash/dist/logstash.conf b/docker/elk/logstash/dist/logstash.conf index ae937fdf..0a78cd6c 100644 --- a/docker/elk/logstash/dist/logstash.conf +++ b/docker/elk/logstash/dist/logstash.conf @@ -690,13 +690,28 @@ if "_jsonparsefailure" in [tags] { drop {} } # Output section output { - elasticsearch { - hosts => ["elasticsearch:9200"] - # With templates now being legacy and ILM in place we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana. - index => "logstash-%{+YYYY.MM.dd}" - template => "/etc/logstash/tpot_es_template.json" -# document_type => "doc" - } + # TODO: switch by auth config + #if "${MY_ELK_COLLECTOR_AUTH}" == "basic" { + elasticsearch { + hosts => ["${MY_ELK_COLLECTOR_HOST}"] + user => "${MY_ELK_COLLECTOR_USER}" + password => "${MY_ELK_COLLECTOR_PASSWORD}" + ssl => true + ssl_certificate_verification => false + # With templates now being legacy and ILM in place we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana. + index => "logstash-%{+YYYY.MM.dd}" + template => "/etc/logstash/tpot_es_template.json" + #document_type => "doc" + } + #} else { + # elasticsearch { + # hosts => ["${MY_ELK_COLLECTOR_HOST}"] + # # With templates now being legacy and ILM in place we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana. + # index => "logstash-%{+YYYY.MM.dd}" + # template => "/etc/logstash/tpot_es_template.json" + # #document_type => "doc" + # } + #} #if [type] == "Suricata" { # file { diff --git a/docker/elk/logstash/docker-compose.yml b/docker/elk/logstash/docker-compose.yml index ed94864b..da00ea24 100644 --- a/docker/elk/logstash/docker-compose.yml +++ b/docker/elk/logstash/docker-compose.yml @@ -14,6 +14,7 @@ services: # condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment + - /opt/tpot/etc/compose/elk_collector image: "dtagdevsec/logstash:2006" volumes: - /data:/data diff --git a/etc/compose/collector.yml b/etc/compose/collector.yml index b20c5125..44df7498 100644 --- a/etc/compose/collector.yml +++ b/etc/compose/collector.yml @@ -167,6 +167,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment + - /opt/tpot/etc/compose/elk_collector image: "dtagdevsec/logstash:2006" volumes: - /data:/data diff --git a/etc/compose/elk_collector b/etc/compose/elk_collector new file mode 100644 index 00000000..59e4403f --- /dev/null +++ b/etc/compose/elk_collector @@ -0,0 +1,8 @@ +MY_ELK_COLLECTOR_HOST=elasticsearch:9200 +MY_ELK_COLLECTOR_AUTH=none +# MY_ELK_COLLECTOR_HOST=https://you_collector_ip:64297/es/ +# MY_ELK_COLLECTOR_AUTH=basic +# Password have to be created at collector host at file /data/nginx/conf/nginxpasswd +# MY_ELK_COLLECTOR_USER=username +# MY_ELK_COLLECTOR_PASSWORD=password + diff --git a/etc/compose/industrial.yml b/etc/compose/industrial.yml index 22839aa7..4e637925 100644 --- a/etc/compose/industrial.yml +++ b/etc/compose/industrial.yml @@ -374,6 +374,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment + - /opt/tpot/etc/compose/elk_collector image: "dtagdevsec/logstash:2006" volumes: - /data:/data diff --git a/etc/compose/medical.yml b/etc/compose/medical.yml index a51a6e86..fd06fae3 100644 --- a/etc/compose/medical.yml +++ b/etc/compose/medical.yml @@ -151,6 +151,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment + - /opt/tpot/etc/compose/elk_collector image: "dtagdevsec/logstash:2006" volumes: - /data:/data diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 37929a7e..1f16d05f 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -537,6 +537,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment + - /opt/tpot/etc/compose/elk_collector image: "dtagdevsec/logstash:2006" volumes: - /data:/data diff --git a/etc/compose/sensor.yml b/etc/compose/sensor.yml index 14d7f70a..a27bc733 100644 --- a/etc/compose/sensor.yml +++ b/etc/compose/sensor.yml @@ -535,3 +535,19 @@ services: volumes: - /data:/data - /data/ews/conf/ews.ip:/opt/ewsposter/ews.ip + +## Logstash service + logstash: + container_name: logstash + restart: always +# environment: +# - LS_JAVA_OPTS=-Xms2048m -Xmx2048m + depends_on: + elasticsearch: + condition: service_healthy + env_file: + - /opt/tpot/etc/compose/elk_environment + - /opt/tpot/etc/compose/elk_collector + image: "dtagdevsec/logstash:2006" + volumes: + - /data:/data \ No newline at end of file diff --git a/etc/compose/standard.yml b/etc/compose/standard.yml index 38297ed0..a110a30c 100644 --- a/etc/compose/standard.yml +++ b/etc/compose/standard.yml @@ -550,6 +550,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment + - /opt/tpot/etc/compose/elk_collector image: "dtagdevsec/logstash:2006" volumes: - /data:/data