From de38e5e86f1c1ada412f9e68db18979c4186cfc4 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Wed, 26 May 2021 11:00:49 +0000 Subject: [PATCH] Rebuild Logstash, Elasticsearch Setting static limits for Elasticsearch / Logstash on Xms, Xmx and Container RAM results in unwanted side effects for some installations. With Elastic supporting dynamic heap management for Java 14+ we now use OpenJDK 16 JRE and as such remove limitations. This should improve stability for T-Pot, provided the minimum requirements will be met. --- docker/elk/docker-compose.yml | 8 ++++---- docker/elk/elasticsearch/Dockerfile | 8 +++++--- docker/elk/logstash/Dockerfile | 5 +++-- etc/compose/collector.yml | 8 ++++---- etc/compose/industrial.yml | 8 ++++---- etc/compose/medical.yml | 8 ++++---- etc/compose/nextgen.yml | 8 ++++---- etc/compose/standard.yml | 8 ++++---- 8 files changed, 32 insertions(+), 29 deletions(-) diff --git a/docker/elk/docker-compose.yml b/docker/elk/docker-compose.yml index c49be155..432db3fc 100644 --- a/docker/elk/docker-compose.yml +++ b/docker/elk/docker-compose.yml @@ -10,7 +10,7 @@ services: restart: always environment: - bootstrap.memory_lock=true - - ES_JAVA_OPTS=-Xms2048m -Xmx2048m +# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK @@ -21,7 +21,7 @@ services: nofile: soft: 65536 hard: 65536 - mem_limit: 4g +# mem_limit: 4g ports: - "127.0.0.1:64298:9200" image: "ghcr.io/telekom-security/elasticsearch:2006" @@ -46,8 +46,8 @@ services: build: logstash/. container_name: logstash restart: always - environment: - - LS_JAVA_OPTS=-Xms2048m -Xmx2048m +# environment: +# - LS_JAVA_OPTS=-Xms2048m -Xmx2048m depends_on: elasticsearch: condition: service_healthy diff --git a/docker/elk/elasticsearch/Dockerfile b/docker/elk/elasticsearch/Dockerfile index 82e4bd23..36d8568c 100644 --- a/docker/elk/elasticsearch/Dockerfile +++ b/docker/elk/elasticsearch/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.13 # # VARS ENV ES_VER=7.12.1 \ - JAVA_HOME=/usr/lib/jvm/java-11-openjdk + ES_JAVA_HOME=/usr/lib/jvm/java-16-openjdk # Include dist ADD dist/ /root/dist/ # @@ -12,8 +12,9 @@ RUN apk -U --no-cache add \ aria2 \ bash \ curl \ - nss \ - openjdk11-jre && \ + nss && \ +# openjdk16-jre && \ + apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing openjdk16-jre && \ # # Get and install packages cd /root/dist/ && \ @@ -30,6 +31,7 @@ RUN apk -U --no-cache add \ addgroup -g 2000 elasticsearch && \ adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \ chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \ + rm -rf /usr/share/elasticsearch/jdk && \ rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ # # Clean up diff --git a/docker/elk/logstash/Dockerfile b/docker/elk/logstash/Dockerfile index dd941a27..c5cb7b90 100644 --- a/docker/elk/logstash/Dockerfile +++ b/docker/elk/logstash/Dockerfile @@ -14,8 +14,9 @@ RUN apk -U --no-cache add \ curl \ libc6-compat \ libzmq \ - nss \ - openjdk11-jre && \ + nss && \ +# openjdk16-jre && \ + apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing openjdk16-jre && \ # # Get and install packages mkdir -p /etc/listbot && \ diff --git a/etc/compose/collector.yml b/etc/compose/collector.yml index b18a95bd..b20c5125 100644 --- a/etc/compose/collector.yml +++ b/etc/compose/collector.yml @@ -127,7 +127,7 @@ services: restart: always environment: - bootstrap.memory_lock=true - - ES_JAVA_OPTS=-Xms2048m -Xmx2048m +# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK @@ -138,7 +138,7 @@ services: nofile: soft: 65536 hard: 65536 - mem_limit: 4g +# mem_limit: 4g ports: - "127.0.0.1:64298:9200" image: "dtagdevsec/elasticsearch:2006" @@ -160,8 +160,8 @@ services: logstash: container_name: logstash restart: always - environment: - - LS_JAVA_OPTS=-Xms2048m -Xmx2048m +# environment: +# - LS_JAVA_OPTS=-Xms2048m -Xmx2048m depends_on: elasticsearch: condition: service_healthy diff --git a/etc/compose/industrial.yml b/etc/compose/industrial.yml index b646311f..22839aa7 100644 --- a/etc/compose/industrial.yml +++ b/etc/compose/industrial.yml @@ -334,7 +334,7 @@ services: restart: always environment: - bootstrap.memory_lock=true - - ES_JAVA_OPTS=-Xms2048m -Xmx2048m +# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK @@ -345,7 +345,7 @@ services: nofile: soft: 65536 hard: 65536 - mem_limit: 4g +# mem_limit: 4g ports: - "127.0.0.1:64298:9200" image: "dtagdevsec/elasticsearch:2006" @@ -367,8 +367,8 @@ services: logstash: container_name: logstash restart: always - environment: - - LS_JAVA_OPTS=-Xms2048m -Xmx2048m +# environment: +# - LS_JAVA_OPTS=-Xms2048m -Xmx2048m depends_on: elasticsearch: condition: service_healthy diff --git a/etc/compose/medical.yml b/etc/compose/medical.yml index 9fad7fbb..a51a6e86 100644 --- a/etc/compose/medical.yml +++ b/etc/compose/medical.yml @@ -111,7 +111,7 @@ services: restart: always environment: - bootstrap.memory_lock=true - - ES_JAVA_OPTS=-Xms2048m -Xmx2048m +# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK @@ -122,7 +122,7 @@ services: nofile: soft: 65536 hard: 65536 - mem_limit: 4g +# mem_limit: 4g ports: - "127.0.0.1:64298:9200" image: "dtagdevsec/elasticsearch:2006" @@ -144,8 +144,8 @@ services: logstash: container_name: logstash restart: always - environment: - - LS_JAVA_OPTS=-Xms2048m -Xmx2048m +# environment: +# - LS_JAVA_OPTS=-Xms2048m -Xmx2048m depends_on: elasticsearch: condition: service_healthy diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index ff03e337..9bbbe69d 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -546,7 +546,7 @@ services: restart: always environment: - bootstrap.memory_lock=true - - ES_JAVA_OPTS=-Xms2048m -Xmx2048m +# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK @@ -557,7 +557,7 @@ services: nofile: soft: 65536 hard: 65536 - mem_limit: 4g +# mem_limit: 4g ports: - "127.0.0.1:64298:9200" image: "dtagdevsec/elasticsearch:2006" @@ -579,8 +579,8 @@ services: logstash: container_name: logstash restart: always - environment: - - LS_JAVA_OPTS=-Xms2048m -Xmx2048m +# environment: +# - LS_JAVA_OPTS=-Xms2048m -Xmx2048m depends_on: elasticsearch: condition: service_healthy diff --git a/etc/compose/standard.yml b/etc/compose/standard.yml index 7ddc9755..38297ed0 100644 --- a/etc/compose/standard.yml +++ b/etc/compose/standard.yml @@ -510,7 +510,7 @@ services: restart: always environment: - bootstrap.memory_lock=true - - ES_JAVA_OPTS=-Xms2048m -Xmx2048m +# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK @@ -521,7 +521,7 @@ services: nofile: soft: 65536 hard: 65536 - mem_limit: 4g +# mem_limit: 4g ports: - "127.0.0.1:64298:9200" image: "dtagdevsec/elasticsearch:2006" @@ -543,8 +543,8 @@ services: logstash: container_name: logstash restart: always - environment: - - LS_JAVA_OPTS=-Xms2048m -Xmx2048m +# environment: +# - LS_JAVA_OPTS=-Xms2048m -Xmx2048m depends_on: elasticsearch: condition: service_healthy