Rebuild Logstash, Elasticsearch

Setting static limits for Elasticsearch / Logstash on Xms, Xmx and Container RAM results in unwanted side effects for some installations. With Elastic supporting dynamic heap management for Java 14+ we now use OpenJDK 16 JRE and as such remove limitations. This should improve stability for T-Pot, provided the minimum requirements will be met.

docker/elk/docker-compose.yml

@@ -10,7 +10,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -21,7 +21,7 @@ services:
       nofile:
         soft: 65536
         hard: 65536
-    mem_limit: 4g
+#    mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
     image: "ghcr.io/telekom-security/elasticsearch:2006"
@@ -46,8 +46,8 @@ services:
     build: logstash/.
     container_name: logstash
     restart: always
-    environment:
-     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+#    environment:
+#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
     depends_on:
       elasticsearch:
         condition: service_healthy

docker/elk/elasticsearch/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:3.13
 #
 # VARS
 ENV ES_VER=7.12.1 \
-    JAVA_HOME=/usr/lib/jvm/java-11-openjdk
+    ES_JAVA_HOME=/usr/lib/jvm/java-16-openjdk
 # Include dist
 ADD dist/ /root/dist/
 #
@@ -12,8 +12,9 @@ RUN apk -U --no-cache add \
              aria2 \
              bash \
              curl \
-             nss \
-             openjdk11-jre && \
+             nss && \
+#             openjdk16-jre && \
+    apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing openjdk16-jre && \
 #
 # Get and install packages
     cd /root/dist/ && \
@@ -30,6 +31,7 @@ RUN apk -U --no-cache add \
     addgroup -g 2000 elasticsearch && \
     adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \
     chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \
+    rm -rf /usr/share/elasticsearch/jdk && \
     rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \
 #
 # Clean up

docker/elk/logstash/Dockerfile

@@ -14,8 +14,9 @@ RUN apk -U --no-cache add \
 	     curl \
              libc6-compat \
              libzmq \
-             nss \
-             openjdk11-jre && \
+             nss && \
+#             openjdk16-jre && \
+    apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing openjdk16-jre && \
 #
 # Get and install packages
     mkdir -p /etc/listbot && \

etc/compose/collector.yml

@@ -127,7 +127,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -138,7 +138,7 @@ services:
       nofile:
         soft: 65536
         hard: 65536
-    mem_limit: 4g
+#    mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
     image: "dtagdevsec/elasticsearch:2006"
@@ -160,8 +160,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-    environment:
-     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+#    environment:
+#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/industrial.yml

@@ -334,7 +334,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -345,7 +345,7 @@ services:
       nofile:
         soft: 65536
         hard: 65536
-    mem_limit: 4g
+#    mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
     image: "dtagdevsec/elasticsearch:2006"
@@ -367,8 +367,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-    environment:
-     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+#    environment:
+#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/medical.yml

@@ -111,7 +111,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -122,7 +122,7 @@ services:
       nofile:
         soft: 65536
         hard: 65536
-    mem_limit: 4g
+#    mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
     image: "dtagdevsec/elasticsearch:2006"
@@ -144,8 +144,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-    environment:
-     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+#    environment:
+#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/nextgen.yml

@@ -546,7 +546,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -557,7 +557,7 @@ services:
       nofile:
         soft: 65536
         hard: 65536
-    mem_limit: 4g
+#    mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
     image: "dtagdevsec/elasticsearch:2006"
@@ -579,8 +579,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-    environment:
-     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+#    environment:
+#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/standard.yml

@@ -510,7 +510,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -521,7 +521,7 @@ services:
       nofile:
         soft: 65536
         hard: 65536
-    mem_limit: 4g
+#    mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
     image: "dtagdevsec/elasticsearch:2006"
@@ -543,8 +543,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-    environment:
-     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+#    environment:
+#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
     depends_on:
       elasticsearch:
         condition: service_healthy