finetune logstash image and compose settings

docker/elk/logstash/Dockerfile

@@ -34,14 +34,15 @@ RUN apt-get update -y && \
     cd /root/dist/ && \
     cp entrypoint.sh /usr/bin/ && \
     chmod u+x /usr/bin/entrypoint.sh && \
-    mkdir -p /etc/logstash/conf.d /usr/share/logstash/config && \
-    cp logstash.conf /etc/logstash/conf.d/ && \
-    cp http_input.conf /etc/logstash/conf.d/ && \
-    cp http_output.conf /etc/logstash/conf.d/ && \
+    mkdir -p /usr/share/logstash/config && \
+    cp logstash.conf /etc/logstash/ && \
+    cp http_input.conf /etc/logstash/ && \
+    cp http_output.conf /etc/logstash/ && \
     cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \
     cp pipelines_sensor.yml /usr/share/logstash/config/pipelines_sensor.yml && \
     cp tpot-template.json /etc/logstash/ && \
     rm /etc/logstash/pipelines.yml && \
+    rm /etc/logstash/logstash.yml && \
 #
 # Setup user, groups and configs
     groupmod -g 2000 logstash && \

docker/elk/logstash/dist/entrypoint.sh

@@ -82,6 +82,6 @@ if [ "$myTPOTILM" == "1" ];
       }
     }'
   else
-    echo "T-Pot ILM already configured."
+    echo "T-Pot ILM already configured or ES not available."
 fi
 echo

docker/elk/logstash/dist/http_input.conf

@@ -4,6 +4,7 @@ input {
     id => "tpot"
     host => "0.0.0.0"
     port => "64305"
+    ecs_compatibility => disabled
   }
 }
 

docker/elk/logstash/dist/pipelines.yml

@@ -1,6 +1,6 @@
 - pipeline.id: logstash
-  path.config: "/etc/logstash/conf.d/logstash.conf"
+  path.config: "/etc/logstash/logstash.conf"
   pipeline.ecs_compatibility: disabled
 - pipeline.id: http_input
-  path.config: "/etc/logstash/conf.d/http_input.conf"
+  path.config: "/etc/logstash/http_input.conf"
   pipeline.ecs_compatibility: disabled

docker/elk/logstash/dist/pipelines_sensor.yml

@@ -1,3 +1,3 @@
 - pipeline.id: http_output
-  path.config: "/etc/logstash/conf.d/http_output.conf"
+  path.config: "/etc/logstash/http_output.conf"
   pipeline.ecs_compatibility: disabled

etc/compose/collector.yml

@@ -151,8 +151,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/hive.yml

@@ -18,7 +18,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -52,8 +52,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/hive_sensor.yml

@@ -508,8 +508,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     env_file:
      - /opt/tpot/etc/compose/elk_environment
     mem_limit: 2g

etc/compose/industrial.yml

@@ -322,8 +322,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/log4j.yml

@@ -141,8 +141,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/medical.yml

@@ -135,8 +135,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/mini.yml

@@ -160,8 +160,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/nextgen.yml

@@ -466,8 +466,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/standard.yml

@@ -545,8 +545,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy

etc/compose/tarpit.yml

@@ -178,8 +178,8 @@ services:
   logstash:
     container_name: logstash
     restart: always
-#    environment:
-#     - LS_JAVA_OPTS=-Xms2048m -Xmx2048m
+    environment:
+     - LS_JAVA_OPTS=-Xms1024m -Xmx1024m
     depends_on:
       elasticsearch:
         condition: service_healthy