logstash cleanup, prep for multiarch, move to ubuntu

log4pot tweaking

docker/elk/logstash/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.15
+FROM ubuntu:20.04
 #
 # VARS
 ENV LS_VER=7.16.3
@@ -6,65 +6,59 @@ ENV LS_VER=7.16.3
 ADD dist/ /root/dist/
 #
 # Setup env and apt
-#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
+RUN apt-get update -y && \
-RUN apk -U --no-cache add \
+    apt-get dist-upgrade -y && \
+    apt-get install -y \
              aria2 \
 	     autossh \
              bash \
              bzip2 \
 	     curl \
-             libc6-compat \
+             openssh-client && \
-             libzmq \
-             nss \
-             openssh && \
-    apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community openjdk16-jre && \
 #
-# Get and install packages
+# Determine arch, get and install packages
+    ARCH=$(arch) && \
+      if [ "$ARCH" = "x86_64" ]; then LS_ARCH="amd64"; fi && \
+      if [ "$ARCH" = "aarch64" ]; then LS_ARCH="arm64"; fi && \
+    echo "$ARCH" && \
     mkdir -p /etc/listbot && \
     cd /etc/listbot && \
     aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \
     aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \
     bunzip2 *.bz2 && \
     cd /root/dist/ && \
-    mkdir -p /usr/share/logstash/ && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-$LS_ARCH.deb && \
-    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-linux-x86_64.tar.gz && \
+    dpkg -i logstash-$LS_VER-$LS_ARCH.deb && \
-    tar xvfz logstash-$LS_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
+#    /usr/share/logstash/bin/logstash-plugin install logstash-output-gelf logstash-output-syslog && \
-    rm -rf /usr/share/logstash/jdk && \
-    # For some reason Alpine 3.14 does not report the -x flag correctly and thus elasticsearch does not find java
-    sed -i 's/! -x/! -e/g' /usr/share/logstash/bin/logstash.lib.sh && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-input-http && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-output-gelf && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-output-http && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
 #
 # Add and move files
     cd /root/dist/ && \
-    cp update.sh /usr/bin/ && \
+    cp entrypoint.sh /usr/bin/ && \
-    chmod u+x /usr/bin/update.sh && \
+    chmod u+x /usr/bin/entrypoint.sh && \
-    mkdir -p /etc/logstash/conf.d && \
+    mkdir -p /etc/logstash/conf.d /usr/share/logstash/config && \
     cp logstash.conf /etc/logstash/conf.d/ && \
     cp http_input.conf /etc/logstash/conf.d/ && \
     cp http_output.conf /etc/logstash/conf.d/ && \
     cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \
-    cp pipelines_pot.yml /usr/share/logstash/config/pipelines_pot.yml && \
+    cp pipelines_sensor.yml /usr/share/logstash/config/pipelines_sensor.yml && \
     cp tpot_es_template.json /etc/logstash/ && \
 #
 # Setup user, groups and configs
-    addgroup -g 2000 logstash && \
+    groupmod -g 2000 logstash && \
-    adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \
+    usermod -u 2000 logstash && \
-    chown -R logstash:logstash /usr/share/logstash && \
+    chown -R logstash:logstash /etc/listbot \
-    chown -R logstash:logstash /etc/listbot && \
+                               /var/log/logstash/ \
-    chmod 755 /usr/bin/update.sh && \
+			       /var/lib/logstash \
+			       /usr/share/logstash/data && \
+    chmod 755 /usr/bin/entrypoint.sh && \
 #
 # Clean up
-    rm -rf /root/* && \
+    apt-get autoremove -y --purge && \
-    rm -rf /tmp/* && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/dist
-    rm -rf /var/cache/apk/*
 #
 # Healthcheck
 HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
 #
 # Start logstash
-#USER logstash:logstash
+USER logstash:logstash
-CMD update.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution
+CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution

docker/elk/logstash/Dockerfile.new

@@ -1,68 +0,0 @@
-FROM alpine:3.14
-#
-# VARS
-ENV LS_VER=7.15.1
-# Include dist
-ADD dist/ /root/dist/
-#
-# Setup env and apt
-#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
-RUN apk -U --no-cache add \
-             aria2 \
-             bash \
-             bzip2 \
-	     curl \
-             libc6-compat \
-             libzmq \
-             nss && \
-    apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community openjdk16-jre && \
-#
-# Get and install packages
-    mkdir -p /etc/listbot && \
-    cd /etc/listbot && \
-    aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \
-    aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \
-    bunzip2 *.bz2 && \
-    cd /root/dist/ && \
-    mkdir -p /usr/share/logstash/ && \
-    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-linux-x86_64.tar.gz && \
-    tar xvfz logstash-$LS_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
-    rm -rf /usr/share/logstash/jdk && \
-    # For some reason Alpine 3.14 does not report the -x flag correctly and thus elasticsearch does not find java
-    sed -i 's/! -x/! -e/g' /usr/share/logstash/bin/logstash.lib.sh && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-input-http && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-output-gelf && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-output-http && \
-    /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
-#
-# Add and move files
-    cd /root/dist/ && \
-    cp update.sh /usr/bin/ && \
-    chmod u+x /usr/bin/update.sh && \
-    mkdir -p /etc/logstash/conf.d && \
-    cp logstash.conf /etc/logstash/conf.d/ && \
-    cp http.conf /etc/logstash/conf.d/ && \
-    cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \
-    cp tpot_es_template.json /etc/logstash/ && \
-#
-# Setup user, groups and configs
-    addgroup -g 2000 logstash && \
-    adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \
-    chown -R logstash:logstash /usr/share/logstash && \
-    chown -R logstash:logstash /etc/listbot && \
-    chmod 755 /usr/bin/update.sh && \
-#
-# Clean up
-    rm -rf /root/* && \
-    rm -rf /tmp/* && \
-    rm -rf /var/cache/apk/*
-#
-# Healthcheck
-HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
-#
-# Start logstash
-#USER logstash:logstash
-#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution --log.level debug
-#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution
-CMD update.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution

docker/elk/logstash/dist/entrypoint.sh

@@ -46,7 +46,7 @@ if [ "$MY_TPOT_TYPE" == "SENSOR" ];
     echo "Hive username: $MY_HIVE_USERNAME"
     echo "Hive IP: $MY_HIVE_IP"
     echo
-    cp /usr/share/logstash/config/pipelines_pot.yml /usr/share/logstash/config/pipelines.yml
+    cp /usr/share/logstash/config/pipelines_sensor.yml /usr/share/logstash/config/pipelines.yml
     autossh -f -M 0 -4 -l $MY_HIVE_USERNAME -i $MY_SENSOR_PRIVATEKEYFILE -p 64295 -N -L64305:127.0.0.1:64305 $MY_HIVE_IP -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null"
     exit 0
 fi

docker/elk/logstash/dist/pipelines.yml

@@ -1,4 +1,6 @@
 - pipeline.id: logstash
   path.config: "/etc/logstash/conf.d/logstash.conf"
+  pipeline.ecs_compatibility: disabled
 - pipeline.id: http_input
   path.config: "/etc/logstash/conf.d/http_input.conf"
+  pipeline.ecs_compatibility: disabled

docker/elk/logstash/dist/pipelines_sensor.yml

@@ -1,2 +1,3 @@
 - pipeline.id: http_output
   path.config: "/etc/logstash/conf.d/http_output.conf"
+  pipeline.ecs_compatibility: disabled

docker/log4pot/Dockerfile

@@ -2,8 +2,7 @@ FROM ubuntu:20.04
 ENV DEBIAN_FRONTEND noninteractive
 #
 # Install packages
-RUN apt-get update && \
+RUN apt-get update -y && \
-    apt-get update -y && \
     apt-get dist-upgrade -y && \
     apt-get install -y \
              build-essential \