elk 7.x dev test

docker/elk/docker-compose.yml

@@ -24,7 +24,7 @@ services:
     mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
-    image: "dtagdevsec/elasticsearch:1903"
+    image: "dtagdevsec/elasticsearch:2006"
     volumes:
      - /data:/data
 
@@ -39,7 +39,7 @@ services:
         condition: service_healthy
     ports:
      - "127.0.0.1:64296:5601"
-    image: "dtagdevsec/kibana:1903"
+    image: "dtagdevsec/kibana:2006"
 
 ## Logstash service
   logstash:
@@ -51,7 +51,7 @@ services:
         condition: service_healthy
     env_file:
      - /opt/tpot/etc/compose/elk_environment
-    image: "dtagdevsec/logstash:1903"
+    image: "dtagdevsec/logstash:2006"
     volumes:
      - /data:/data
      - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
@@ -66,5 +66,5 @@ services:
         condition: service_healthy
     ports:
      - "127.0.0.1:64302:9100"
-    image: "dtagdevsec/head:1903"
+    image: "dtagdevsec/head:2006"
     read_only: true

docker/elk/elasticsearch/Dockerfile

@@ -1,5 +1,8 @@
 FROM alpine
 #
+# VARS
+ENV ES_VER=7.5.2 \
+    JAVA_HOME=/usr/lib/jvm/java-11-openjdk
 # Include dist
 ADD dist/ /root/dist/
 #
@@ -10,13 +13,13 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
              bash \
              curl \
              nss \
-             openjdk8-jre && \
+             openjdk11-jre && \
 #
 # Get and install packages
     cd /root/dist/ && \
     mkdir -p /usr/share/elasticsearch/ && \
-    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VER-linux-x86_64.tar.gz && \
-    tar xvfz elasticsearch-6.8.6.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
+    tar xvfz elasticsearch-$ES_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
 #
 # Add and move files
     cd /root/dist/ && \
@@ -40,5 +43,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
 #
 # Start ELK
 USER elasticsearch:elasticsearch
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
 CMD ["/usr/share/elasticsearch/bin/elasticsearch"]

docker/elk/elasticsearch/Dockerfile.new

@@ -0,0 +1,46 @@
+FROM alpine
+#
+# VARS
+ENV ES_VER=7.5.2 \
+    JAVA_HOME=/usr/lib/jvm/java-11-openjdk
+# Include dist
+ADD dist/ /root/dist/
+#
+# Setup env and apt
+RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
+    apk -U --no-cache add \
+             aria2 \
+             bash \
+             curl \
+             nss \
+             openjdk11-jre && \
+#
+# Get and install packages
+    cd /root/dist/ && \
+    mkdir -p /usr/share/elasticsearch/ && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VER-linux-x86_64.tar.gz && \
+    tar xvfz elasticsearch-$ES_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
+#
+# Add and move files
+    cd /root/dist/ && \
+    mkdir -p /usr/share/elasticsearch/config && \
+    cp elasticsearch.yml /usr/share/elasticsearch/config/ && \
+#
+# Setup user, groups and configs
+    addgroup -g 2000 elasticsearch && \
+    adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \
+    chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \
+    rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \
+#
+# Clean up
+    apk del --purge aria2 && \
+    rm -rf /root/* && \
+    rm -rf /tmp/* && \
+    rm -rf /var/cache/apk/*
+#
+# Healthcheck
+HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
+#
+# Start ELK
+USER elasticsearch:elasticsearch
+CMD ["/usr/share/elasticsearch/bin/elasticsearch"]

docker/elk/elasticsearch/Dockerfile.old

@@ -0,0 +1,44 @@
+FROM alpine
+#
+# Include dist
+ADD dist/ /root/dist/
+#
+# Setup env and apt
+RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
+    apk -U --no-cache add \
+             aria2 \
+             bash \
+             curl \
+             nss \
+             openjdk8-jre && \
+#
+# Get and install packages
+    cd /root/dist/ && \
+    mkdir -p /usr/share/elasticsearch/ && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && \
+    tar xvfz elasticsearch-6.8.6.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
+#
+# Add and move files
+    cd /root/dist/ && \
+    mkdir -p /usr/share/elasticsearch/config && \
+    cp elasticsearch.yml /usr/share/elasticsearch/config/ && \
+#
+# Setup user, groups and configs
+    addgroup -g 2000 elasticsearch && \
+    adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \
+    chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \
+    rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \
+#
+# Clean up
+    apk del --purge aria2 && \
+    rm -rf /root/* && \
+    rm -rf /tmp/* && \
+    rm -rf /var/cache/apk/*
+#
+# Healthcheck
+HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
+#
+# Start ELK
+USER elasticsearch:elasticsearch
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
+CMD ["/usr/share/elasticsearch/bin/elasticsearch"]

docker/elk/elasticsearch/dist/elasticsearch.yml

@@ -1,11 +1,14 @@
 cluster.name: tpotcluster
 node.name: "tpotcluster-node-01"
 xpack.ml.enabled: false
+xpack.security.enabled: false
 path:
     logs: /data/elk/log
     data: /data/elk/data
 http.host: 0.0.0.0
 http.cors.enabled: true
 http.cors.allow-origin: "*"
+cluster.initial_master_nodes:
+- "tpotcluster-node-01"
 discovery.zen.ping.unicast.hosts:
-   - localhost 
+- localhost 

docker/elk/elasticsearch/docker-compose.yml

@@ -24,6 +24,6 @@ services:
     mem_limit: 2g
     ports:
      - "127.0.0.1:64298:9200"
-    image: "dtagdevsec/elasticsearch:1903"
+    image: "dtagdevsec/elasticsearch:2006"
     volumes:
      - /data:/data

docker/elk/head/docker-compose.yml

@@ -12,5 +12,5 @@ services:
     #        condition: service_healthy
     ports:
      - "127.0.0.1:64302:9100"
-    image: "dtagdevsec/head:1903"
+    image: "dtagdevsec/head:2006"
     read_only: true

docker/elk/kibana/Dockerfile

@@ -1,5 +1,8 @@
 FROM node:10.15.2-alpine
 #
+# VARS
+ENV KB_VER=7.5.2
+# 
 # Include dist
 ADD dist/ /root/dist/
 #
@@ -12,20 +15,20 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
 # Get and install packages
     cd /root/dist/ && \
     mkdir -p /usr/share/kibana/ && \
-    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-6.8.6-linux-x86_64.tar.gz && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-$KB_VER-linux-x86_64.tar.gz && \
-    tar xvfz kibana-6.8.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \
+    tar xvfz kibana-$KB_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \
 #
 # Kibana's bundled node does not work in alpine
     rm /usr/share/kibana/node/bin/node && \
-    ln -s /usr/bin/node /usr/share/kibana/node/bin/node && \
+    ln -s /usr/local/bin/node /usr/share/kibana/node/bin/node && \
 #
 # Add and move files
     cd /root/dist/ && \
-    cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \
+#    cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \
-    cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \
+#    cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \
-    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \
+#    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \
-    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \
+#    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \
-    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \
+#    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \
 #
 # Setup user, groups and configs
     sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \
@@ -33,17 +36,19 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \
     sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \
     sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \
-    sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
+#    sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
-    sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
+#    sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
-    sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
+#    sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
     echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ 
     echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     rm -rf /usr/share/kibana/optimize/bundles/* && \
-    /usr/share/kibana/bin/kibana --optimize && \
+    /usr/share/kibana/bin/kibana --optimize --allow-root && \
     addgroup -g 2000 kibana && \
     adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \
     chown -R kibana:kibana /usr/share/kibana/ && \

docker/elk/kibana/Dockerfile.old

@@ -0,0 +1,63 @@
+FROM node:10.15.2-alpine
+#
+# Include dist
+ADD dist/ /root/dist/
+#
+# Setup env and apt
+RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
+    apk -U --no-cache add \
+            aria2 \
+            curl && \
+#
+# Get and install packages
+    cd /root/dist/ && \
+    mkdir -p /usr/share/kibana/ && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-6.8.6-linux-x86_64.tar.gz && \
+    tar xvfz kibana-6.8.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \
+#
+# Kibana's bundled node does not work in alpine
+    rm /usr/share/kibana/node/bin/node && \
+    ln -s /usr/bin/node /usr/share/kibana/node/bin/node && \
+#
+# Add and move files
+    cd /root/dist/ && \
+    cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \
+    cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \
+    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \
+    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \
+    cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \
+#
+# Setup user, groups and configs
+    sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \
+    sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
+    sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \
+    sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \
+    sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \
+    sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
+    sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
+    sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \
+    echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ 
+    echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    rm -rf /usr/share/kibana/optimize/bundles/* && \
+    /usr/share/kibana/bin/kibana --optimize && \
+    addgroup -g 2000 kibana && \
+    adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \
+    chown -R kibana:kibana /usr/share/kibana/ && \
+#
+# Clean up
+    apk del --purge aria2 && \
+    rm -rf /root/* && \
+    rm -rf /tmp/* && \
+    rm -rf /var/cache/apk/*
+#
+# Healthcheck
+HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:5601'
+#
+# Start kibana
+STOPSIGNAL SIGKILL
+USER kibana:kibana
+CMD ["/usr/share/kibana/bin/kibana"]

docker/elk/kibana/docker-compose.yml

@@ -12,4 +12,4 @@ services:
 #        condition: service_healthy
     ports:
      - "127.0.0.1:64296:5601"
-    image: "dtagdevsec/kibana:1903"
+    image: "dtagdevsec/kibana:2006"

docker/elk/logstash/Dockerfile

@@ -1,5 +1,7 @@
 FROM alpine
 #
+# VARS
+ENV LS_VER=7.5.2
 # Include dist
 ADD dist/ /root/dist/
 #
@@ -13,7 +15,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
              libc6-compat \
              libzmq \
              nss \
-             openjdk8-jre && \
+             openjdk11-jre && \
 #
 # Get and install packages
     mkdir -p /etc/listbot && \
@@ -23,8 +25,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     bunzip2 *.bz2 && \
     cd /root/dist/ && \
     mkdir -p /usr/share/logstash/ && \
-    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER.tar.gz && \
-    tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
+    tar xvfz logstash-$LS_VER.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
     /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
     /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
 #
@@ -34,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     chmod u+x /usr/bin/update.sh && \
     mkdir -p /etc/logstash/conf.d && \
     cp logstash.conf /etc/logstash/conf.d/ && \
-    cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \
+    cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.2.3-java/lib/logstash/outputs/elasticsearch/ && \
 #
 # Setup user, groups and configs
     addgroup -g 2000 logstash && \

docker/elk/logstash/Dockerfile.old

@@ -0,0 +1,56 @@
+FROM alpine
+#
+# Include dist
+ADD dist/ /root/dist/
+#
+# Setup env and apt
+RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
+    apk -U --no-cache add \
+             aria2 \
+             bash \
+             bzip2 \
+	     curl \
+             libc6-compat \
+             libzmq \
+             nss \
+             openjdk8-jre && \
+#
+# Get and install packages
+    mkdir -p /etc/listbot && \
+    cd /etc/listbot && \
+    aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/cve.yaml.bz2 && \
+    aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/iprep.yaml.bz2 && \
+    bunzip2 *.bz2 && \
+    cd /root/dist/ && \
+    mkdir -p /usr/share/logstash/ && \
+    aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \
+    tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
+    /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
+    /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
+#
+# Add and move files
+    cd /root/dist/ && \
+    cp update.sh /usr/bin/ && \
+    chmod u+x /usr/bin/update.sh && \
+    mkdir -p /etc/logstash/conf.d && \
+    cp logstash.conf /etc/logstash/conf.d/ && \
+    cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \
+#
+# Setup user, groups and configs
+    addgroup -g 2000 logstash && \
+    adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \
+    chown -R logstash:logstash /usr/share/logstash && \
+    chown -R logstash:logstash /etc/listbot && \
+    chmod 755 /usr/bin/update.sh && \
+#
+# Clean up
+    rm -rf /root/* && \
+    rm -rf /tmp/* && \
+    rm -rf /var/cache/apk/*
+#
+# Healthcheck
+HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
+#
+# Start logstash
+#USER logstash:logstash
+CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution

docker/elk/logstash/dist/elasticsearch-template-es5x.json

@@ -1,53 +0,0 @@
-{
-  "template" : "logstash-*",
-  "version" : 50001,
-  "settings" : {
-    "index.refresh_interval" : "5s",
-    "index.number_of_shards" : "1",
-    "index.number_of_replicas" : "0",
-        "mapping" : {
-          "total_fields" : {
-            "limit" : "2000"
-          }
-        }
-  },
-  "mappings" : {
-    "_default_" : {
-      "_all" : {"enabled" : true, "norms" : false},
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text",
-            "norms" : false
-          }
-        }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text", "norms" : false,
-            "fields" : {
-              "keyword" : { "type": "keyword", "ignore_above": 256 }
-            }
-          }
-        }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date", "include_in_all": false },
-        "@version": { "type": "keyword", "include_in_all": false },
-        "geoip"  : {
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip" },
-            "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "half_float" },
-            "longitude" : { "type" : "half_float" }
-          }
-        }
-      }
-    }
-  }
-}

docker/elk/logstash/dist/elasticsearch-template-es6x.json

@@ -1,48 +0,0 @@
-{
-  "template" : "logstash-*",
-  "version" : 60001,
-  "settings" : {
-    "index.refresh_interval" : "5s",
-    "index.number_of_shards" : "1",
-    "index.number_of_replicas" : "0",
-    "index.mapping.total_fields.limit": "2000"
-  },
-  "mappings" : {
-    "_default_" : {
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text",
-            "norms" : false
-          }
-        }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text", "norms" : false,
-            "fields" : {
-              "keyword" : { "type": "keyword", "ignore_above": 256 }
-            }
-          }
-        }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date"},
-        "@version": { "type": "keyword"},
-        "geoip"  : {
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip" },
-            "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "half_float" },
-            "longitude" : { "type" : "half_float" }
-          }
-        }
-      }
-    }
-  }
-}

docker/elk/logstash/dist/elasticsearch-template-es7x.json

@@ -0,0 +1,46 @@
+{
+  "index_patterns" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s",
+    "number_of_shards": 1,
+    "index.number_of_replicas" : "0",
+    "index.mapping.total_fields.limit": "2000"
+  },
+  "mappings" : {
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
+        }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
+          }
+        }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
+        }
+      }
+    }
+  }
+}

docker/elk/logstash/dist/logstash.conf

@@ -413,12 +413,12 @@ if "_grokparsefailure" in [tags] { drop {} }
     geoip {
       cache_size => 10000
       source => "src_ip"
-      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb"
+      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-City.mmdb"
     }
     geoip {
       cache_size => 10000
       source => "src_ip"
-      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-ASN.mmdb"
+      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-ASN.mmdb"
     }
     translate {
       refresh_interval => 86400

docker/elk/logstash/docker-compose.yml

@@ -12,7 +12,7 @@ services:
 #        condition: service_healthy
     env_file:
      - /opt/tpot/etc/compose/elk_environment
-    image: "dtagdevsec/logstash:1903"
+    image: "dtagdevsec/logstash:2006"
     volumes:
      - /data:/data
      - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf

etc/compose/nextgen.yml

@@ -496,7 +496,7 @@ services:
     mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
-    image: "dtagdevsec/elasticsearch:1903"
+    image: "dtagdevsec/elasticsearch:2006"
     volumes:
      - /data:/data
 
@@ -509,7 +509,7 @@ services:
         condition: service_healthy
     ports:
      - "127.0.0.1:64296:5601"
-    image: "dtagdevsec/kibana:1903"
+    image: "dtagdevsec/kibana:2006"
 
 ## Logstash service
   logstash:
@@ -520,7 +520,7 @@ services:
         condition: service_healthy
     env_file:
      - /opt/tpot/etc/compose/elk_environment
-    image: "dtagdevsec/logstash:1903"
+    image: "dtagdevsec/logstash:2006"
     volumes:
      - /data:/data
 
@@ -533,7 +533,7 @@ services:
         condition: service_healthy
     ports:
      - "127.0.0.1:64302:9100"
-    image: "dtagdevsec/head:1903"
+    image: "dtagdevsec/head:2006"
     read_only: true
 
 # Ewsposter service