bump elk stack to 6.6.1

docker/elk/docker-compose.yml

@@ -24,7 +24,7 @@ services:
     mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
-    image: "dtagdevsec/elasticsearch:1811"
+    image: "dtagdevsec/elasticsearch:1903"
     volumes:
      - /data:/data
 
@@ -39,7 +39,7 @@ services:
         condition: service_healthy
     ports:
      - "127.0.0.1:64296:5601"
-    image: "dtagdevsec/kibana:1811"
+    image: "dtagdevsec/kibana:1903"
 
 ## Logstash service
   logstash:
@@ -51,7 +51,7 @@ services:
         condition: service_healthy
     env_file:
      - /opt/tpot/etc/compose/elk_environment
-    image: "dtagdevsec/logstash:1811"
+    image: "dtagdevsec/logstash:1903"
     volumes:
      - /data:/data
      - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
@@ -66,5 +66,5 @@ services:
         condition: service_healthy
     ports:
      - "127.0.0.1:64302:9100"
-    image: "dtagdevsec/head:1811"
+    image: "dtagdevsec/head:1903"
     read_only: true

docker/elk/elasticsearch/Dockerfile

@@ -13,8 +13,8 @@ RUN apk -U add \
 # Get and install packages
     cd /root/dist/ && \
     mkdir -p /usr/share/elasticsearch/ && \
-    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.5.4.tar.gz && \
+    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.1.tar.gz && \
-    tar xvfz elasticsearch-6.5.4.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
+    tar xvfz elasticsearch-6.6.1.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
 
 # Add and move files
     cd /root/dist/ && \

docker/elk/elasticsearch/docker-compose.yml

@@ -24,6 +24,6 @@ services:
     mem_limit: 2g
     ports:
      - "127.0.0.1:64298:9200"
-    image: "dtagdevsec/elasticsearch:1811"
+    image: "dtagdevsec/elasticsearch:1903"
     volumes:
      - /data:/data

docker/elk/kibana/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.8
+FROM node:10.15.1-alpine
 
 # Include dist
 ADD dist/ /root/dist/
@@ -6,14 +6,13 @@ ADD dist/ /root/dist/
 # Setup env and apt
 RUN apk -U add \
             curl \
-            nodejs \
             wget && \
 
 # Get and install packages
     cd /root/dist/ && \
     mkdir -p /usr/share/kibana/ && \
-    wget https://artifacts.elastic.co/downloads/kibana/kibana-6.5.4-linux-x86_64.tar.gz && \
+    wget https://artifacts.elastic.co/downloads/kibana/kibana-6.6.1-linux-x86_64.tar.gz && \
-    tar xvfz kibana-6.5.4-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \
+    tar xvfz kibana-6.6.1-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \
 
 # Kibana's bundled node does not work in alpine
     rm /usr/share/kibana/node/bin/node && \
@@ -26,31 +25,22 @@ RUN apk -U add \
     cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \
     cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \
     cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \
-    cp create_kibana_index.js /usr/share/kibana/src/core_plugins/elasticsearch/lib/ && \
-
-# Setup plugins, rebuild bundle
-    #cd /usr/share/kibana/plugins && \
-    #wget https://github.com/dlumbrer/kbn_radar/releases/download/Kibana-6.X/kbn_radar.tar.gz && \
-    #wget https://github.com/dlumbrer/kbn_network/releases/download/6.0.X-1/network_vis.tar.gz && \
-    #tar xvfz kbn_radar.tar.gz && \
-    #tar xvfz network_vis.tar.gz && \
-    #rm *.tar.gz && \
-    rm -rf /usr/share/kibana/optimize/bundles/* && \
 
 # Setup user, groups and configs
     sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \
     sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
     sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \
-    sed -i 's/#elasticsearch.url: "http:\/\/localhost:9200"/elasticsearch.url: "http:\/\/elasticsearch:9200"/' /usr/share/kibana/config/kibana.yml && \
+    sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \
     sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \
-    sed -i "s/#005571/#e20074/g" /usr/share/kibana/src/ui/public/chrome/directives/global_nav/global_nav.less && \
+    sed -i "s/#005571/#e20074/g" /usr/share/kibana/src/legacy/core_plugins/kibana/public/index.css && \
-    sed -i "s/globalColorBlue/globalColorMagenta/g" /usr/share/kibana/src/ui/public/chrome/directives/global_nav/global_nav_link/global_nav_link.less && \
+    sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/src/legacy/core_plugins/kibana/public/index.css && \
-    echo "@globalColorMagenta: #9E0051;" >> /usr/share/kibana/src/ui/public/styles/variables/colors.less && \
+    sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/src/legacy/core_plugins/kibana/public/index.css && \
     echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ 
     echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
     echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \
+    rm -rf /usr/share/kibana/optimize/bundles/* && \
     /usr/share/kibana/bin/kibana --optimize && \
     addgroup -g 2000 kibana && \
     adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \

docker/elk/kibana/dist/create_kibana_index.js

@@ -1,38 +0,0 @@
-'use strict';
-
-var _setup_error = require('./setup_error');
-
-var _setup_error2 = _interopRequireDefault(_setup_error);
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-module.exports = function (server, mappings) {
-  var _server$plugins$elast = server.plugins.elasticsearch.getCluster('admin');
-
-  const callWithInternalUser = _server$plugins$elast.callWithInternalUser;
-
-  const index = server.config().get('kibana.index');
-
-  function handleError(message) {
-    return function (err) {
-      throw new _setup_error2.default(server, message, err);
-    };
-  }
-
-  return callWithInternalUser('indices.create', {
-    index: index,
-    body: {
-      settings: {
-        number_of_shards: 1,
-        number_of_replicas: 0,
-        'index.mapper.dynamic': false
-      },
-      mappings
-    }
-  }).catch(handleError('Unable to create Kibana index "<%= kibana.index %>"')).then(function () {
-    return callWithInternalUser('cluster.health', {
-      waitForStatus: 'yellow',
-      index: index
-    }).catch(handleError('Waiting for Kibana index "<%= kibana.index %>" to come online failed.'));
-  });
-};

docker/elk/kibana/docker-compose.yml

@@ -12,4 +12,4 @@ services:
 #        condition: service_healthy
     ports:
      - "127.0.0.1:64296:5601"
-    image: "dtagdevsec/kibana:1811"
+    image: "dtagdevsec/kibana:1903"

docker/elk/logstash/Dockerfile

@@ -17,9 +17,9 @@ RUN apk -U add \
     git clone --depth=1 https://github.com/dtag-dev-sec/listbot /etc/listbot && \
     cd /root/dist/ && \
     mkdir -p /usr/share/logstash/ && \
-    wget https://artifacts.elastic.co/downloads/logstash/logstash-6.5.4.tar.gz && \
+    wget https://artifacts.elastic.co/downloads/logstash/logstash-6.6.1.tar.gz && \
     wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz && \
-    tar xvfz logstash-6.5.4.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
+    tar xvfz logstash-6.6.1.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
     /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
     /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
     tar xvfz GeoLite2-ASN.tar.gz --strip-components=1 -C /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/ && \
@@ -30,7 +30,7 @@ RUN apk -U add \
     chmod u+x /usr/bin/update.sh && \
     mkdir -p /etc/logstash/conf.d && \
     cp logstash.conf /etc/logstash/conf.d/ && \
-    cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.4-java/lib/logstash/outputs/elasticsearch/ && \
+    cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.3.2-java/lib/logstash/outputs/elasticsearch/ && \
 
 # Setup user, groups and configs
     addgroup -g 2000 logstash && \
@@ -50,4 +50,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
 
 # Start logstash
 #USER logstash:logstash
-CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --java-execution
+CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution

docker/elk/logstash/dist/logstash.conf

@@ -131,6 +131,7 @@ filter {
       field => "[alert][signature_id]"
       destination => "[alert][cve_id]"
       dictionary_path => "/etc/listbot/cve.yaml"
+      fallback => "-"
     }
   }
 

docker/elk/logstash/docker-compose.yml

@@ -12,7 +12,7 @@ services:
 #        condition: service_healthy
     env_file:
      - /opt/tpot/etc/compose/elk_environment
-    image: "dtagdevsec/logstash:1811"
+    image: "dtagdevsec/logstash:1903"
     volumes:
      - /data:/data
      - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf