elk 6.x

start adjusting helper scripts for elk 6.x
migrate patterns, dashboards, viz, search, etc.
tweaking

bin/export_kibana-objects.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
-# Export all Kibana objects
+# Export all Kibana objects through Kibana Saved Objects API
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
+myKIBANA="http://127.0.0.1:64296/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
 if ! [ "$myESSTATUS" = "1" ]
   then
@@ -14,10 +15,10 @@ fi
 
 # Set vars
 myDATE=$(date +%Y%m%d%H%M)
-myINDEXCOUNT=$(curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*' | tr '\\' '\n' | grep "scripted" | wc -w)
+myINDEXCOUNT=$(curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern' | jq '.saved_objects[].attributes' | tr '\\' '\n' | grep "scripted" | wc -w)
-myDASHBOARDS=$(curl -s -XGET ''$myES'.kibana/dashboard/_search?filter_path=hits.hits._id&pretty&size=10000'  | jq '.hits.hits[] | {_id}' | jq -r '._id')
+myDASHBOARDS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
-myVISUALIZATIONS=$(curl -s -XGET ''$myES'.kibana/visualization/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
+myVISUALIZATIONS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/visualization?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
-mySEARCHES=$(curl -s -XGET ''$myES'.kibana/search/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
+mySEARCHES=$(curl -s -XGET ''$myKIBANA'api/saved_objects/search?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
 myCOL1=""
 myCOL0=""
 
@@ -30,7 +31,7 @@ trap fuCLEANUP EXIT
 # Export index patterns
 mkdir -p patterns
 echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
-curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*?' | jq '._source' > patterns/index-patterns.json
+curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern' | jq '.saved_objects[] | {attributes}' > patterns/index-patterns.json
 echo
 
 # Export dashboards
@@ -39,7 +40,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"das
 for i in $myDASHBOARDS;
   do
     echo $myCOL1"###### "$i $myCOL0
-    curl -s -XGET ''$myES'.kibana/dashboard/'$i'' | jq '._source' > dashboards/$i.json
+    curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard/'$i'' | jq '. | {attributes}' > dashboards/$i.json
   done;
 echo
 
@@ -49,7 +50,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1
 for i in $myVISUALIZATIONS;
   do
     echo $myCOL1"###### "$i $myCOL0
-    curl -s -XGET ''$myES'.kibana/visualization/'$i'' | jq '._source' > visualizations/$i.json
+    curl -s -XGET ''$myKIBANA'api/saved_objects/visualization/'$i'' | jq '. | {attributes}' > visualizations/$i.json
   done;
 echo
 
@@ -59,7 +60,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searc
 for i in $mySEARCHES;
   do
     echo $myCOL1"###### "$i $myCOL0
-    curl -s -XGET ''$myES'.kibana/search/'$i'' | jq '._source' > searches/$i.json
+    curl -s -XGET ''$myKIBANA'api/saved_objects/search/'$i'' | jq '. | {attributes}' > searches/$i.json
   done;
 echo
 

docker/elk/docker-compose.yml

@@ -7,6 +7,7 @@ services:
 # ELK services
 ## Elasticsearch service
   elasticsearch:
+    build: .
     container_name: elasticsearch
     restart: always
     environment:
@@ -30,8 +31,10 @@ services:
 
 ## Kibana service
   kibana:
+    build: .
     container_name: kibana
     restart: always
+    stop_signal: SIGKILL
     depends_on:
       elasticsearch:
         condition: service_healthy
@@ -41,6 +44,7 @@ services:
 
 ## Logstash service
   logstash:
+    build: .
     container_name: logstash
     restart: always
     depends_on:
@@ -52,9 +56,11 @@ services:
     volumes:
      - /data:/data
      - /var/log:/data/host/log
+     - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
 
 ## Elasticsearch-head service
   head:
+    build: .
     container_name: head
     restart: always
     depends_on:
@@ -63,3 +69,4 @@ services:
     ports:
      - "127.0.0.1:64302:9100"
     image: "dtagdevsec/head:1804"
+    read_only: true

docker/elk/elasticsearch/Dockerfile

@@ -14,8 +14,8 @@ RUN apk -U upgrade && \
 # Get and install packages
     cd /root/dist/ && \
     mkdir -p /usr/share/elasticsearch/ && \
-    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.tar.gz && \
+    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.4.tar.gz && \
-    tar xvfz elasticsearch-5.6.8.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
+    tar xvfz elasticsearch-6.2.4.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
 
 # Add and move files
     cd /root/dist/ && \
@@ -36,4 +36,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
 
 # Start ELK
 USER elasticsearch:elasticsearch
-CMD ["/usr/share/elasticsearch/bin/elasticsearch"]
+CMD export ES_TMPDIR=/tmp && exec /usr/share/elasticsearch/bin/elasticsearch

docker/elk/elasticsearch/Dockerfile.5x

@@ -0,0 +1,39 @@
+FROM alpine
+
+# Include dist
+ADD dist/ /root/dist/
+
+# Setup env and apt
+RUN apk -U upgrade && \
+    apk add bash \
+            curl \
+            openjdk8-jre \
+            procps \
+            wget && \
+
+# Get and install packages
+    cd /root/dist/ && \
+    mkdir -p /usr/share/elasticsearch/ && \
+    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.tar.gz && \
+    tar xvfz elasticsearch-5.6.8.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
+
+# Add and move files
+    cd /root/dist/ && \
+    mkdir -p /usr/share/elasticsearch/config && \
+    cp elasticsearch.yml /usr/share/elasticsearch/config/ && \
+
+# Setup user, groups and configs
+    addgroup -g 2000 elasticsearch && \
+    adduser -S -H -s /bin/bash -u 2000 -D -g 2000 elasticsearch && \
+    chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \
+
+# Clean up
+    apk del --purge wget && \
+    rm -rf /root/*
+
+# Healthcheck
+HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
+
+# Start ELK
+USER elasticsearch:elasticsearch
+CMD ["/usr/share/elasticsearch/bin/elasticsearch"]

docker/elk/elasticsearch/docker-compose.yml.5x

@@ -0,0 +1,30 @@
+# T-Pot (Standard)
+# For docker-compose ...
+version: '2.2'
+
+services:
+
+# ELK services
+## Elasticsearch service
+  elasticsearch:
+    build: .
+    container_name: elasticsearch
+    restart: always
+    environment:
+     - bootstrap.memory_lock=true
+     - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
+    cap_add:
+     - IPC_LOCK
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 2g
+    ports:
+     - "127.0.0.1:64298:9200"
+    image: "dtagdevsec/elasticsearch:1804"
+    volumes:
+     - /data:/data

docker/elk/kibana/Dockerfile

@@ -44,7 +44,7 @@ RUN apk -U upgrade && \
 
 # Setup user, groups and configs
     sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \
-    sed -i 's/#kibana.defaultAppId: "discover"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
+    sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
     sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \
     sed -i 's/#elasticsearch.url: "http:\/\/localhost:9200"/elasticsearch.url: "http:\/\/elasticsearch:9200"/' /usr/share/kibana/config/kibana.yml && \
     /usr/share/kibana/bin/kibana 2>&1 | grep -m 1 "Optimization of bundles" && \

docker/elk/logstash/Dockerfile

@@ -49,4 +49,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
 
 # Start logstash
 #USER logstash:logstash
-CMD update.sh && /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf
+CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf

iso/installer/install.sh

@@ -327,7 +327,7 @@ apt-get autoremove -y 2>&1 | dialog --title "[ Pulling updates ]" $myPROGRESSBOX
 pip install --upgrade pip 2>&1 | dialog --title "[ Installing pip ]" $myPROGRESSBOXCONF
 pip install elasticsearch-curator==5.4.1 2>&1 | dialog --title "[ Installing elasticsearch-curator ]" $myPROGRESSBOXCONF
 pip install yq==2.4.1 2>&1 | dialog --title "[ Installing yq ]" $myPROGRESSBOXCONF
-npm install https://github.com/taskrabbit/elasticsearch-dump#ac9f62a -g 2>&1 | dialog --title "[ Installing elasticsearch-dump ]" $myPROGRESSBOXCONF
+npm install https://github.com/taskrabbit/elasticsearch-dump#9fcc8cc -g 2>&1 | dialog --title "[ Installing elasticsearch-dump ]" $myPROGRESSBOXCONF
 wget https://github.com/bcicen/ctop/releases/download/v0.7/ctop-0.7-linux-amd64 -O ctop 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF
 mv ctop /usr/bin/ 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF
 chmod +x /usr/bin/ctop 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF