bump elastic stack to 8.0.0

bin/backup_es_folders.sh

@@ -1,12 +1,21 @@
 #!/bin/bash
 # Run as root only.
 myWHOAMI=$(whoami)
-if [ "$myWHOAMI" != "root" ]
+if [ "$myWHOAMI" != "root" ];
   then
     echo "Need to run as root ..."
     exit
 fi
 
+if [ "$1" == "" ] || [ "$1" != "all" ] && [ "$1" != "base" ];
+  then
+    echo "Usage: backup_es_folders [all, base]"
+    echo "       all  = backup all ES folder"
+    echo "       base = backup only Kibana index".
+    echo
+    exit
+fi
+
 # Backup all ES relevant folders
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
@@ -25,7 +34,7 @@ myCOUNT=1
 myDATE=$(date +%Y%m%d%H%M)
 myELKPATH="/data/elk/data"
 myKIBANAINDEXNAME=$(curl -s -XGET ''$myES'_cat/indices/.kibana' | awk '{ print $4 }')
-myKIBANAINDEXPATH=$myELKPATH/nodes/0/indices/$myKIBANAINDEXNAME
+myKIBANAINDEXPATH=$myELKPATH/indices/$myKIBANAINDEXNAME
 
 # Let's ensure normal operation on exit or if interrupted ...
 function fuCLEANUP {
@@ -42,5 +51,11 @@ sleep 2
 
 # Backup DB in 2 flavors
 echo "### Now backing up Elasticsearch folders ..."
-tar cvfz "elkall_"$myDATE".tgz" $myELKPATH
+if [ "$1" == "all" ];
-tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH
+  then
+    tar cvfz "elkall_"$myDATE".tgz" $myELKPATH
+elif [ "$1" == "base" ];
+  then
+    tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH
+fi
+

docker/elk/docker-compose.yml

@@ -10,7 +10,7 @@ services:
     restart: always
     environment:
      - bootstrap.memory_lock=true
-#     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
+     - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
      - ES_TMPDIR=/tmp
     cap_add:
      - IPC_LOCK
@@ -21,7 +21,7 @@ services:
       nofile:
         soft: 65536
         hard: 65536
-#    mem_limit: 4g
+    mem_limit: 4g
     ports:
      - "127.0.0.1:64298:9200"
     image: "dtagdevsec/elasticsearch:2203"

docker/elk/elasticsearch/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 #
 # VARS
-ENV ES_VER=7.17.0
+ENV ES_VER=8.0.0
 #
 # Include dist
 ADD dist/ /root/dist/
@@ -22,7 +22,7 @@ RUN apt-get update -y && \
     dpkg -i elasticsearch-$ES_VER-$ES_ARCH.deb && \
 #
 # Add and move files
-    rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \
+#    rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \
     mkdir -p /usr/share/elasticsearch/config && \
     cp elasticsearch.yml /etc/elasticsearch/ && \
 #

docker/elk/elasticsearch/dist/elasticsearch.yml

@@ -2,6 +2,8 @@ cluster.name: tpotcluster
 node.name: "tpotcluster-node-01"
 xpack.ml.enabled: false
 xpack.security.enabled: false
+xpack.security.transport.ssl.enabled: false
+xpack.security.http.ssl.enabled: false
 path:
     logs: /data/elk/log
     data: /data/elk/data

docker/elk/kibana/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 #
 # VARS
-ENV KB_VER=7.17.0
+ENV KB_VER=8.0.0
 # Include dist
 ADD dist/ /root/dist/
 #

docker/elk/logstash/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 #
 # VARS
-ENV LS_VER=7.17.0
+ENV LS_VER=8.0.0
 # Include dist
 ADD dist/ /root/dist/
 #
@@ -37,11 +37,12 @@ RUN apt-get update -y && \
     chmod u+x /usr/bin/entrypoint.sh && \
     mkdir -p /etc/logstash/conf.d /usr/share/logstash/config && \
     cp logstash.conf /etc/logstash/conf.d/ && \
+    cp logstash.yml /etc/logstash/conf.d/ && \
     cp http_input.conf /etc/logstash/conf.d/ && \
     cp http_output.conf /etc/logstash/conf.d/ && \
     cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \
     cp pipelines_sensor.yml /usr/share/logstash/config/pipelines_sensor.yml && \
-    cp tpot_es_template.json /etc/logstash/ && \
+    cp tpot-template.json /etc/logstash/ && \
 #
 # Setup user, groups and configs
     groupmod -g 2000 logstash && \
@@ -61,4 +62,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
 #
 # Start logstash
 USER logstash:logstash
-CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution
+CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic

docker/elk/logstash/dist/http_input.conf

@@ -13,7 +13,7 @@ output {
     hosts => ["elasticsearch:9200"]
     # With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
     index => "logstash-%{+YYYY.MM.dd}"
-    template => "/etc/logstash/tpot_es_template.json"
+    template => "/etc/logstash/tpot-template.json"
     template_overwrite => "true"
   }
 

docker/elk/logstash/dist/logstash.conf

@@ -725,9 +725,8 @@ output {
     hosts => ["elasticsearch:9200"]
     # With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
     index => "logstash-%{+YYYY.MM.dd}"
-    template => "/etc/logstash/tpot_es_template.json"
+    template => "/etc/logstash/tpot-template.json"
     template_overwrite => "true"
-    #document_type => "doc"
   }
 
   #if [type] == "Suricata" {

docker/elk/logstash/dist/logstash.yml

@@ -0,0 +1 @@
+path.config: "/usr/sahre/logstash/config/pipelines.yml"

docker/elk/logstash/dist/tpot-template.json

@@ -0,0 +1,94 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "tpot"
+        },
+        "mapping": {
+          "total_fields": {
+            "limit": "2000"
+          }
+        },
+        "refresh_interval": "5s",
+        "number_of_shards": "1",
+        "number_of_replicas": "0",
+        "query": {
+          "default_field": "*"
+        }
+      }
+    },
+    "mappings": {
+      "dynamic": "true",
+      "dynamic_date_formats": [
+        "strict_date_optional_time",
+        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
+      ],
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "path_match": "message",
+            "match_mapping_type": "string",
+            "mapping": {
+              "norms": false,
+              "type": "text"
+            }
+          }
+        },
+        {
+          "string_fields": {
+            "match": "*",
+            "match_mapping_type": "string",
+            "mapping": {
+              "fields": {
+                "keyword": {
+                  "ignore_above": 256,
+                  "type": "keyword"
+                }
+              },
+              "norms": false,
+              "type": "text"
+            }
+          }
+        }
+      ],
+      "date_detection": true,
+      "numeric_detection": false,
+      "properties": {
+        "geoip": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "latitude": {
+              "type": "half_float"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "longitude": {
+              "type": "half_float"
+            }
+          }
+        },
+        "geoip_ext": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "latitude": {
+              "type": "half_float"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "longitude": {
+              "type": "half_float"
+            }
+          }
+        }
+      }
+    },
+    "aliases": {}
+  }
+}

docker/elk/logstash/dist/tpot_es_template.json

@@ -1,59 +0,0 @@
-{
-  "index_patterns" : "logstash-*",
-  "version" : 60001,
-  "settings" : {
-    "index.lifecycle.name": "tpot",
-    "index.refresh_interval" : "5s",
-    "number_of_shards" : 1,
-    "index.number_of_replicas" : "0",
-    "index.mapping.total_fields.limit" : "2000",
-    "index.query": {
-      "default_field": "*"
-     }
-  },
-  "mappings" : {
-    "dynamic_templates" : [ {
-      "message_field" : {
-        "path_match" : "message",
-        "match_mapping_type" : "string",
-        "mapping" : {
-          "type" : "text",
-          "norms" : false
-        }
-      }
-    }, {
-      "string_fields" : {
-        "match" : "*",
-        "match_mapping_type" : "string",
-        "mapping" : {
-          "type" : "text", "norms" : false,
-          "fields" : {
-            "keyword" : { "type": "keyword", "ignore_above": 256 }
-          }
-        }
-      }
-    } ],
-    "properties" : {
-      "@timestamp": { "type": "date"},
-      "@version": { "type": "keyword"},
-      "geoip"  : {
-        "dynamic": true,
-        "properties" : {
-          "ip": { "type": "ip" },
-          "location" : { "type" : "geo_point" },
-          "latitude" : { "type" : "half_float" },
-          "longitude" : { "type" : "half_float" }
-        }
-      },
-      "geoip_ext"  : {
-        "dynamic": true,
-        "properties" : {
-          "ip": { "type": "ip" },
-          "location" : { "type" : "geo_point" },
-          "latitude" : { "type" : "half_float" },
-          "longitude" : { "type" : "half_float" }
-        }
-      }
-    }
-  }
-}

docker/elk/logstash/docker-compose.yml

@@ -21,3 +21,4 @@ services:
      - /data:/data
 #     - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
 #     - /root/tpotce/docker/elk/logstash/dist/http.conf:/etc/logstash/conf.d/http.conf
+#     - /root/tpotce/docker/elk/logstash/dist/logstash.yml:/etc/logstash/conf.d/logstash.yml