diff --git a/bin/backup_es_folders.sh b/bin/backup_es_folders.sh index 88a279be..3d15261b 100755 --- a/bin/backup_es_folders.sh +++ b/bin/backup_es_folders.sh @@ -1,12 +1,21 @@ #!/bin/bash # Run as root only. myWHOAMI=$(whoami) -if [ "$myWHOAMI" != "root" ] +if [ "$myWHOAMI" != "root" ]; then echo "Need to run as root ..." exit fi +if [ "$1" == "" ] || [ "$1" != "all" ] && [ "$1" != "base" ]; + then + echo "Usage: backup_es_folders [all, base]" + echo " all = backup all ES folder" + echo " base = backup only Kibana index". + echo + exit +fi + # Backup all ES relevant folders # Make sure ES is available myES="http://127.0.0.1:64298/" @@ -25,7 +34,7 @@ myCOUNT=1 myDATE=$(date +%Y%m%d%H%M) myELKPATH="/data/elk/data" myKIBANAINDEXNAME=$(curl -s -XGET ''$myES'_cat/indices/.kibana' | awk '{ print $4 }') -myKIBANAINDEXPATH=$myELKPATH/nodes/0/indices/$myKIBANAINDEXNAME +myKIBANAINDEXPATH=$myELKPATH/indices/$myKIBANAINDEXNAME # Let's ensure normal operation on exit or if interrupted ... function fuCLEANUP { @@ -42,5 +51,11 @@ sleep 2 # Backup DB in 2 flavors echo "### Now backing up Elasticsearch folders ..." -tar cvfz "elkall_"$myDATE".tgz" $myELKPATH -tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH +if [ "$1" == "all" ]; + then + tar cvfz "elkall_"$myDATE".tgz" $myELKPATH +elif [ "$1" == "base" ]; + then + tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH +fi + diff --git a/docker/elk/docker-compose.yml b/docker/elk/docker-compose.yml index 75d7702c..c5d333b3 100644 --- a/docker/elk/docker-compose.yml +++ b/docker/elk/docker-compose.yml @@ -10,7 +10,7 @@ services: restart: always environment: - bootstrap.memory_lock=true -# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m + - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK @@ -21,7 +21,7 @@ services: nofile: soft: 65536 hard: 65536 -# mem_limit: 4g + mem_limit: 4g ports: - "127.0.0.1:64298:9200" image: "dtagdevsec/elasticsearch:2203" diff --git a/docker/elk/elasticsearch/Dockerfile b/docker/elk/elasticsearch/Dockerfile index 94a651e7..e39b7abd 100644 --- a/docker/elk/elasticsearch/Dockerfile +++ b/docker/elk/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 # # VARS -ENV ES_VER=7.17.0 +ENV ES_VER=8.0.0 # # Include dist ADD dist/ /root/dist/ @@ -22,7 +22,7 @@ RUN apt-get update -y && \ dpkg -i elasticsearch-$ES_VER-$ES_ARCH.deb && \ # # Add and move files - rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ +# rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ mkdir -p /usr/share/elasticsearch/config && \ cp elasticsearch.yml /etc/elasticsearch/ && \ # diff --git a/docker/elk/elasticsearch/dist/elasticsearch.yml b/docker/elk/elasticsearch/dist/elasticsearch.yml index a5ccd137..35d79569 100644 --- a/docker/elk/elasticsearch/dist/elasticsearch.yml +++ b/docker/elk/elasticsearch/dist/elasticsearch.yml @@ -2,6 +2,8 @@ cluster.name: tpotcluster node.name: "tpotcluster-node-01" xpack.ml.enabled: false xpack.security.enabled: false +xpack.security.transport.ssl.enabled: false +xpack.security.http.ssl.enabled: false path: logs: /data/elk/log data: /data/elk/data diff --git a/docker/elk/kibana/Dockerfile b/docker/elk/kibana/Dockerfile index cdb03165..0ebb59d6 100644 --- a/docker/elk/kibana/Dockerfile +++ b/docker/elk/kibana/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 # # VARS -ENV KB_VER=7.17.0 +ENV KB_VER=8.0.0 # Include dist ADD dist/ /root/dist/ # diff --git a/docker/elk/logstash/Dockerfile b/docker/elk/logstash/Dockerfile index 441b6b85..78d1d22c 100644 --- a/docker/elk/logstash/Dockerfile +++ b/docker/elk/logstash/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 # # VARS -ENV LS_VER=7.17.0 +ENV LS_VER=8.0.0 # Include dist ADD dist/ /root/dist/ # @@ -37,11 +37,12 @@ RUN apt-get update -y && \ chmod u+x /usr/bin/entrypoint.sh && \ mkdir -p /etc/logstash/conf.d /usr/share/logstash/config && \ cp logstash.conf /etc/logstash/conf.d/ && \ + cp logstash.yml /etc/logstash/conf.d/ && \ cp http_input.conf /etc/logstash/conf.d/ && \ cp http_output.conf /etc/logstash/conf.d/ && \ cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \ cp pipelines_sensor.yml /usr/share/logstash/config/pipelines_sensor.yml && \ - cp tpot_es_template.json /etc/logstash/ && \ + cp tpot-template.json /etc/logstash/ && \ # # Setup user, groups and configs groupmod -g 2000 logstash && \ @@ -61,4 +62,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600' # # Start logstash USER logstash:logstash -CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution +CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic diff --git a/docker/elk/logstash/dist/http_input.conf b/docker/elk/logstash/dist/http_input.conf index d1fea672..b51823d8 100644 --- a/docker/elk/logstash/dist/http_input.conf +++ b/docker/elk/logstash/dist/http_input.conf @@ -13,7 +13,7 @@ output { hosts => ["elasticsearch:9200"] # With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana. index => "logstash-%{+YYYY.MM.dd}" - template => "/etc/logstash/tpot_es_template.json" + template => "/etc/logstash/tpot-template.json" template_overwrite => "true" } diff --git a/docker/elk/logstash/dist/logstash.conf b/docker/elk/logstash/dist/logstash.conf index 15b40756..8e0322ab 100644 --- a/docker/elk/logstash/dist/logstash.conf +++ b/docker/elk/logstash/dist/logstash.conf @@ -725,9 +725,8 @@ output { hosts => ["elasticsearch:9200"] # With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana. index => "logstash-%{+YYYY.MM.dd}" - template => "/etc/logstash/tpot_es_template.json" + template => "/etc/logstash/tpot-template.json" template_overwrite => "true" - #document_type => "doc" } #if [type] == "Suricata" { diff --git a/docker/elk/logstash/dist/logstash.yml b/docker/elk/logstash/dist/logstash.yml new file mode 100644 index 00000000..b133627b --- /dev/null +++ b/docker/elk/logstash/dist/logstash.yml @@ -0,0 +1 @@ +path.config: "/usr/sahre/logstash/config/pipelines.yml" diff --git a/docker/elk/logstash/dist/tpot-template.json b/docker/elk/logstash/dist/tpot-template.json new file mode 100644 index 00000000..9b952bc5 --- /dev/null +++ b/docker/elk/logstash/dist/tpot-template.json @@ -0,0 +1,94 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "tpot" + }, + "mapping": { + "total_fields": { + "limit": "2000" + } + }, + "refresh_interval": "5s", + "number_of_shards": "1", + "number_of_replicas": "0", + "query": { + "default_field": "*" + } + } + }, + "mappings": { + "dynamic": "true", + "dynamic_date_formats": [ + "strict_date_optional_time", + "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z" + ], + "dynamic_templates": [ + { + "message_field": { + "path_match": "message", + "match_mapping_type": "string", + "mapping": { + "norms": false, + "type": "text" + } + } + }, + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "norms": false, + "type": "text" + } + } + } + ], + "date_detection": true, + "numeric_detection": false, + "properties": { + "geoip": { + "properties": { + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + } + } + }, + "geoip_ext": { + "properties": { + "ip": { + "type": "ip" + }, + "latitude": { + "type": "half_float" + }, + "location": { + "type": "geo_point" + }, + "longitude": { + "type": "half_float" + } + } + } + } + }, + "aliases": {} + } +} diff --git a/docker/elk/logstash/dist/tpot_es_template.json b/docker/elk/logstash/dist/tpot_es_template.json deleted file mode 100644 index 24ecf624..00000000 --- a/docker/elk/logstash/dist/tpot_es_template.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "index_patterns" : "logstash-*", - "version" : 60001, - "settings" : { - "index.lifecycle.name": "tpot", - "index.refresh_interval" : "5s", - "number_of_shards" : 1, - "index.number_of_replicas" : "0", - "index.mapping.total_fields.limit" : "2000", - "index.query": { - "default_field": "*" - } - }, - "mappings" : { - "dynamic_templates" : [ { - "message_field" : { - "path_match" : "message", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", - "norms" : false - } - } - }, { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", "norms" : false, - "fields" : { - "keyword" : { "type": "keyword", "ignore_above": 256 } - } - } - } - } ], - "properties" : { - "@timestamp": { "type": "date"}, - "@version": { "type": "keyword"}, - "geoip" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - }, - "geoip_ext" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - } - } - } -} diff --git a/docker/elk/logstash/docker-compose.yml b/docker/elk/logstash/docker-compose.yml index d480d051..01eb454e 100644 --- a/docker/elk/logstash/docker-compose.yml +++ b/docker/elk/logstash/docker-compose.yml @@ -21,3 +21,4 @@ services: - /data:/data # - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf # - /root/tpotce/docker/elk/logstash/dist/http.conf:/etc/logstash/conf.d/http.conf +# - /root/tpotce/docker/elk/logstash/dist/logstash.yml:/etc/logstash/conf.d/logstash.yml diff --git a/etc/objects/elkbase.tgz b/etc/objects/elkbase.tgz index 65ba153b..e7c7e2f1 100644 Binary files a/etc/objects/elkbase.tgz and b/etc/objects/elkbase.tgz differ