Alex-Devera/tpotce
1
0
Fork 0
mirror of https://github.com/telekom-security/tpotce.git synced 2025-05-11 02:46:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions1

bump elastic stack to 8.0.0

Browse source
This commit is contained in:
t3chn0m4g3 2022-02-16 02:55:20 +00:00
parent 17a1cbdf40
commit 7ee263e8e8
13 changed files with 128 additions and 74 deletions

23
bin/backup_es_folders.sh
View file

@ -1,12 +1,21 @@
#!/bin/bash
# Run as root only.
myWHOAMI=$(whoami)
if [ "$myWHOAMI" != "root" ]
if [ "$myWHOAMI" != "root" ];
then
echo "Need to run as root ..."
exit
fi
if [ "$1" == "" ] || [ "$1" != "all" ] && [ "$1" != "base" ];
then
echo "Usage: backup_es_folders [all, base]"
echo " all = backup all ES folder"
echo " base = backup only Kibana index".
echo
exit
fi
# Backup all ES relevant folders
# Make sure ES is available
myES="http://127.0.0.1:64298/"
@ -25,7 +34,7 @@ myCOUNT=1
myDATE=$(date +%Y%m%d%H%M)
myELKPATH="/data/elk/data"
myKIBANAINDEXNAME=$(curl -s -XGET ''$myES'_cat/indices/.kibana' | awk '{ print $4 }')
myKIBANAINDEXPATH=$myELKPATH/nodes/0/indices/$myKIBANAINDEXNAME
myKIBANAINDEXPATH=$myELKPATH/indices/$myKIBANAINDEXNAME
# Let's ensure normal operation on exit or if interrupted ...
function fuCLEANUP {
@ -42,5 +51,11 @@ sleep 2
# Backup DB in 2 flavors
echo "### Now backing up Elasticsearch folders ..."
tar cvfz "elkall_"$myDATE".tgz" $myELKPATH
tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH
if [ "$1" == "all" ];
then
tar cvfz "elkall_"$myDATE".tgz" $myELKPATH
elif [ "$1" == "base" ];
then
tar cvfz "elkbase_"$myDATE".tgz" $myKIBANAINDEXPATH
fi

4
docker/elk/docker-compose.yml
View file

@ -10,7 +10,7 @@ services:
restart: always
environment:
- bootstrap.memory_lock=true
# - ES_JAVA_OPTS=-Xms2048m -Xmx2048m
- ES_JAVA_OPTS=-Xms2048m -Xmx2048m
- ES_TMPDIR=/tmp
cap_add:
- IPC_LOCK
@ -21,7 +21,7 @@ services:
nofile:
soft: 65536
hard: 65536
# mem_limit: 4g
mem_limit: 4g
ports:
- "127.0.0.1:64298:9200"
image: "dtagdevsec/elasticsearch:2203"

4
docker/elk/elasticsearch/Dockerfile
View file

@ -1,7 +1,7 @@
FROM ubuntu:20.04
#
# VARS
ENV ES_VER=7.17.0
ENV ES_VER=8.0.0
#
# Include dist
ADD dist/ /root/dist/
@ -22,7 +22,7 @@ RUN apt-get update -y && \
dpkg -i elasticsearch-$ES_VER-$ES_ARCH.deb && \
#
# Add and move files
rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \
# rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \
mkdir -p /usr/share/elasticsearch/config && \
cp elasticsearch.yml /etc/elasticsearch/ && \
#

2
docker/elk/elasticsearch/dist/elasticsearch.yml vendored
View file

@ -2,6 +2,8 @@ cluster.name: tpotcluster
node.name: "tpotcluster-node-01"
xpack.ml.enabled: false
xpack.security.enabled: false
xpack.security.transport.ssl.enabled: false
xpack.security.http.ssl.enabled: false
path:
logs: /data/elk/log
data: /data/elk/data

2
docker/elk/kibana/Dockerfile
View file

@ -1,7 +1,7 @@
FROM ubuntu:20.04
#
# VARS
ENV KB_VER=7.17.0
ENV KB_VER=8.0.0
# Include dist
ADD dist/ /root/dist/
#

7
docker/elk/logstash/Dockerfile
View file

@ -1,7 +1,7 @@
FROM ubuntu:20.04
#
# VARS
ENV LS_VER=7.17.0
ENV LS_VER=8.0.0
# Include dist
ADD dist/ /root/dist/
#
@ -37,11 +37,12 @@ RUN apt-get update -y && \
chmod u+x /usr/bin/entrypoint.sh && \
mkdir -p /etc/logstash/conf.d /usr/share/logstash/config && \
cp logstash.conf /etc/logstash/conf.d/ && \
cp logstash.yml /etc/logstash/conf.d/ && \
cp http_input.conf /etc/logstash/conf.d/ && \
cp http_output.conf /etc/logstash/conf.d/ && \
cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \
cp pipelines_sensor.yml /usr/share/logstash/config/pipelines_sensor.yml && \
cp tpot_es_template.json /etc/logstash/ && \
cp tpot-template.json /etc/logstash/ && \
#
# Setup user, groups and configs
groupmod -g 2000 logstash && \
@ -61,4 +62,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
#
# Start logstash
USER logstash:logstash
CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution
CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic

2
docker/elk/logstash/dist/http_input.conf vendored
View file

@ -13,7 +13,7 @@ output {
hosts => ["elasticsearch:9200"]
# With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
index => "logstash-%{+YYYY.MM.dd}"
template => "/etc/logstash/tpot_es_template.json"
template => "/etc/logstash/tpot-template.json"
template_overwrite => "true"
}

3
docker/elk/logstash/dist/logstash.conf vendored
View file

@ -725,9 +725,8 @@ output {
hosts => ["elasticsearch:9200"]
# With templates now being legacy we need to set the daily index with its template manually. Otherwise a new index might be created with differents settings configured through Kibana.
index => "logstash-%{+YYYY.MM.dd}"
template => "/etc/logstash/tpot_es_template.json"
template => "/etc/logstash/tpot-template.json"
template_overwrite => "true"
#document_type => "doc"
}
#if [type] == "Suricata" {

1
docker/elk/logstash/dist/logstash.yml vendored Normal file
View file

@ -0,0 +1 @@
path.config: "/usr/sahre/logstash/config/pipelines.yml"

94
docker/elk/logstash/dist/tpot-template.json vendored Normal file
View file

@ -0,0 +1,94 @@
{
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "tpot"
},
"mapping": {
"total_fields": {
"limit": "2000"
}
},
"refresh_interval": "5s",
"number_of_shards": "1",
"number_of_replicas": "0",
"query": {
"default_field": "*"
}
}
},
"mappings": {
"dynamic": "true",
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"norms": false,
"type": "text"
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"norms": false,
"type": "text"
}
}
}
],
"date_detection": true,
"numeric_detection": false,
"properties": {
"geoip": {
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
},
"geoip_ext": {
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
}
}
},
"aliases": {}
}
}

59
docker/elk/logstash/dist/tpot_es_template.json vendored
View file

@ -1,59 +0,0 @@
{
"index_patterns" : "logstash-*",
"version" : 60001,
"settings" : {
"index.lifecycle.name": "tpot",
"index.refresh_interval" : "5s",
"number_of_shards" : 1,
"index.number_of_replicas" : "0",
"index.mapping.total_fields.limit" : "2000",
"index.query": {
"default_field": "*"
}
},
"mappings" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
},
"geoip_ext" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}

1
docker/elk/logstash/docker-compose.yml
View file

@ -21,3 +21,4 @@ services:
- /data:/data
# - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
# - /root/tpotce/docker/elk/logstash/dist/http.conf:/etc/logstash/conf.d/http.conf
# - /root/tpotce/docker/elk/logstash/dist/logstash.yml:/etc/logstash/conf.d/logstash.yml

BIN
etc/objects/elkbase.tgz
View file

Binary file not shown.