From 788a4c4f986bc23f868e50ed86956ea439ab1d30 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Tue, 4 Jan 2022 16:16:27 +0000 Subject: [PATCH] prepare for new attack map feature tweaking, cleanup --- docker/elk/logstash/deploy.sh | 18 +++--- docker/elk/logstash/dist/http_output.conf | 31 ++++++++--- docker/elk/logstash/dist/logstash.conf | 31 ++++++++--- docker/elk/logstash/dist/logstash.sh | 55 ------------------- .../elk/logstash/dist/tpot_es_template.json | 9 +++ docker/elk/logstash/dist/update.sh | 13 ++++- docker/elk/logstash/hive.yml | 2 + 7 files changed, 74 insertions(+), 85 deletions(-) delete mode 100644 docker/elk/logstash/dist/logstash.sh diff --git a/docker/elk/logstash/deploy.sh b/docker/elk/logstash/deploy.sh index 5ce67bca..d8f7ddfb 100755 --- a/docker/elk/logstash/deploy.sh +++ b/docker/elk/logstash/deploy.sh @@ -108,7 +108,6 @@ MY_HIVE_USERNAME="$(hostname)" MY_TPOT_TYPE="POT" MY_LS_ENVCONFIGFILE="/data/elk/logstash/ls_environment" -#echo "$SSHPASS" MY_POT_PUBLICKEYFILE="/data/elk/logstash/$MY_HIVE_USERNAME.pub" MY_POT_PRIVATEKEYFILE="/data/elk/logstash/$MY_HIVE_USERNAME" if ! [ -s "$MY_POT_PRIVATEKEYFILE" ] && ! [ -s "$MY_POT_PUBLICKEYFILE" ]; @@ -121,7 +120,6 @@ if ! [ -s "$MY_POT_PRIVATEKEYFILE" ] && ! [ -s "$MY_POT_PUBLICKEYFILE" ]; mkdir -p /data/elk/logstash ssh-keygen -f "$MY_POT_PRIVATEKEYFILE" -N "" -C "$MY_HIVE_USERNAME" MY_POT_PUBLICKEY="$(cat "$MY_POT_PUBLICKEYFILE")" - echo "$MY_POT_PUBLICKEY" else echo echo "#############################################" @@ -138,10 +136,10 @@ echo "# run /opt/tpot/bin/updateip.sh. #" echo "###########################################################" echo tee $MY_LS_ENVCONFIGFILE << EOF -MY_TPOT_TYPE="$MY_TPOT_TYPE" -MY_POT_PRIVATEKEYFILE="$MY_POT_PRIVATEKEYFILE" -MY_HIVE_USERNAME="$MY_HIVE_USERNAME" -MY_HIVE_IP="$MY_HIVE_IP" +MY_TPOT_TYPE=$MY_TPOT_TYPE +MY_POT_PRIVATEKEYFILE=$MY_POT_PRIVATEKEYFILE +MY_HIVE_USERNAME=$MY_HIVE_USERNAME +MY_HIVE_IP=$MY_HIVE_IP EOF } @@ -152,10 +150,10 @@ echo "#################################" echo "# Ship T-Pot Logs to T-Pot Hive #" echo "#################################" echo -echo "If ..." -echo "1. You already have a T-Pot Hive installation running and" -echo "2. This T-Pot installation is running the type \"Pot\"" -echo "... the script will guide you to deploy this T-Pot's logs to the Hive." +echo "If you already have a T-Pot Hive installation running and" +echo "this T-Pot installation is running the type \"Pot\" the" +echo "script will automagically setup this T-Pot to ship and" +echo "prepare the Hive to receive logs from this T-Pot." echo echo echo "###################################" diff --git a/docker/elk/logstash/dist/http_output.conf b/docker/elk/logstash/dist/http_output.conf index 07bf0c08..c727824b 100644 --- a/docker/elk/logstash/dist/http_output.conf +++ b/docker/elk/logstash/dist/http_output.conf @@ -664,6 +664,14 @@ filter { if "_grokparsefailure" in [tags] { drop {} } if "_jsonparsefailure" in [tags] { drop {} } +# Add T-Pot hostname and external IP + mutate { + add_field => { + "t-pot_ip_ext" => "${MY_EXTIP}" + "t-pot_ip_int" => "${MY_INTIP}" + "t-pot_hostname" => "${MY_HOSTNAME}" + } + } # Add geo coordinates / ASN info / IP rep. if [src_ip] { @@ -684,6 +692,20 @@ if "_jsonparsefailure" in [tags] { drop {} } dictionary_path => "/etc/listbot/iprep.yaml" } } + if [t-pot_ip_ext] { + geoip { + cache_size => 10000 + source => "t-pot_ip_ext" + target => "geoip_ext" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.8-java/vendor/GeoLite2-City.mmdb" + } + geoip { + cache_size => 10000 + source => "t-pot_ip_ext" + target => "geoip_ext" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.8-java/vendor/GeoLite2-ASN.mmdb" + } + } # In some rare conditions dest_port, src_port, status are indexed as string, forcing integer for now if [dest_port] { @@ -712,15 +734,6 @@ if "_jsonparsefailure" in [tags] { drop {} } } } -# Add T-Pot hostname and external IP - mutate { - add_field => { - "t-pot_ip_ext" => "${MY_EXTIP}" - "t-pot_ip_int" => "${MY_INTIP}" - "t-pot_hostname" => "${MY_HOSTNAME}" - } - } - } # Output section diff --git a/docker/elk/logstash/dist/logstash.conf b/docker/elk/logstash/dist/logstash.conf index 4d72e1f6..63fd324e 100644 --- a/docker/elk/logstash/dist/logstash.conf +++ b/docker/elk/logstash/dist/logstash.conf @@ -664,6 +664,14 @@ filter { if "_grokparsefailure" in [tags] { drop {} } if "_jsonparsefailure" in [tags] { drop {} } +# Add T-Pot hostname and external IP + mutate { + add_field => { + "t-pot_ip_ext" => "${MY_EXTIP}" + "t-pot_ip_int" => "${MY_INTIP}" + "t-pot_hostname" => "${MY_HOSTNAME}" + } + } # Add geo coordinates / ASN info / IP rep. if [src_ip] { @@ -684,6 +692,20 @@ if "_jsonparsefailure" in [tags] { drop {} } dictionary_path => "/etc/listbot/iprep.yaml" } } + if [t-pot_ip_ext] { + geoip { + cache_size => 10000 + source => "t-pot_ip_ext" + target => "geoip_ext" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.8-java/vendor/GeoLite2-City.mmdb" + } + geoip { + cache_size => 10000 + source => "t-pot_ip_ext" + target => "geoip_ext" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.8-java/vendor/GeoLite2-ASN.mmdb" + } + } # In some rare conditions dest_port, src_port, status are indexed as string, forcing integer for now if [dest_port] { @@ -712,15 +734,6 @@ if "_jsonparsefailure" in [tags] { drop {} } } } -# Add T-Pot hostname and external IP - mutate { - add_field => { - "t-pot_ip_ext" => "${MY_EXTIP}" - "t-pot_ip_int" => "${MY_INTIP}" - "t-pot_hostname" => "${MY_HOSTNAME}" - } - } - } # Output section diff --git a/docker/elk/logstash/dist/logstash.sh b/docker/elk/logstash/dist/logstash.sh deleted file mode 100644 index 3fb06fb9..00000000 --- a/docker/elk/logstash/dist/logstash.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Let's ensure normal operation on exit or if interrupted ... -function fuCLEANUP { - exit 0 -} -trap fuCLEANUP EXIT - -# Check internet availability -function fuCHECKINET () { -mySITES=$1 -error=0 -for i in $mySITES; - do - curl --connect-timeout 5 -Is $i 2>&1 > /dev/null - if [ $? -ne 0 ]; - then - let error+=1 - fi; - done; - echo $error -} - -# Check for connectivity and download latest translation maps -myCHECK=$(fuCHECKINET "listbot.sicherheitstacho.eu") -if [ "$myCHECK" == "0" ]; - then - echo "Connection to Listbot looks good, now downloading latest translation maps." - cd /etc/listbot - aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \ - aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \ - bunzip2 -f *.bz2 - cd / - else - echo "Cannot reach Listbot, starting Logstash without latest translation maps." -fi - -exit - - -# notizen - -MY_TPOT_TYPE Standard = SINGLE, Distributed = POT - -Wenn POT -autossh -f -M 0 -4 -l tpot01 -i /data/elk/logstash/tpot01 -p 64295 -N -L64305:127.0.0.1:64305 172.20.254.194 -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/http_output.conf --config.reload.automatic --java-execution - - -Wenn SINGLE -exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution - -Umgebungsvariable holen aus /data/elk/logstash -m besten über das ELK Environment file, damit es keine probleme gibt - diff --git a/docker/elk/logstash/dist/tpot_es_template.json b/docker/elk/logstash/dist/tpot_es_template.json index 2e10d80e..0ee8dd62 100644 --- a/docker/elk/logstash/dist/tpot_es_template.json +++ b/docker/elk/logstash/dist/tpot_es_template.json @@ -43,6 +43,15 @@ "latitude" : { "type" : "half_float" }, "longitude" : { "type" : "half_float" } } + }, + "geoip_ext" : { + "dynamic": true, + "properties" : { + "ip": { "type": "ip" }, + "location" : { "type" : "geo_point" }, + "latitude" : { "type" : "half_float" }, + "longitude" : { "type" : "half_float" } + } } } } diff --git a/docker/elk/logstash/dist/update.sh b/docker/elk/logstash/dist/update.sh index cf5137c3..0ec6f57f 100644 --- a/docker/elk/logstash/dist/update.sh +++ b/docker/elk/logstash/dist/update.sh @@ -47,7 +47,7 @@ if [ "$MY_TPOT_TYPE" == "POT" ]; echo "Hive IP: $MY_HIVE_IP" echo cp /usr/share/logstash/config/pipelines_pot.yml /usr/share/logstash/config/pipelines.yml - autossh -f -M 0 -v -4 -l $MY_HIVE_USERNAME -i $MY_POT_PRIVATEKEYFILE -p 64295 -N -L64305:127.0.0.1:64305 $MY_HIVE_IP -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" + autossh -f -M 0 -4 -l $MY_HIVE_USERNAME -i $MY_POT_PRIVATEKEYFILE -p 64295 -N -L64305:127.0.0.1:64305 $MY_HIVE_IP -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" exit 0 fi @@ -60,7 +60,7 @@ echo "Checking if empty." curl -s -XGET http://elasticsearch:9200/_template/logstash echo echo "Putting default template." -curl -s -XPUT "http://elasticsearch:9200/_template/logstash" -H 'Content-Type: application/json' -d' +curl -XPUT "http://elasticsearch:9200/_template/logstash" -H 'Content-Type: application/json' -d' { "index_patterns" : "logstash-*", "version" : 60001, @@ -106,6 +106,15 @@ curl -s -XPUT "http://elasticsearch:9200/_template/logstash" -H 'Content-Type: a "latitude" : { "type" : "half_float" }, "longitude" : { "type" : "half_float" } } + }, + "geoip_ext" : { + "dynamic": true, + "properties" : { + "ip": { "type": "ip" }, + "location" : { "type" : "geo_point" }, + "latitude" : { "type" : "half_float" }, + "longitude" : { "type" : "half_float" } + } } } } diff --git a/docker/elk/logstash/hive.yml b/docker/elk/logstash/hive.yml index b0ab788f..32011ec3 100644 --- a/docker/elk/logstash/hive.yml +++ b/docker/elk/logstash/hive.yml @@ -70,6 +70,8 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment + ports: + - "127.0.0.1:64305:80" image: "dtagdevsec/logstash:2006" volumes: - /data:/data