diff --git a/installer/bin/backup_es_folders.sh b/installer/bin/backup_es_folders.sh index c3c19de0..08155332 100755 --- a/installer/bin/backup_es_folders.sh +++ b/installer/bin/backup_es_folders.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Backup all ES relevant folders # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/check.sh b/installer/bin/check.sh deleted file mode 100755 index 0ea3423c..00000000 --- a/installer/bin/check.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -######################################################## -# T-Pot # -# Check container and services script # -# # -# v16.10.0 by mo, DTAG, 2016-05-12 # -######################################################## -if [ -a /var/run/check.lock ]; - then - echo "Lock exists. Exiting now." - exit -fi - -myIMAGES=$(cat /etc/tpot/images.conf) - -touch /var/run/check.lock - -myUPTIME=$(awk '{print int($1/60)}' /proc/uptime) -for i in $myIMAGES - do - if [ "$i" != "ui-for-docker" ] && [ "$i" != "netdata" ] && [ "$i" != "spiderfoot" ]; - then - myCIDSTATUS=$(docker exec $i supervisorctl status) - if [ $? -ne 0 ]; - then - myCIDSTATUS=1 - else - myCIDSTATUS=$(echo $myCIDSTATUS | egrep -c "(STOPPED|FATAL)") - fi - if [ $myUPTIME -gt 4 ] && [ $myCIDSTATUS -gt 0 ]; - then - echo "Restarting "$i"." - systemctl stop $i - sleep 5 - systemctl start $i - fi - fi -done - -rm /var/run/check.lock diff --git a/installer/bin/clean.sh b/installer/bin/clean.sh index f3906114..ae60d124 100755 --- a/installer/bin/clean.sh +++ b/installer/bin/clean.sh @@ -1,11 +1,5 @@ #!/bin/bash - -######################################################## -# T-Pot # -# Container Data Cleaner # -# # -# v16.10.0 by mo, DTAG, 2016-05-28 # -######################################################## +# T-Pot Container Data Cleaner # Set persistence myPERSISTENCE=$1 diff --git a/installer/bin/dcres.sh b/installer/bin/dcres.sh deleted file mode 100755 index 26e1f884..00000000 --- a/installer/bin/dcres.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -######################################################## -# T-Pot # -# Container and services restart script # -# # -# v16.10.0 by mo, DTAG, 2016-05-12 # -######################################################## -myCOUNT=1 - -while true -do - if ! [ -a /var/run/check.lock ]; - then break - fi - sleep 0.1 - if [ "$myCOUNT" = "1" ]; - then - echo -n "Waiting for services " - else echo -n . - fi - if [ "$myCOUNT" = "6000" ]; - then - echo - echo "Overriding check.lock" - rm /var/run/check.lock - break - fi - myCOUNT=$[$myCOUNT +1] -done - -myIMAGES=$(cat /etc/tpot/images.conf) - -touch /var/run/check.lock - -myUPTIME=$(awk '{print int($1/60)}' /proc/uptime) -if [ $myUPTIME -gt 4 ]; - then - for i in $myIMAGES - do - systemctl stop $i - done - echo "### Waiting 10 seconds before restarting docker ..." - sleep 10 - iptables -w -F - systemctl restart docker - while true - do - docker info > /dev/null - if [ $? -ne 0 ]; - then - echo Docker daemon is still starting. - else - echo Docker daemon is now available. - break - fi - sleep 0.1 - done - echo "### Docker is now up and running again." - echo "### Removing obsolete container data ..." - docker rm -v $(docker ps -aq) - echo "### Removing obsolete image data ..." - docker rmi $(docker images | grep "" | awk '{print $3}') - echo "### Starting T-Pot services ..." - for i in $myIMAGES - do - systemctl start $i - done - sleep 5 - else - echo "### T-Pot needs to be up and running for at least 5 minutes." -fi - -rm /var/run/check.lock - -/etc/rc.local diff --git a/installer/bin/dps.sh b/installer/bin/dps.sh index 6607b170..2f3910b1 100755 --- a/installer/bin/dps.sh +++ b/installer/bin/dps.sh @@ -1,5 +1,5 @@ #/bin/bash - +# Show current status of all running containers # Let's ensure normal operation on exit or if interrupted ... function fuCLEANUP { stty sane @@ -7,8 +7,6 @@ function fuCLEANUP { trap fuCLEANUP EXIT stty -echo -icanon time 0 min 0 -#myIMAGES=$(cat /etc/tpot/images.conf) -#myIMAGES=$(/usr/bin/docker ps -a -f name=$i --format "table {{.Names}}" | grep -v NAMES) myIMAGES=$(cat /etc/tpot/tpot.yml | grep container_name | cut -d: -f2) while true do diff --git a/installer/bin/dump_es.sh b/installer/bin/dump_es.sh index 74e95043..d496a98e 100755 --- a/installer/bin/dump_es.sh +++ b/installer/bin/dump_es.sh @@ -1,4 +1,5 @@ #/bin/bash +# Dump all ES data # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/export_kibana-objects.sh b/installer/bin/export_kibana-objects.sh index ac630dd3..a48b9011 100755 --- a/installer/bin/export_kibana-objects.sh +++ b/installer/bin/export_kibana-objects.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Export all Kibana objects # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/import_kibana-objects.sh b/installer/bin/import_kibana-objects.sh index c291ce63..2ae37e6a 100755 --- a/installer/bin/import_kibana-objects.sh +++ b/installer/bin/import_kibana-objects.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Import Kibana objects # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/myip.sh b/installer/bin/myip.sh index 10580971..86a9114e 100755 --- a/installer/bin/myip.sh +++ b/installer/bin/myip.sh @@ -34,8 +34,6 @@ httplist=( whatismyip.akamai.com ) - - # function to shuffle the global array "array" shuffle() { local i tmp size max rand @@ -48,7 +46,6 @@ shuffle() { done } - # if we have dig and a list of dns methods, try that first if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then eval array=( \"\${dnslist[@]}\" ) @@ -64,9 +61,7 @@ if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then done fi - # if we haven't succeeded with DNS, try HTTP - if [ ${#httplist[*]} == 0 ]; then echo "No hosts in httplist array!" >&2 exit 1 diff --git a/installer/bin/restore_es.sh b/installer/bin/restore_es.sh index e1dc01d3..506a5c8c 100755 --- a/installer/bin/restore_es.sh +++ b/installer/bin/restore_es.sh @@ -1,4 +1,5 @@ #/bin/bash +# Restore folder based ES backup # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/status.sh b/installer/bin/status.sh index a37900c1..c4f27eb9 100755 --- a/installer/bin/status.sh +++ b/installer/bin/status.sh @@ -1,16 +1,10 @@ #!/bin/bash - -######################################################## -# T-Pot # -# Container and services status script # -# # -# v16.10.0 by mo, DTAG, 2016-05-12 # -######################################################## +# Show status of SupervisorD within running containers myCOUNT=1 if [[ $1 == "" ]] then - myIMAGES=$(cat /etc/tpot/images.conf) + myIMAGES=$(cat /etc/tpot/tpot.yml | grep container_name | cut -d: -f2) else myIMAGES=$1 fi diff --git a/installer/bin/update-images.sh b/installer/bin/update-images.sh deleted file mode 100755 index 0ee431eb..00000000 --- a/installer/bin/update-images.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -########################################################## -# T-Pot # -# Only start the containers found in /etc/systemd/system # -# # -# v17.06 by mo, DTAG, 2017-03-13 # -########################################################## - -# Make sure not to interrupt a check -while true -do - if ! [ -a /var/run/check.lock ]; - then break - fi - sleep 0.1 - if [ "$myCOUNT" = "1" ]; - then - echo -n "Waiting for services " - else echo -n . - fi - if [ "$myCOUNT" = "6000" ]; - then - echo - echo "Overriding check.lock" - rm /var/run/check.lock - break - fi - myCOUNT=$[$myCOUNT +1] -done - -# We do not want to get interrupted by a check -touch /var/run/check.lock - -# Stop T-Pot services and disable all T-Pot services -echo "### Stopping T-Pot services and cleaning up." -for i in $(cat /etc/tpot/imgcfg/all_images.conf); - do - systemctl stop $i - sleep 2 - systemctl disable $i; - rm /etc/systemd/system/$i.service -done - -# Restarting docker services and optionally clear local repository -echo "### Stopping docker services ..." -systemctl stop docker -sleep 1 -# If option "hard" clear the whole repository -if [ "$1" = "hard" ]; - then - echo "### Clearing local docker repository." - rm -rf /var/lib/docker - sleep 1 -fi -echo "### Starting docker services ..." -systemctl start docker -sleep 1 - -# Enable only T-Pot systemd scripts from images.conf and pull the images -for i in $(cat /etc/tpot/images.conf); - do - echo - echo "### Now pulling "$i - docker pull dtagdevsec/$i:1706; - cp /etc/tpot/systemd/$i.service /etc/systemd/system/ - systemctl enable $i; -done - -# Announce reboot -echo -echo "### Rebooting." - -# Allow checks to resume -rm /var/run/check.lock - -# Reboot -reboot diff --git a/installer/etc/rc.local b/installer/etc/rc.local index 8e5ade55..7b64eb94 100755 --- a/installer/etc/rc.local +++ b/installer/etc/rc.local @@ -16,6 +16,3 @@ MY_HOSTNAME=$HOSTNAME EOF echo $myLOCALIP > /data/elk/logstash/mylocal.ip chown tpot:tpot /data/ews/conf/ews.ip -if [ -f /var/run/check.lock ]; - then rm /var/run/check.lock -fi diff --git a/installer/install.sh b/installer/install.sh index 472146ce..24717ac9 100755 --- a/installer/install.sh +++ b/installer/install.sh @@ -1,10 +1,5 @@ #!/bin/bash -######################################################## -# T-Pot post install script # -# Ubuntu server 16.04.0, x64 # -# # -# v17.06 by mo, DTAG, 2017-03-22 # -######################################################## +# T-Pot post install script # Set TERM, DIALOGRC export TERM=linux @@ -32,6 +27,8 @@ fuRANDOMWORD () { } # Let's wait a few seconds to avoid interference with service messages +sleep 3 +tput civis dialog --no-ok --no-cancel --backtitle "$myBACKTITLE" --title "[ Wait to avoid interference with service messages ]" --pause "" 6 80 7 # Let's setup the proxy for env @@ -104,6 +101,7 @@ rm -rf /usr/share/nginx/html/index.html 2>&1 | dialog --title "[ Removing NGINX # Let's ask user for install flavor # Install types are TPOT, HP, INDUSTRIAL, ALL +tput cnorm myFLAVOR=$(dialog --no-cancel --backtitle "$myBACKTITLE" --title "[ Choose your edition ]" --no-tags --menu \ "\nRequired: 4GB RAM, 64GB disk\nRecommended: 8GB RAM, 128GB SSD" 14 60 4 \ "TPOT" "Standard Honeypots, Suricata & ELK" \ @@ -198,6 +196,7 @@ while [ "$myPASS1" != "$myPASS2" ] && [ "$mySECURE" == "0" ] htpasswd -b -c /etc/nginx/nginxpasswd "$myUSER" "$myPASS1" 2>&1 | dialog --title "[ Setting up user and password ]" $myPROGRESSBOXCONF; # Let's generate a SSL self-signed certificate without interaction (browsers will see it invalid anyway) +tput civis mkdir -p /etc/nginx/ssl 2>&1 | dialog --title "[ Generating a self-signed-certificate for NGINX ]" $myPROGRESSBOXCONF; openssl req \ -nodes \ @@ -372,15 +371,15 @@ esac # Let's load docker images myIMAGESCOUNT=$(cat /root/tpot/etc/tpot/tpot.yml | grep container_name | cut -d: -f2 | wc -l) j=0 -for name in $(cat /root/tpot/etc/tpot/tpot.yml | grep container_name | cut -d: -f2) +for name in $(cat /root/tpot/etc/tpot/tpot.yml | grep image | cut -d'"' -f2) do dialog --title "[ Downloading docker images, please be patient ]" --backtitle "$myBACKTITLE" \ - --gauge "\n Now downloading: dtagdevsec/$name:1706\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <&1>/dev/null + docker pull $name 2>&1>/dev/null let j+=1 dialog --title "[ Downloading docker images, please be patient ]" --backtitle "$myBACKTITLE" \ - --gauge "\n Now downloading: dtagdevsec/$name:1706\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <&1>/dev/null <:/api delete --filters resource= && alerta --endpoint-url http://:/api send -e IP -r -E Production -s ok -S T-Pot -t \$(cat /data/elk/logstash/mylocal.ip) --status open +#*/5 * * * * root alerta --endpoint-url http://:/api delete --filters resource= && alerta --endpoint-url http://:/api send -e IP -r -E Production -s ok -S T-Pot -t \$(cat /data/elk/logstash/mylocal.ip) --status open # Check if updated images are available and download them -27 1 * * * root /usr/bin/docker-compose -f /etc/tpot/tpot.yml pull - -# Restart docker service and containers -#27 3 * * * root dcres.sh +27 1 * * * root /usr/bin/docker-compose -f /etc/tpot/tpot.yml pull # Delete elastic indices older than 90 days (kibana index is omitted by default) -#27 4 * * * root docker exec elk bash -c '/usr/local/bin/curator --host 127.0.0.1 delete indices --older-than 90 --time-unit days --timestring \%Y.\%m.\%d' - -# Update IP and erase check.lock if it exists -27 5 * * * root /etc/rc.local +#27 4 * * * root docker exec elk bash -c '/usr/local/bin/curator --host 127.0.0.1 delete indices --older-than 90 --time-unit days --timestring \%Y.\%m.\%d' # Daily reboot -27 23 * * * root reboot +27 3 * * * root reboot # Check for updated packages every sunday, upgrade and reboot -27 16 * * 0 root apt-get autoclean -y && apt-get autoremove -y && apt-get update -y && apt-get upgrade -y && sleep 10 && reboot +27 16 * * 0 root apt-get autoclean -y && apt-get autoremove -y && apt-get update -y && apt-get upgrade -y && sleep 10 && reboot EOF # Let's create some files and folders