From 3de02ee7b02d3040f6e190351e0eef3f84bc99b9 Mon Sep 17 00:00:00 2001 From: Marco Ochse Date: Mon, 1 May 2017 19:03:27 +0000 Subject: [PATCH] tweaking for docker-compose get rid of self-check scripts, docker-compose takes care of that now use tpot.yml config for tpot scripts wipe crontab clean of legacy scripts check.lock no longer needed (rc.local) adjust installer (invisible cursor, get image info from tpot.yml, some tweaking) --- installer/bin/backup_es_folders.sh | 1 + installer/bin/check.sh | 41 -------------- installer/bin/clean.sh | 8 +-- installer/bin/dcres.sh | 76 ------------------------- installer/bin/dps.sh | 4 +- installer/bin/dump_es.sh | 1 + installer/bin/export_kibana-objects.sh | 1 + installer/bin/import_kibana-objects.sh | 1 + installer/bin/myip.sh | 5 -- installer/bin/restore_es.sh | 1 + installer/bin/status.sh | 10 +--- installer/bin/update-images.sh | 78 -------------------------- installer/etc/rc.local | 3 - installer/install.sh | 38 +++++-------- 14 files changed, 23 insertions(+), 245 deletions(-) delete mode 100755 installer/bin/check.sh delete mode 100755 installer/bin/dcres.sh delete mode 100755 installer/bin/update-images.sh diff --git a/installer/bin/backup_es_folders.sh b/installer/bin/backup_es_folders.sh index c3c19de0..08155332 100755 --- a/installer/bin/backup_es_folders.sh +++ b/installer/bin/backup_es_folders.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Backup all ES relevant folders # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/check.sh b/installer/bin/check.sh deleted file mode 100755 index 0ea3423c..00000000 --- a/installer/bin/check.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -######################################################## -# T-Pot # -# Check container and services script # -# # -# v16.10.0 by mo, DTAG, 2016-05-12 # -######################################################## -if [ -a /var/run/check.lock ]; - then - echo "Lock exists. Exiting now." - exit -fi - -myIMAGES=$(cat /etc/tpot/images.conf) - -touch /var/run/check.lock - -myUPTIME=$(awk '{print int($1/60)}' /proc/uptime) -for i in $myIMAGES - do - if [ "$i" != "ui-for-docker" ] && [ "$i" != "netdata" ] && [ "$i" != "spiderfoot" ]; - then - myCIDSTATUS=$(docker exec $i supervisorctl status) - if [ $? -ne 0 ]; - then - myCIDSTATUS=1 - else - myCIDSTATUS=$(echo $myCIDSTATUS | egrep -c "(STOPPED|FATAL)") - fi - if [ $myUPTIME -gt 4 ] && [ $myCIDSTATUS -gt 0 ]; - then - echo "Restarting "$i"." - systemctl stop $i - sleep 5 - systemctl start $i - fi - fi -done - -rm /var/run/check.lock diff --git a/installer/bin/clean.sh b/installer/bin/clean.sh index f3906114..ae60d124 100755 --- a/installer/bin/clean.sh +++ b/installer/bin/clean.sh @@ -1,11 +1,5 @@ #!/bin/bash - -######################################################## -# T-Pot # -# Container Data Cleaner # -# # -# v16.10.0 by mo, DTAG, 2016-05-28 # -######################################################## +# T-Pot Container Data Cleaner # Set persistence myPERSISTENCE=$1 diff --git a/installer/bin/dcres.sh b/installer/bin/dcres.sh deleted file mode 100755 index 26e1f884..00000000 --- a/installer/bin/dcres.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -######################################################## -# T-Pot # -# Container and services restart script # -# # -# v16.10.0 by mo, DTAG, 2016-05-12 # -######################################################## -myCOUNT=1 - -while true -do - if ! [ -a /var/run/check.lock ]; - then break - fi - sleep 0.1 - if [ "$myCOUNT" = "1" ]; - then - echo -n "Waiting for services " - else echo -n . - fi - if [ "$myCOUNT" = "6000" ]; - then - echo - echo "Overriding check.lock" - rm /var/run/check.lock - break - fi - myCOUNT=$[$myCOUNT +1] -done - -myIMAGES=$(cat /etc/tpot/images.conf) - -touch /var/run/check.lock - -myUPTIME=$(awk '{print int($1/60)}' /proc/uptime) -if [ $myUPTIME -gt 4 ]; - then - for i in $myIMAGES - do - systemctl stop $i - done - echo "### Waiting 10 seconds before restarting docker ..." - sleep 10 - iptables -w -F - systemctl restart docker - while true - do - docker info > /dev/null - if [ $? -ne 0 ]; - then - echo Docker daemon is still starting. - else - echo Docker daemon is now available. - break - fi - sleep 0.1 - done - echo "### Docker is now up and running again." - echo "### Removing obsolete container data ..." - docker rm -v $(docker ps -aq) - echo "### Removing obsolete image data ..." - docker rmi $(docker images | grep "" | awk '{print $3}') - echo "### Starting T-Pot services ..." - for i in $myIMAGES - do - systemctl start $i - done - sleep 5 - else - echo "### T-Pot needs to be up and running for at least 5 minutes." -fi - -rm /var/run/check.lock - -/etc/rc.local diff --git a/installer/bin/dps.sh b/installer/bin/dps.sh index 6607b170..2f3910b1 100755 --- a/installer/bin/dps.sh +++ b/installer/bin/dps.sh @@ -1,5 +1,5 @@ #/bin/bash - +# Show current status of all running containers # Let's ensure normal operation on exit or if interrupted ... function fuCLEANUP { stty sane @@ -7,8 +7,6 @@ function fuCLEANUP { trap fuCLEANUP EXIT stty -echo -icanon time 0 min 0 -#myIMAGES=$(cat /etc/tpot/images.conf) -#myIMAGES=$(/usr/bin/docker ps -a -f name=$i --format "table {{.Names}}" | grep -v NAMES) myIMAGES=$(cat /etc/tpot/tpot.yml | grep container_name | cut -d: -f2) while true do diff --git a/installer/bin/dump_es.sh b/installer/bin/dump_es.sh index 74e95043..d496a98e 100755 --- a/installer/bin/dump_es.sh +++ b/installer/bin/dump_es.sh @@ -1,4 +1,5 @@ #/bin/bash +# Dump all ES data # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/export_kibana-objects.sh b/installer/bin/export_kibana-objects.sh index ac630dd3..a48b9011 100755 --- a/installer/bin/export_kibana-objects.sh +++ b/installer/bin/export_kibana-objects.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Export all Kibana objects # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/import_kibana-objects.sh b/installer/bin/import_kibana-objects.sh index c291ce63..2ae37e6a 100755 --- a/installer/bin/import_kibana-objects.sh +++ b/installer/bin/import_kibana-objects.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Import Kibana objects # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/myip.sh b/installer/bin/myip.sh index 10580971..86a9114e 100755 --- a/installer/bin/myip.sh +++ b/installer/bin/myip.sh @@ -34,8 +34,6 @@ httplist=( whatismyip.akamai.com ) - - # function to shuffle the global array "array" shuffle() { local i tmp size max rand @@ -48,7 +46,6 @@ shuffle() { done } - # if we have dig and a list of dns methods, try that first if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then eval array=( \"\${dnslist[@]}\" ) @@ -64,9 +61,7 @@ if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then done fi - # if we haven't succeeded with DNS, try HTTP - if [ ${#httplist[*]} == 0 ]; then echo "No hosts in httplist array!" >&2 exit 1 diff --git a/installer/bin/restore_es.sh b/installer/bin/restore_es.sh index e1dc01d3..506a5c8c 100755 --- a/installer/bin/restore_es.sh +++ b/installer/bin/restore_es.sh @@ -1,4 +1,5 @@ #/bin/bash +# Restore folder based ES backup # Make sure ES is available myES="http://127.0.0.1:64298/" myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green) diff --git a/installer/bin/status.sh b/installer/bin/status.sh index a37900c1..c4f27eb9 100755 --- a/installer/bin/status.sh +++ b/installer/bin/status.sh @@ -1,16 +1,10 @@ #!/bin/bash - -######################################################## -# T-Pot # -# Container and services status script # -# # -# v16.10.0 by mo, DTAG, 2016-05-12 # -######################################################## +# Show status of SupervisorD within running containers myCOUNT=1 if [[ $1 == "" ]] then - myIMAGES=$(cat /etc/tpot/images.conf) + myIMAGES=$(cat /etc/tpot/tpot.yml | grep container_name | cut -d: -f2) else myIMAGES=$1 fi diff --git a/installer/bin/update-images.sh b/installer/bin/update-images.sh deleted file mode 100755 index 0ee431eb..00000000 --- a/installer/bin/update-images.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -########################################################## -# T-Pot # -# Only start the containers found in /etc/systemd/system # -# # -# v17.06 by mo, DTAG, 2017-03-13 # -########################################################## - -# Make sure not to interrupt a check -while true -do - if ! [ -a /var/run/check.lock ]; - then break - fi - sleep 0.1 - if [ "$myCOUNT" = "1" ]; - then - echo -n "Waiting for services " - else echo -n . - fi - if [ "$myCOUNT" = "6000" ]; - then - echo - echo "Overriding check.lock" - rm /var/run/check.lock - break - fi - myCOUNT=$[$myCOUNT +1] -done - -# We do not want to get interrupted by a check -touch /var/run/check.lock - -# Stop T-Pot services and disable all T-Pot services -echo "### Stopping T-Pot services and cleaning up." -for i in $(cat /etc/tpot/imgcfg/all_images.conf); - do - systemctl stop $i - sleep 2 - systemctl disable $i; - rm /etc/systemd/system/$i.service -done - -# Restarting docker services and optionally clear local repository -echo "### Stopping docker services ..." -systemctl stop docker -sleep 1 -# If option "hard" clear the whole repository -if [ "$1" = "hard" ]; - then - echo "### Clearing local docker repository." - rm -rf /var/lib/docker - sleep 1 -fi -echo "### Starting docker services ..." -systemctl start docker -sleep 1 - -# Enable only T-Pot systemd scripts from images.conf and pull the images -for i in $(cat /etc/tpot/images.conf); - do - echo - echo "### Now pulling "$i - docker pull dtagdevsec/$i:1706; - cp /etc/tpot/systemd/$i.service /etc/systemd/system/ - systemctl enable $i; -done - -# Announce reboot -echo -echo "### Rebooting." - -# Allow checks to resume -rm /var/run/check.lock - -# Reboot -reboot diff --git a/installer/etc/rc.local b/installer/etc/rc.local index 8e5ade55..7b64eb94 100755 --- a/installer/etc/rc.local +++ b/installer/etc/rc.local @@ -16,6 +16,3 @@ MY_HOSTNAME=$HOSTNAME EOF echo $myLOCALIP > /data/elk/logstash/mylocal.ip chown tpot:tpot /data/ews/conf/ews.ip -if [ -f /var/run/check.lock ]; - then rm /var/run/check.lock -fi diff --git a/installer/install.sh b/installer/install.sh index 472146ce..24717ac9 100755 --- a/installer/install.sh +++ b/installer/install.sh @@ -1,10 +1,5 @@ #!/bin/bash -######################################################## -# T-Pot post install script # -# Ubuntu server 16.04.0, x64 # -# # -# v17.06 by mo, DTAG, 2017-03-22 # -######################################################## +# T-Pot post install script # Set TERM, DIALOGRC export TERM=linux @@ -32,6 +27,8 @@ fuRANDOMWORD () { } # Let's wait a few seconds to avoid interference with service messages +sleep 3 +tput civis dialog --no-ok --no-cancel --backtitle "$myBACKTITLE" --title "[ Wait to avoid interference with service messages ]" --pause "" 6 80 7 # Let's setup the proxy for env @@ -104,6 +101,7 @@ rm -rf /usr/share/nginx/html/index.html 2>&1 | dialog --title "[ Removing NGINX # Let's ask user for install flavor # Install types are TPOT, HP, INDUSTRIAL, ALL +tput cnorm myFLAVOR=$(dialog --no-cancel --backtitle "$myBACKTITLE" --title "[ Choose your edition ]" --no-tags --menu \ "\nRequired: 4GB RAM, 64GB disk\nRecommended: 8GB RAM, 128GB SSD" 14 60 4 \ "TPOT" "Standard Honeypots, Suricata & ELK" \ @@ -198,6 +196,7 @@ while [ "$myPASS1" != "$myPASS2" ] && [ "$mySECURE" == "0" ] htpasswd -b -c /etc/nginx/nginxpasswd "$myUSER" "$myPASS1" 2>&1 | dialog --title "[ Setting up user and password ]" $myPROGRESSBOXCONF; # Let's generate a SSL self-signed certificate without interaction (browsers will see it invalid anyway) +tput civis mkdir -p /etc/nginx/ssl 2>&1 | dialog --title "[ Generating a self-signed-certificate for NGINX ]" $myPROGRESSBOXCONF; openssl req \ -nodes \ @@ -372,15 +371,15 @@ esac # Let's load docker images myIMAGESCOUNT=$(cat /root/tpot/etc/tpot/tpot.yml | grep container_name | cut -d: -f2 | wc -l) j=0 -for name in $(cat /root/tpot/etc/tpot/tpot.yml | grep container_name | cut -d: -f2) +for name in $(cat /root/tpot/etc/tpot/tpot.yml | grep image | cut -d'"' -f2) do dialog --title "[ Downloading docker images, please be patient ]" --backtitle "$myBACKTITLE" \ - --gauge "\n Now downloading: dtagdevsec/$name:1706\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <&1>/dev/null + docker pull $name 2>&1>/dev/null let j+=1 dialog --title "[ Downloading docker images, please be patient ]" --backtitle "$myBACKTITLE" \ - --gauge "\n Now downloading: dtagdevsec/$name:1706\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <&1>/dev/null <:/api delete --filters resource= && alerta --endpoint-url http://:/api send -e IP -r -E Production -s ok -S T-Pot -t \$(cat /data/elk/logstash/mylocal.ip) --status open +#*/5 * * * * root alerta --endpoint-url http://:/api delete --filters resource= && alerta --endpoint-url http://:/api send -e IP -r -E Production -s ok -S T-Pot -t \$(cat /data/elk/logstash/mylocal.ip) --status open # Check if updated images are available and download them -27 1 * * * root /usr/bin/docker-compose -f /etc/tpot/tpot.yml pull - -# Restart docker service and containers -#27 3 * * * root dcres.sh +27 1 * * * root /usr/bin/docker-compose -f /etc/tpot/tpot.yml pull # Delete elastic indices older than 90 days (kibana index is omitted by default) -#27 4 * * * root docker exec elk bash -c '/usr/local/bin/curator --host 127.0.0.1 delete indices --older-than 90 --time-unit days --timestring \%Y.\%m.\%d' - -# Update IP and erase check.lock if it exists -27 5 * * * root /etc/rc.local +#27 4 * * * root docker exec elk bash -c '/usr/local/bin/curator --host 127.0.0.1 delete indices --older-than 90 --time-unit days --timestring \%Y.\%m.\%d' # Daily reboot -27 23 * * * root reboot +27 3 * * * root reboot # Check for updated packages every sunday, upgrade and reboot -27 16 * * 0 root apt-get autoclean -y && apt-get autoremove -y && apt-get update -y && apt-get upgrade -y && sleep 10 && reboot +27 16 * * 0 root apt-get autoclean -y && apt-get autoremove -y && apt-get update -y && apt-get upgrade -y && sleep 10 && reboot EOF # Let's create some files and folders