tweaking for docker-compose

get rid of self-check scripts, docker-compose takes care of that now
use tpot.yml config for tpot scripts
wipe crontab clean of legacy scripts
check.lock no longer needed (rc.local)
adjust installer (invisible cursor, get image info from tpot.yml, some tweaking)

installer/bin/backup_es_folders.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# Backup all ES relevant folders
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)

installer/bin/check.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-########################################################
-# T-Pot                                                #
-# Check container and services script                  #
-#                                                      #
-# v16.10.0 by mo, DTAG, 2016-05-12                     #
-########################################################
-if [ -a /var/run/check.lock ];
-  then
-    echo "Lock exists. Exiting now."
-    exit
-fi
-
-myIMAGES=$(cat /etc/tpot/images.conf)
-
-touch /var/run/check.lock
-
-myUPTIME=$(awk '{print int($1/60)}' /proc/uptime)
-for i in $myIMAGES
-  do
-    if [ "$i" != "ui-for-docker" ] && [ "$i" != "netdata" ] && [ "$i" != "spiderfoot" ];
-      then
-        myCIDSTATUS=$(docker exec $i supervisorctl status)
-        if [ $? -ne 0 ];
-          then
-            myCIDSTATUS=1
-          else
-            myCIDSTATUS=$(echo $myCIDSTATUS | egrep -c "(STOPPED|FATAL)")
-        fi
-        if [ $myUPTIME -gt 4 ] && [ $myCIDSTATUS -gt 0 ];
-          then
-            echo "Restarting "$i"."
-            systemctl stop $i
-            sleep 5
-            systemctl start $i
-        fi
-    fi
-done
-
-rm /var/run/check.lock

installer/bin/clean.sh

@@ -1,11 +1,5 @@
 #!/bin/bash
-
+# T-Pot Container Data Cleaner
-########################################################
-# T-Pot                                                #
-# Container Data Cleaner                               #
-#                                                      #
-# v16.10.0 by mo, DTAG, 2016-05-28                     #
-########################################################
 
 # Set persistence
 myPERSISTENCE=$1

installer/bin/dcres.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-
-########################################################
-# T-Pot                                                #
-# Container and services restart script                #
-#                                                      #
-# v16.10.0 by mo, DTAG, 2016-05-12                     #
-########################################################
-myCOUNT=1
-
-while true
-do
-  if ! [ -a /var/run/check.lock ];
-    then break
-  fi
-  sleep 0.1
-  if [ "$myCOUNT" = "1" ];
-    then
-      echo -n "Waiting for services "
-    else echo -n .
-  fi
-  if [ "$myCOUNT" = "6000" ];
-    then
-      echo
-      echo "Overriding check.lock"
-      rm /var/run/check.lock
-    break
-  fi
-  myCOUNT=$[$myCOUNT +1]
-done
-
-myIMAGES=$(cat /etc/tpot/images.conf)
-
-touch /var/run/check.lock
-
-myUPTIME=$(awk '{print int($1/60)}' /proc/uptime)
-if [ $myUPTIME -gt 4 ];
-  then
-    for i in $myIMAGES
-      do
-        systemctl stop $i
-    done
-    echo "### Waiting 10 seconds before restarting docker ..."
-    sleep 10
-    iptables -w -F
-    systemctl restart docker
-    while true
-      do
-        docker info > /dev/null
-        if [ $? -ne 0 ];
-          then
-            echo Docker daemon is still starting.
-          else
-            echo Docker daemon is now available.
-            break
-        fi
-        sleep 0.1
-    done
-    echo "### Docker is now up and running again."
-    echo "### Removing obsolete container data ..."
-    docker rm -v $(docker ps -aq)
-    echo "### Removing obsolete image data ..."
-    docker rmi $(docker images | grep "<none>" | awk '{print $3}')
-    echo "### Starting T-Pot services ..."
-    for i in $myIMAGES
-      do
-        systemctl start $i
-    done
-    sleep 5
-  else
-    echo "### T-Pot needs to be up and running for at least 5 minutes."
-fi
-
-rm /var/run/check.lock
-
-/etc/rc.local

installer/bin/dps.sh

@@ -1,5 +1,5 @@
 #/bin/bash
-
+# Show current status of all running containers
 # Let's ensure normal operation on exit or if interrupted ...
 function fuCLEANUP {
   stty sane
@@ -7,8 +7,6 @@ function fuCLEANUP {
 trap fuCLEANUP EXIT
 
 stty -echo -icanon time 0 min 0
-#myIMAGES=$(cat /etc/tpot/images.conf)
-#myIMAGES=$(/usr/bin/docker ps -a -f name=$i --format "table {{.Names}}" | grep -v NAMES)
 myIMAGES=$(cat /etc/tpot/tpot.yml | grep container_name | cut -d: -f2)
 while true
   do

installer/bin/dump_es.sh

@@ -1,4 +1,5 @@
 #/bin/bash
+# Dump all ES data
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)

installer/bin/export_kibana-objects.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# Export all Kibana objects
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)

installer/bin/import_kibana-objects.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# Import Kibana objects
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)

installer/bin/myip.sh

@@ -34,8 +34,6 @@ httplist=(
     whatismyip.akamai.com
 )
 
-
-
 # function to shuffle the global array "array"
 shuffle() {
    local i tmp size max rand
@@ -48,7 +46,6 @@ shuffle() {
    done
 }
 
-
 # if we have dig and a list of dns methods, try that first
 if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then
     eval array=( \"\${dnslist[@]}\" )
@@ -64,9 +61,7 @@ if hash dig 2>/dev/null && [ ${#dnslist[*]} -gt 0 ]; then
     done
 fi
 
-
 # if we haven't succeeded with DNS, try HTTP
-
 if [ ${#httplist[*]} == 0 ]; then
     echo "No hosts in httplist array!" >&2
     exit 1

installer/bin/restore_es.sh

@@ -1,4 +1,5 @@
 #/bin/bash
+# Restore folder based ES backup
 # Make sure ES is available
 myES="http://127.0.0.1:64298/"
 myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)

installer/bin/status.sh

@@ -1,16 +1,10 @@
 #!/bin/bash
-
+# Show status of SupervisorD within running containers
-########################################################
-# T-Pot                                                #
-# Container and services status script                 #
-#                                                      #
-# v16.10.0 by mo, DTAG, 2016-05-12                     #
-########################################################
 myCOUNT=1
 
 if [[ $1 == "" ]]
   then
-    myIMAGES=$(cat /etc/tpot/images.conf)
+    myIMAGES=$(cat /etc/tpot/tpot.yml | grep container_name | cut -d: -f2)
   else myIMAGES=$1
 fi
 

installer/bin/update-images.sh

@@ -1,78 +0,0 @@
-#!/bin/bash
-
-##########################################################
-# T-Pot                                                  #
-# Only start the containers found in /etc/systemd/system #
-#                                                        #
-# v17.06 by mo, DTAG, 2017-03-13                         #
-##########################################################
-
-# Make sure not to interrupt a check
-while true
-do
-  if ! [ -a /var/run/check.lock ];
-    then break
-  fi
-  sleep 0.1
-  if [ "$myCOUNT" = "1" ];
-    then
-      echo -n "Waiting for services "
-    else echo -n .
-  fi
-  if [ "$myCOUNT" = "6000" ];
-    then
-    echo
-    echo "Overriding check.lock"
-    rm /var/run/check.lock
-    break
-  fi
-  myCOUNT=$[$myCOUNT +1]
-done
-
-# We do not want to get interrupted by a check
-touch /var/run/check.lock
-
-# Stop T-Pot services and disable all T-Pot services
-echo "### Stopping T-Pot services and cleaning up."
-for i in $(cat /etc/tpot/imgcfg/all_images.conf);
-  do
-    systemctl stop $i
-    sleep 2
-    systemctl disable $i;
-    rm /etc/systemd/system/$i.service
-done
-
-# Restarting docker services and optionally clear local repository
-echo "### Stopping docker services ..."
-systemctl stop docker
-sleep 1
-# If option "hard" clear the whole repository
-if [ "$1" = "hard" ];
-  then
-    echo "### Clearing local docker repository."
-    rm -rf /var/lib/docker
-    sleep 1
-fi
-echo "### Starting docker services ..."
-systemctl start docker
-sleep 1
-
-# Enable only T-Pot systemd scripts from images.conf and pull the images
-for i in $(cat /etc/tpot/images.conf);
-  do
-    echo
-    echo "### Now pulling "$i
-    docker pull dtagdevsec/$i:1706;
-    cp /etc/tpot/systemd/$i.service /etc/systemd/system/
-    systemctl enable $i;
-done
-
-# Announce reboot
-echo
-echo "### Rebooting."
-
-# Allow checks to resume
-rm /var/run/check.lock
-
-# Reboot
-reboot

installer/etc/rc.local

@@ -16,6 +16,3 @@ MY_HOSTNAME=$HOSTNAME
 EOF
 echo $myLOCALIP > /data/elk/logstash/mylocal.ip
 chown tpot:tpot /data/ews/conf/ews.ip
-if [ -f /var/run/check.lock ];
-  then rm /var/run/check.lock
-fi

installer/install.sh

@@ -1,10 +1,5 @@
 #!/bin/bash
-########################################################
+# T-Pot post install script
-# T-Pot post install script                            #
-# Ubuntu server 16.04.0, x64                           #
-#                                                      #
-# v17.06 by mo, DTAG, 2017-03-22                       #
-########################################################
 
 # Set TERM, DIALOGRC
 export TERM=linux
@@ -32,6 +27,8 @@ fuRANDOMWORD () {
 }
 
 # Let's wait a few seconds to avoid interference with service messages
+sleep 3
+tput civis
 dialog --no-ok --no-cancel --backtitle "$myBACKTITLE" --title "[ Wait to avoid interference with service messages ]" --pause "" 6 80 7
 
 # Let's setup the proxy for env
@@ -104,6 +101,7 @@ rm -rf /usr/share/nginx/html/index.html 2>&1 | dialog --title "[ Removing NGINX
 
 # Let's ask user for install flavor
 # Install types are TPOT, HP, INDUSTRIAL, ALL
+tput cnorm
 myFLAVOR=$(dialog --no-cancel --backtitle "$myBACKTITLE" --title "[ Choose your edition ]" --no-tags --menu \
 "\nRequired: 4GB RAM, 64GB disk\nRecommended: 8GB RAM, 128GB SSD" 14 60 4 \
 "TPOT" "Standard Honeypots, Suricata & ELK" \
@@ -198,6 +196,7 @@ while [ "$myPASS1" != "$myPASS2"  ] && [ "$mySECURE" == "0" ]
 htpasswd -b -c /etc/nginx/nginxpasswd "$myUSER" "$myPASS1" 2>&1 | dialog --title "[ Setting up user and password ]" $myPROGRESSBOXCONF;
 
 # Let's generate a SSL self-signed certificate without interaction (browsers will see it invalid anyway)
+tput civis
 mkdir -p /etc/nginx/ssl 2>&1 | dialog --title "[ Generating a self-signed-certificate for NGINX ]" $myPROGRESSBOXCONF;
 openssl req \
         -nodes \
@@ -372,15 +371,15 @@ esac
 # Let's load docker images
 myIMAGESCOUNT=$(cat /root/tpot/etc/tpot/tpot.yml | grep container_name | cut -d: -f2 | wc -l)
 j=0
-for name in $(cat /root/tpot/etc/tpot/tpot.yml | grep container_name | cut -d: -f2)
+for name in $(cat /root/tpot/etc/tpot/tpot.yml | grep image | cut -d'"' -f2)
   do
     dialog --title "[ Downloading docker images, please be patient ]" --backtitle "$myBACKTITLE" \
-           --gauge "\n  Now downloading: dtagdevsec/$name:1706\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <<EOF
+           --gauge "\n  Now downloading: $name\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <<EOF
 EOF
-    docker pull dtagdevsec/$name:1706 2>&1>/dev/null
+    docker pull $name 2>&1>/dev/null
     let j+=1
     dialog --title "[ Downloading docker images, please be patient ]" --backtitle "$myBACKTITLE" \
-           --gauge "\n  Now downloading: dtagdevsec/$name:1706\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <<EOF
+           --gauge "\n  Now downloading: $name\n" 8 80 $(expr 100 \* $j / $myIMAGESCOUNT) <<EOF
 EOF
   done
 
@@ -410,26 +409,17 @@ dialog --title "[ Adding cronjobs ]" $myPROGRESSBOXCONF <<EOF
 EOF
 tee -a /etc/crontab 2>&1>/dev/null <<EOF
 
-# Check if containers and services are up
-#*/5 * * * *	root	check.sh
-
 # Example for alerta-cli IP update
 #*/5 * * * *    root    alerta --endpoint-url http://<ip>:<port>/api delete --filters resource=<host> && alerta --endpoint-url http://<ip>:<port>/api send -e IP -r <host> -E Production -s ok -S T-Pot -t \$(cat /data/elk/logstash/mylocal.ip) --status open
 
 # Check if updated images are available and download them
 27 1 * * *      root    /usr/bin/docker-compose -f /etc/tpot/tpot.yml pull
 
-# Restart docker service and containers
-#27 3 * * *	root	dcres.sh
-
 # Delete elastic indices older than 90 days (kibana index is omitted by default)
 #27 4 * * *     root    docker exec elk bash -c '/usr/local/bin/curator --host 127.0.0.1 delete indices --older-than 90 --time-unit days --timestring \%Y.\%m.\%d'
 
-# Update IP and erase check.lock if it exists
-27 5 * * *	root	/etc/rc.local
-
 # Daily reboot
-27 23 * * *	root	reboot
+27 3 * * *      root    reboot
 
 # Check for updated packages every sunday, upgrade and reboot
 27 16 * * 0     root    apt-get autoclean -y && apt-get autoremove -y && apt-get update -y && apt-get upgrade -y && sleep 10 && reboot