scripts, configs, optimizations, dashboard base

installer/bin/backup_elk.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+########################################################
+# T-Pot                                                #
+# ELK DB backup script                                 #
+#                                                      #
+# v0.01 by mo, DTAG, 2016-02-12                        #
+########################################################
+myCOUNT=1
+myDATE=$(date +%Y%m%d%H%M)
+myELKPATH="/data/elk/"
+myBACKUPPATH="/data/"
+
+# Make sure not to interrupt a check
+while true
+do
+  if ! [ -a /var/run/check.lock ];
+    then break
+  fi
+  sleep 0.1
+  if [ "$myCOUNT" = "1" ];
+    then
+      echo -n "Waiting for services "
+    else echo -n .
+  fi
+  if [ "$myCOUNT" = "6000" ];
+    then
+    echo
+    echo "Overriding check.lock"
+    rm /var/run/check.lock
+    break
+  fi
+  myCOUNT=$[$myCOUNT +1]
+done
+
+# We do not want to get interrupted by a check
+touch /var/run/check.lock
+
+# Stop ELK to lift db lock
+echo "Now stopping ELK ..."
+service elk stop
+sleep 10
+
+# Backup DB in 2 flavors
+echo "Now backing up Elasticsearch data ..."
+tar cvfz $myBACKUPPATH"$myDATE"_elkall.tgz $myELKPATH  
+rm -rf "$myELKPATH"log/*
+rm -rf "$myELKPATH"data/elasticsearch/nodes/0/indices/logstash*
+tar cvfz $myBACKUPPATH"$myDATE"_elkbase.tgz $myELKPATH
+rm -rf $myELKPATH
+tar xvfz $myBACKUPPATH"$myDATE"_elkall.tgz -C / 
+#tar xvfz $myBACKUPPATH"$myDATE"_elkbase.tgz -C /
+chmod 760 -R $myELKPATH
+chown tpot:tpot -R $myELKPATH
+
+# Start ELK
+service elk start
+echo "Now starting up ELK ..."
+
+# Allow checks to resume
+rm /var/run/check.lock
+

installer/bin/check.sh

@@ -4,10 +4,12 @@
 # T-Pot                                                #
 # Check container and services script                  #
 #                                                      #
-# v0.02 by mo, DTAG, 2015-08-08                        #
+# v0.03 by mo, DTAG, 2016-02-12                        #
 ########################################################
 if [ -a /var/run/check.lock ];
-  then exit
+  then 
+    echo "Lock exists. Exiting now."
+    exit
 fi
 
 myIMAGES=$(cat /data/images.conf)
@@ -24,38 +26,13 @@ for i in $myIMAGES
         else
           myCIDSTATUS=$(echo $myCIDSTATUS | egrep -c "(STOPPED|FATAL)")
       fi
-      if [ $myCIDSTATUS -gt 0 ];
+      if [ $myUPTIME -gt 4 ] && [ $myCIDSTATUS -gt 0 ];
         then
-          if [ $myUPTIME -gt 5 ];
-            then
-              for j in $myIMAGES
-                do
-                  service $j stop
-              done
-              iptables -w -F
-              service docker restart
-              while true
-                do
-                  docker info > /dev/null
-                  if [ $? -ne 0 ];
-                    then
-                      echo Docker daemon is still starting.
-                    else
-                      echo Docker daemon is now available.
-                      break
-                  fi
-                  sleep 0.1
-              done
-              docker rm -v $(docker ps -aq)
-              for j in $myIMAGES
-                do
-                  service $j start
-                  sleep $(((RANDOM %5)+5))
-              done
-              rm /var/run/check.lock
-              exit
-          fi
+          echo "Restarting "$i"."
+          service $i stop
+          sleep 5
+          service $i start
       fi
 done
-
+        
 rm /var/run/check.lock

installer/bin/dcres.sh

@@ -4,7 +4,7 @@
 # T-Pot                                                #
 # Container and services restart script                #
 #                                                      #
-# v0.03 by mo, DTAG, 2015-11-02                        #
+# v0.04 by mo, DTAG, 2016-02-12                        #
 ########################################################
 myCOUNT=1
 
@@ -21,9 +21,9 @@ do
   fi
   if [ "$myCOUNT" = "6000" ];
     then
-    echo
-    echo "Overriding check.lock"
-    rm /var/run/check.lock
+      echo
+      echo "Overriding check.lock"
+      rm /var/run/check.lock
     break
   fi
   myCOUNT=$[$myCOUNT +1]
@@ -34,12 +34,14 @@ myIMAGES=$(cat /data/images.conf)
 touch /var/run/check.lock
 
 myUPTIME=$(awk '{print int($1/60)}' /proc/uptime)
-if [ $myUPTIME -gt 5 ];
+if [ $myUPTIME -gt 4 ];
   then
     for i in $myIMAGES
       do
         service $i stop
     done
+    echo "Waiting 10 seconds before restarting docker ..."
+    sleep 10
     iptables -w -F
     service docker restart
     while true
@@ -54,15 +56,25 @@ if [ $myUPTIME -gt 5 ];
         fi
         sleep 0.1
     done
+    echo "Docker is now up and running again."
+    echo "Removing obsolete container data ..."
     docker rm -v $(docker ps -aq)
+    echo "Removing obsolete image data ..."
     docker rmi $(docker images | grep "^<none>" | awk '{print $3}')
+    echo "Starting T-Pot services ..."
     for i in $myIMAGES
       do
         service $i start
-        sleep $(((RANDOM %5)+5))
     done
+    sleep 5
+  else 
+    echo "T-Pot needs to be up and running for at least 5 minutes."
 fi
 
 rm /var/run/check.lock
 
 /etc/rc.local
+
+echo "Done. Now running status.sh"
+/usr/bin/status.sh
+

installer/bin/status.sh

@@ -4,10 +4,16 @@
 # T-Pot                                                #
 # Container and services status script                 #
 #                                                      #
-# v0.04 by mo, DTAG, 2015-08-20                        #
+# v0.05 by mo, DTAG, 2016-02-12                        #
 ########################################################
 myCOUNT=1
-myIMAGES=$(cat /data/images.conf)
+
+if [[ $1 == "" ]]
+  then
+    myIMAGES=$(cat /data/images.conf)
+  else myIMAGES=$1
+fi
+
 while true
 do
   if ! [ -a /var/run/check.lock ];

installer/bin/update-images.sh

@@ -2,11 +2,36 @@
 
 ########################################################
 # T-Pot                                                #
-# Only start the container found in /etc/init/t-pot    #
+# Only start the containers found in /etc/init/        #
 #                                                      #
-# v0.02 by mo, DTAG, 2016-02-08                        #
+# v0.03 by mo, DTAG, 2016-02-12                        #
 ########################################################
 
+# Make sure not to interrupt a check
+while true
+do
+  if ! [ -a /var/run/check.lock ];
+    then break
+  fi
+  sleep 0.1
+  if [ "$myCOUNT" = "1" ];
+    then
+      echo -n "Waiting for services "
+    else echo -n .
+  fi
+  if [ "$myCOUNT" = "6000" ];
+    then
+    echo
+    echo "Overriding check.lock"
+    rm /var/run/check.lock
+    break
+  fi
+  myCOUNT=$[$myCOUNT +1]
+done
+
+# We do not want to get interrupted by a check
+touch /var/run/check.lock
+
 # Delete all T-Pot upstart scripts
 for i in $(ls /data/upstart/);
   do
@@ -20,4 +45,12 @@ for i in $(cat /data/images.conf);
     cp /data/upstart/"$i".conf /etc/init/;
 done
 
-echo Please reboot for the changes to take effect.
+# Allow checks to resume
+rm /var/run/check.lock
+
+# Announce reboot
+echo "Rebooting in 5 seconds for the changes to take effect."
+sleep 5
+
+# Reboot
+reboot

installer/data/all_images.conf

@@ -0,0 +1,8 @@
+cowrie
+dionaea
+elasticpot
+elk
+emobility
+glastopf
+honeytrap
+suricata

installer/data/upstart/cowrie.conf

@@ -27,5 +27,5 @@ script
 end script
 post-start script
   # Delay next start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2
 end script

installer/data/upstart/dionaea.conf

@@ -28,5 +28,5 @@ script
 end script
 post-start script
   # Delay next start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2
 end script

installer/data/upstart/elasticpot.conf

@@ -27,5 +27,5 @@ script
 end script
 post-start script
   # Delay next start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2
 end script

installer/data/upstart/elk.conf

@@ -22,5 +22,5 @@ script
 end script
 post-start script
   # Delay next  start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2
 end script

installer/data/upstart/emobility.conf

@@ -25,6 +25,6 @@ pre-start script
 end script
 script
   # Delayed start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2 
   /usr/bin/docker run --name emobility --cap-add=NET_ADMIN -p 8080:8080 -v /data/emobility:/data/eMobility -v /data/ews:/data/ews --rm=true dtagdevsec/emobility:latest1603
 end script

installer/data/upstart/glastopf.conf

@@ -24,5 +24,5 @@ script
 end script
 post-start script
   # Delay next start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2
 end script

installer/data/upstart/honeytrap.conf

@@ -28,7 +28,7 @@ script
 end script
 post-start script
   # Delay next start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2
 end script
 post-stop script
   /sbin/iptables -w -D INPUT -p tcp --syn -m state --state NEW -j NFQUEUE

installer/data/upstart/suricata.conf

@@ -28,7 +28,7 @@ pre-start script
 end script
 script
   # Delayed start to avoid rapid respawning
-  sleep $(((RANDOM % 5)+5))
+  sleep 2
   /usr/bin/docker run --name suricata --cap-add=NET_ADMIN --net=host --rm=true -v /data/suricata:/data/suricata dtagdevsec/suricata:latest1603
 end script
 post-start script

installer/install.sh

@@ -3,11 +3,11 @@
 # T-Pot post install script                            #
 # Ubuntu server 14.04.3, x64                           #
 #                                                      #
-# v16.03.7 by mo, DTAG, 2016-02-11                     #
+# v16.03.8 by mo, DTAG, 2016-02-12                     #
 ########################################################
 
 # Type of install, SENSOR, INDUSTRIAL or FULL?
-myFLAVOR="FULL"
+myFLAVOR="TPOT"
 
 # Some global vars
 myPROXYFILEPATH="/root/tpot/etc/proxy"
@@ -206,18 +206,23 @@ DOCKER_OPTS="-r=false"
 EOF
 
 # Let's make sure only myFLAVOR images will be downloaded and started
-if [ "$myFLAVOR" = "SENSOR" ]
+if [ "$myFLAVOR" = "HP" ]
   then
-    cp /root/tpot/data/sensor_images.conf /root/tpot/data/images.conf
+    cp /root/tpot/data/hp_images.conf /root/tpot/data/images.conf
 fi
 if [ "$myFLAVOR" = "INDUSTRIAL" ]
   then
     cp /root/tpot/data/industrial_images.conf /root/tpot/data/images.conf
 fi
-if [ "$myFLAVOR" = "FULL" ]
+if [ "$myFLAVOR" = "TPOT" ]
   then
-    cp /root/tpot/data/full_images.conf /root/tpot/data/images.conf
+    cp /root/tpot/data/tpot_images.conf /root/tpot/data/images.conf
 fi
+if [ "$myFLAVOR" = "ALL" ]
+  then
+    cp /root/tpot/data/all_images.conf /root/tpot/data/images.conf
+fi
+
 
 # Let's load docker images
 fuECHO "### Loading docker images. Please be patient, this may take a while."
@@ -308,6 +313,7 @@ for i in $(cat /data/images.conf);
   do
     cp /data/upstart/$i.conf /etc/init/;
 done
+tar xvfz /data/elkbase.tgz -C / 
 
 # Let's take care of some files and permissions
 chmod 760 -R /data

makeiso.sh

@@ -86,7 +86,7 @@ if [ "$mySTART" = "1" ];
 fi
 
 # Let's ask for the type of installation SENSOR, INDUSTRIAL or FULL?
-myFLAVOR=$(dialog --no-cancel --backtitle "$myBACKTITLE" --title "[ Installation type ... ]" --radiolist "" 9 70 4 "FULL" "Everything (w/o INDUSTRIAL)" on "SENSOR" "Honeypots only (w/o INDUSTRIAL)" off "INDUSTRIAL" "eMobility, ELK, Suricata (8GB RAM required)" off 3>&1 1>&2 2>&3 3>&-)
+myFLAVOR=$(dialog --no-cancel --backtitle "$myBACKTITLE" --title "[ Installation type ... ]" --radiolist "" 11 70 4 "TPOT" "Standard (w/o INDUSTRIAL)" on "HP" "Honeypots only (w/o INDUSTRIAL)" off "INDUSTRIAL" "eMobility, ELK, Suricata (8GB RAM recommended)" off "ALL" "Everything (8GB RAM required)" off 3>&1 1>&2 2>&3 3>&-)
 sed -i 's#^myFLAVOR=.*#myFLAVOR="'$myFLAVOR'"#' $myINSTALLERPATH
 
 # Let's ask the user for a proxy ...