compress and rotate logs if persistence enabled

if persistence is enabled, log files, downloads, binaries, etc. will be compressed and rotated
each start / stop of the t-pot service will account for a full rotation cycle if files are not empty
basically the rotation will recycle logs after 30 days, unless the service is stopped / started manually which will cause for a shorter period

installer/bin/clean.sh

@@ -1,19 +1,72 @@
 #!/bin/bash
-# T-Pot Container Data Cleaner
+# T-Pot Container Data Cleaner & Log Rotator
+
+# Set colors
+myRED=""
+myGREEN=""
+myWHITE=""
 
 # Set persistence
 myPERSISTENCE=$1
 
-# Check persistence
+# Let's create a function to check if folder is empty
-if [ "$myPERSISTENCE" = "on" ];
+fuEMPTY () {
-  then
+  local myFOLDER=$1
-    echo "### Persistence enabled, nothing to do."
+
-    exit
+echo $(ls $myFOLDER | wc -l)
-fi
+}
+
+# Let's create a function to rotate and compress logs
+fuLOGROTATE () {
+  local mySTATUS="/etc/tpot/logrotate/status"
+  local myCONF="/etc/tpot/logrotate/logrotate.conf"
+  local myCOWRIETTYLOGS="/data/cowrie/log/tty/"
+  local myCOWRIETTYTGZ="/data/cowrie/log/ttylogs.tgz"
+  local myCOWRIEDL="/data/cowrie/downloads/"
+  local myCOWRIEDLTGZ="/data/cowrie/downloads.tgz"
+  local myDIONAEABI="/data/dionaea/bistreams/"
+  local myDIONAEABITGZ="/data/dionaea/bistreams.tgz"
+  local myDIONAEABIN="/data/dionaea/binaries/"
+  local myDIONAEABINTGZ="/data/dionaea/binaries.tgz"
+  local myHONEYTRAPATTACKS="/data/honeytrap/attacks/"
+  local myHONEYTRAPATTACKSTGZ="/data/honeytrap/attacks.tgz"
+  local myHONEYTRAPDL="/data/honeytrap/downloads/"
+  local myHONEYTRAPDLTGZ="/data/honeytrap/downloads.tgz"
+
+# Ensure correct permissions and ownerships for logrotate to run without issues
+chmod 760 /data/ -R
+chown tpot:tpot /data -R
+
+# Run logrotate with force (-f) first, so the status file can be written and race conditions (with tar) be avoided
+logrotate -f -s $mySTATUS $myCONF
+
+# Compressing some folders first and rotate them later
+if [ "$(fuEMPTY $myCOWRIETTYLOGS)" != "0" ]; then tar cvfz $myCOWRIETTYTGZ $myCOWRIETTYLOGS; fi
+if [ "$(fuEMPTY $myCOWRIEDL)" != "0" ]; then tar cvfz $myCOWRIEDLTGZ $myCOWRIEDL; fi
+if [ "$(fuEMPTY $myDIONAEABI)" != "0" ]; then tar cvfz $myDIONAEABITGZ $myDIONAEABI; fi
+if [ "$(fuEMPTY $myDIONAEABIN)" != "0" ]; then tar cvfz $myDIONAEABINTGZ $myDIONAEABIN; fi
+if [ "$(fuEMPTY $myHONEYTRAPATTACKS)" != "0" ]; then tar cvfz $myHONEYTRAPATTACKSTGZ $myHONEYTRAPATTACKS; fi
+if [ "$(fuEMPTY $myHONEYTRAPDL)" != "0" ]; then tar cvfz $myHONEYTRAPDLTGZ $myHONEYTRAPDL; fi
+
+# Ensure correct permissions and ownership for previously created archives
+chmod 760 $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
+chown tpot:tpot $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ
+
+# Need to remove subfolders since too many files cause rm to exit with errors
+rm -rf $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
+
+# Recreate subfolders with correct permissions and ownership
+mkdir -p $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
+chmod 760 $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL 
+chown tpot:tpot $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL
+
+# Run logrotate again to account for previously created archives - DO NOT FORCE HERE!
+logrotate -s $mySTATUS $myCONF
+}
 
 # Let's create a function to clean up and prepare conpot data
 fuCONPOT () {
-  rm -rf /data/conpot/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/conpot/*; fi
   mkdir -p /data/conpot/log
   chmod 760 /data/conpot -R
   chown tpot:tpot /data/conpot -R
@@ -21,7 +74,7 @@ fuCONPOT () {
 
 # Let's create a function to clean up and prepare cowrie data
 fuCOWRIE () {
-  rm -rf /data/cowrie/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/cowrie/*; fi
   mkdir -p /data/cowrie/log/tty/ /data/cowrie/downloads/ /data/cowrie/keys/ /data/cowrie/misc/
   chmod 760 /data/cowrie -R
   chown tpot:tpot /data/cowrie -R
@@ -29,7 +82,7 @@ fuCOWRIE () {
 
 # Let's create a function to clean up and prepare dionaea data
 fuDIONAEA () {
-  rm -rf /data/dionaea/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/dionaea/*; fi
   mkdir -p /data/dionaea/log /data/dionaea/bistreams /data/dionaea/binaries /data/dionaea/rtp /data/dionaea/roots/ftp /data/dionaea/roots/tftp /data/dionaea/roots/www /data/dionaea/roots/upnp
   chmod 760 /data/dionaea -R
   chown tpot:tpot /data/dionaea -R
@@ -37,7 +90,7 @@ fuDIONAEA () {
 
 # Let's create a function to clean up and prepare elasticpot data
 fuELASTICPOT () {
-  rm -rf /data/elasticpot/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elasticpot/*; fi
   mkdir -p /data/elasticpot/log
   chmod 760 /data/elasticpot -R
   chown tpot:tpot /data/elasticpot -R
@@ -47,7 +100,7 @@ fuELASTICPOT () {
 fuELK () {
   # ELK data will be kept for <= 90 days, check /etc/crontab for curator modification
   # ELK daemon log files will be removed
-  rm -rf /data/elk/log/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elk/log/*; fi
   mkdir -p /data/elk 
   chmod 760 /data/elk -R
   chown tpot:tpot /data/elk -R
@@ -55,16 +108,15 @@ fuELK () {
 
 # Let's create a function to clean up and prepare emobility data
 fuEMOBILITY () {
-  rm -rf /data/emobility/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/emobility/*; fi
-  rm /data/ews/emobility/ews.json
+  mkdir -p /data/emobility/log
-  mkdir -p /data/emobility/log /data/ews/emobility
   chmod 760 /data/emobility -R
   chown tpot:tpot /data/emobility -R
 }
 
 # Let's create a function to clean up and prepare glastopf data
 fuGLASTOPF () {
-  rm -rf /data/glastopf/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/glastopf/*; fi
   mkdir -p /data/glastopf
   chmod 760 /data/glastopf -R
   chown tpot:tpot /data/glastopf -R
@@ -72,7 +124,7 @@ fuGLASTOPF () {
 
 # Let's create a function to clean up and prepare honeytrap data
 fuHONEYTRAP () {
-  rm -rf /data/honeytrap/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/honeytrap/*; fi
   mkdir -p /data/honeytrap/log/ /data/honeytrap/attacks/ /data/honeytrap/downloads/
   chmod 760 /data/honeytrap/ -R
   chown tpot:tpot /data/honeytrap/ -R
@@ -80,7 +132,7 @@ fuHONEYTRAP () {
 
 # Let's create a function to clean up and prepare mailoney data
 fuMAILONEY () {
-  rm -rf /data/mailoney/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/mailoney/*; fi
   mkdir -p /data/mailoney/log/
   chmod 760 /data/mailoney/ -R
   chown tpot:tpot /data/mailoney/ -R
@@ -88,7 +140,7 @@ fuMAILONEY () {
 
 # Let's create a function to clean up and prepare maltrail data
 fuMALTRAIL () {
-  rm -rf /data/maltrail/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/maltrail/*; fi
   mkdir -p /data/maltrail/log/
   chmod 760 /data/maltrail/ -R
   chown tpot:tpot /data/maltrail/ -R
@@ -104,7 +156,7 @@ fuSPIDERFOOT () {
 
 # Let's create a function to clean up and prepare suricata data
 fuSURICATA () {
-  rm -rf /data/suricata/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/suricata/*; fi
   mkdir -p /data/suricata/log
   chmod 760 -R /data/suricata
   chown tpot:tpot -R /data/suricata
@@ -112,12 +164,37 @@ fuSURICATA () {
 
 # Let's create a function to clean up and prepare p0f data
 fuP0F () {
-  rm -rf /data/p0f/*
+  if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/p0f/*; fi
   mkdir -p /data/p0f/log
   chmod 760 -R /data/p0f
   chown tpot:tpot -R /data/p0f
 }
 
+
+# Avoid unwanted cleaning
+if [ "$myPERSISTENCE" = "" ];
+  then
+    echo $myRED"!!! WARNING !!! - This will delete ALL honeypot logs. "$myWHITE
+    while [ "$myQST" != "y" ] && [ "$myQST" != "n" ];
+      do
+        read -p "Continue? (y/n) " myQST
+    done
+    if [ "$myQST" = "n" ];
+      then
+        echo $myGREEN"Puuh! That was close! Aborting!"$myWHITE
+        exit
+    fi
+fi
+
+# Check persistence, if enabled compress and rotate logs
+if [ "$myPERSISTENCE" = "on" ];
+  then
+    echo "Persistence enabled, now rotating and compressing logs."
+    fuLOGROTATE
+  else
+    echo "Cleaning up and preparing data folders."
+fi
+
 fuCONPOT
 fuCOWRIE
 fuDIONAEA

installer/etc/tpot/logrotate/logrotate.conf

@@ -0,0 +1,35 @@
+/data/conpot/log/conpot.json
+/data/conpot/log/conpot.log
+/data/cowrie/log/cowrie.json
+/data/cowrie/log/cowrie-textlog.log
+/data/cowrie/log/lastlog.txt
+/data/cowrie/log/ttylogs.tgz
+/data/cowrie/downloads.tgz
+/data/dionaea/log/dionaea.json
+/data/dionaea/log/dionaea.sqlite
+/data/dionaea/bistreams.tgz
+/data/dionaea/binaries.tgz
+/data/dionaea/dionaea-errors.log
+/data/elasticpot/log/elasticpot.log
+/data/elk/log/*.log
+/data/emobility/log/centralsystem.log
+/data/emobility/log/centralsystemEWS.log
+/data/glastopf/log/glastopf.log
+/data/honeytrap/log/*.log
+/data/honeytrap/log/*.json
+/data/honeytrap/attacks.tgz
+/data/honeytrap/downloads.tgz
+/data/mailoney/log/commands.log
+/data/p0f/log/p0f.json
+/data/suricata/log/*.log
+/data/suricata/log/*.json
+{
+	su tpot tpot
+        copytruncate
+        create 760 tpot tpot
+	daily
+        missingok
+        notifempty
+	rotate 30
+	compress
+}

installer/etc/tpot/systemd/tpot.service

@@ -10,7 +10,7 @@ RestartSec=5
 # Get and set internal, external IP infos, but ignore errors
 ExecStartPre=-/usr/share/tpot/bin/updateip.sh
 
-# Clear state from /data
+# Clear state or if persistence is enabled rotate and compress logs from /data
 ExecStartPre=-/bin/bash -c '/usr/share/tpot/bin/clean.sh off'
 
 # Remove old containers, images and volumes