From 07c3f48894211c8d3785cb0b53d4e266eb316d23 Mon Sep 17 00:00:00 2001 From: Marco Ochse Date: Thu, 20 Jul 2017 20:25:49 +0000 Subject: [PATCH] compress and rotate logs if persistence enabled if persistence is enabled, log files, downloads, binaries, etc. will be compressed and rotated each start / stop of the t-pot service will account for a full rotation cycle if files are not empty basically the rotation will recycle logs after 30 days, unless the service is stopped / started manually which will cause for a shorter period --- installer/bin/clean.sh | 119 ++++++++++++++++---- installer/etc/tpot/logrotate/logrotate.conf | 35 ++++++ installer/etc/tpot/systemd/tpot.service | 2 +- 3 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 installer/etc/tpot/logrotate/logrotate.conf diff --git a/installer/bin/clean.sh b/installer/bin/clean.sh index f970dfda..453112b3 100755 --- a/installer/bin/clean.sh +++ b/installer/bin/clean.sh @@ -1,19 +1,72 @@ #!/bin/bash -# T-Pot Container Data Cleaner +# T-Pot Container Data Cleaner & Log Rotator + +# Set colors +myRED="" +myGREEN="" +myWHITE="" # Set persistence myPERSISTENCE=$1 -# Check persistence -if [ "$myPERSISTENCE" = "on" ]; - then - echo "### Persistence enabled, nothing to do." - exit -fi +# Let's create a function to check if folder is empty +fuEMPTY () { + local myFOLDER=$1 + +echo $(ls $myFOLDER | wc -l) +} + +# Let's create a function to rotate and compress logs +fuLOGROTATE () { + local mySTATUS="/etc/tpot/logrotate/status" + local myCONF="/etc/tpot/logrotate/logrotate.conf" + local myCOWRIETTYLOGS="/data/cowrie/log/tty/" + local myCOWRIETTYTGZ="/data/cowrie/log/ttylogs.tgz" + local myCOWRIEDL="/data/cowrie/downloads/" + local myCOWRIEDLTGZ="/data/cowrie/downloads.tgz" + local myDIONAEABI="/data/dionaea/bistreams/" + local myDIONAEABITGZ="/data/dionaea/bistreams.tgz" + local myDIONAEABIN="/data/dionaea/binaries/" + local myDIONAEABINTGZ="/data/dionaea/binaries.tgz" + local myHONEYTRAPATTACKS="/data/honeytrap/attacks/" + local myHONEYTRAPATTACKSTGZ="/data/honeytrap/attacks.tgz" + local myHONEYTRAPDL="/data/honeytrap/downloads/" + local myHONEYTRAPDLTGZ="/data/honeytrap/downloads.tgz" + +# Ensure correct permissions and ownerships for logrotate to run without issues +chmod 760 /data/ -R +chown tpot:tpot /data -R + +# Run logrotate with force (-f) first, so the status file can be written and race conditions (with tar) be avoided +logrotate -f -s $mySTATUS $myCONF + +# Compressing some folders first and rotate them later +if [ "$(fuEMPTY $myCOWRIETTYLOGS)" != "0" ]; then tar cvfz $myCOWRIETTYTGZ $myCOWRIETTYLOGS; fi +if [ "$(fuEMPTY $myCOWRIEDL)" != "0" ]; then tar cvfz $myCOWRIEDLTGZ $myCOWRIEDL; fi +if [ "$(fuEMPTY $myDIONAEABI)" != "0" ]; then tar cvfz $myDIONAEABITGZ $myDIONAEABI; fi +if [ "$(fuEMPTY $myDIONAEABIN)" != "0" ]; then tar cvfz $myDIONAEABINTGZ $myDIONAEABIN; fi +if [ "$(fuEMPTY $myHONEYTRAPATTACKS)" != "0" ]; then tar cvfz $myHONEYTRAPATTACKSTGZ $myHONEYTRAPATTACKS; fi +if [ "$(fuEMPTY $myHONEYTRAPDL)" != "0" ]; then tar cvfz $myHONEYTRAPDLTGZ $myHONEYTRAPDL; fi + +# Ensure correct permissions and ownership for previously created archives +chmod 760 $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ +chown tpot:tpot $myCOWRIETTYTGZ $myCOWRIEDLTGZ $myDIONAEABITGZ $myDIONAEABINTGZ $myHONEYTRAPATTACKSTGZ $myHONEYTRAPDLTGZ + +# Need to remove subfolders since too many files cause rm to exit with errors +rm -rf $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL + +# Recreate subfolders with correct permissions and ownership +mkdir -p $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL +chmod 760 $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL +chown tpot:tpot $myCOWRIETTYLOGS $myCOWRIEDL $myDIONAEABI $myDIONAEABIN $myHONEYTRAPATTACKS $myHONEYTRAPDL + +# Run logrotate again to account for previously created archives - DO NOT FORCE HERE! +logrotate -s $mySTATUS $myCONF +} # Let's create a function to clean up and prepare conpot data fuCONPOT () { - rm -rf /data/conpot/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/conpot/*; fi mkdir -p /data/conpot/log chmod 760 /data/conpot -R chown tpot:tpot /data/conpot -R @@ -21,7 +74,7 @@ fuCONPOT () { # Let's create a function to clean up and prepare cowrie data fuCOWRIE () { - rm -rf /data/cowrie/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/cowrie/*; fi mkdir -p /data/cowrie/log/tty/ /data/cowrie/downloads/ /data/cowrie/keys/ /data/cowrie/misc/ chmod 760 /data/cowrie -R chown tpot:tpot /data/cowrie -R @@ -29,7 +82,7 @@ fuCOWRIE () { # Let's create a function to clean up and prepare dionaea data fuDIONAEA () { - rm -rf /data/dionaea/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/dionaea/*; fi mkdir -p /data/dionaea/log /data/dionaea/bistreams /data/dionaea/binaries /data/dionaea/rtp /data/dionaea/roots/ftp /data/dionaea/roots/tftp /data/dionaea/roots/www /data/dionaea/roots/upnp chmod 760 /data/dionaea -R chown tpot:tpot /data/dionaea -R @@ -37,7 +90,7 @@ fuDIONAEA () { # Let's create a function to clean up and prepare elasticpot data fuELASTICPOT () { - rm -rf /data/elasticpot/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elasticpot/*; fi mkdir -p /data/elasticpot/log chmod 760 /data/elasticpot -R chown tpot:tpot /data/elasticpot -R @@ -47,7 +100,7 @@ fuELASTICPOT () { fuELK () { # ELK data will be kept for <= 90 days, check /etc/crontab for curator modification # ELK daemon log files will be removed - rm -rf /data/elk/log/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/elk/log/*; fi mkdir -p /data/elk chmod 760 /data/elk -R chown tpot:tpot /data/elk -R @@ -55,16 +108,15 @@ fuELK () { # Let's create a function to clean up and prepare emobility data fuEMOBILITY () { - rm -rf /data/emobility/* - rm /data/ews/emobility/ews.json - mkdir -p /data/emobility/log /data/ews/emobility + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/emobility/*; fi + mkdir -p /data/emobility/log chmod 760 /data/emobility -R chown tpot:tpot /data/emobility -R } # Let's create a function to clean up and prepare glastopf data fuGLASTOPF () { - rm -rf /data/glastopf/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/glastopf/*; fi mkdir -p /data/glastopf chmod 760 /data/glastopf -R chown tpot:tpot /data/glastopf -R @@ -72,7 +124,7 @@ fuGLASTOPF () { # Let's create a function to clean up and prepare honeytrap data fuHONEYTRAP () { - rm -rf /data/honeytrap/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/honeytrap/*; fi mkdir -p /data/honeytrap/log/ /data/honeytrap/attacks/ /data/honeytrap/downloads/ chmod 760 /data/honeytrap/ -R chown tpot:tpot /data/honeytrap/ -R @@ -80,7 +132,7 @@ fuHONEYTRAP () { # Let's create a function to clean up and prepare mailoney data fuMAILONEY () { - rm -rf /data/mailoney/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/mailoney/*; fi mkdir -p /data/mailoney/log/ chmod 760 /data/mailoney/ -R chown tpot:tpot /data/mailoney/ -R @@ -88,7 +140,7 @@ fuMAILONEY () { # Let's create a function to clean up and prepare maltrail data fuMALTRAIL () { - rm -rf /data/maltrail/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/maltrail/*; fi mkdir -p /data/maltrail/log/ chmod 760 /data/maltrail/ -R chown tpot:tpot /data/maltrail/ -R @@ -104,7 +156,7 @@ fuSPIDERFOOT () { # Let's create a function to clean up and prepare suricata data fuSURICATA () { - rm -rf /data/suricata/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/suricata/*; fi mkdir -p /data/suricata/log chmod 760 -R /data/suricata chown tpot:tpot -R /data/suricata @@ -112,12 +164,37 @@ fuSURICATA () { # Let's create a function to clean up and prepare p0f data fuP0F () { - rm -rf /data/p0f/* + if [ "$myPERSISTENCE" != "on" ]; then rm -rf /data/p0f/*; fi mkdir -p /data/p0f/log chmod 760 -R /data/p0f chown tpot:tpot -R /data/p0f } + +# Avoid unwanted cleaning +if [ "$myPERSISTENCE" = "" ]; + then + echo $myRED"!!! WARNING !!! - This will delete ALL honeypot logs. "$myWHITE + while [ "$myQST" != "y" ] && [ "$myQST" != "n" ]; + do + read -p "Continue? (y/n) " myQST + done + if [ "$myQST" = "n" ]; + then + echo $myGREEN"Puuh! That was close! Aborting!"$myWHITE + exit + fi +fi + +# Check persistence, if enabled compress and rotate logs +if [ "$myPERSISTENCE" = "on" ]; + then + echo "Persistence enabled, now rotating and compressing logs." + fuLOGROTATE + else + echo "Cleaning up and preparing data folders." +fi + fuCONPOT fuCOWRIE fuDIONAEA diff --git a/installer/etc/tpot/logrotate/logrotate.conf b/installer/etc/tpot/logrotate/logrotate.conf new file mode 100644 index 00000000..7b57608b --- /dev/null +++ b/installer/etc/tpot/logrotate/logrotate.conf @@ -0,0 +1,35 @@ +/data/conpot/log/conpot.json +/data/conpot/log/conpot.log +/data/cowrie/log/cowrie.json +/data/cowrie/log/cowrie-textlog.log +/data/cowrie/log/lastlog.txt +/data/cowrie/log/ttylogs.tgz +/data/cowrie/downloads.tgz +/data/dionaea/log/dionaea.json +/data/dionaea/log/dionaea.sqlite +/data/dionaea/bistreams.tgz +/data/dionaea/binaries.tgz +/data/dionaea/dionaea-errors.log +/data/elasticpot/log/elasticpot.log +/data/elk/log/*.log +/data/emobility/log/centralsystem.log +/data/emobility/log/centralsystemEWS.log +/data/glastopf/log/glastopf.log +/data/honeytrap/log/*.log +/data/honeytrap/log/*.json +/data/honeytrap/attacks.tgz +/data/honeytrap/downloads.tgz +/data/mailoney/log/commands.log +/data/p0f/log/p0f.json +/data/suricata/log/*.log +/data/suricata/log/*.json +{ + su tpot tpot + copytruncate + create 760 tpot tpot + daily + missingok + notifempty + rotate 30 + compress +} diff --git a/installer/etc/tpot/systemd/tpot.service b/installer/etc/tpot/systemd/tpot.service index bc612a29..ca911a99 100644 --- a/installer/etc/tpot/systemd/tpot.service +++ b/installer/etc/tpot/systemd/tpot.service @@ -10,7 +10,7 @@ RestartSec=5 # Get and set internal, external IP infos, but ignore errors ExecStartPre=-/usr/share/tpot/bin/updateip.sh -# Clear state from /data +# Clear state or if persistence is enabled rotate and compress logs from /data ExecStartPre=-/bin/bash -c '/usr/share/tpot/bin/clean.sh off' # Remove old containers, images and volumes