Merge 50beeef63a into d6b3e842fb

- makes logrotate cycles configurable, instead of static 30 days
- adjust .env / env.example for setting cycles
- adjust tpotinit dockerfile to include envsubst
- add logrotate.template
- add checks / validations

.env

@@ -40,6 +40,14 @@ TPOT_BLACKHOLE=DISABLED
 #       if you just do not need any of the logfiles.
 TPOT_PERSISTENCE=on
 
+# T-Pot Persistence Cycles
+#  <1-999>: Set the number of T-Pot restart cycles for logrotate.
+#           Be mindful of this setting as the logs will use up a lot of available disk space.
+#           In case the setting is invalid, T-Pot will default to 30 cycles.
+#  Remember to adjust the Elastic Search Lifecycle Policy (https://github.com/telekom-security/tpotce/?tab=readme-ov-file#log-persistence) 
+#  as this setting only accounts for the honeypot logs in the ~/tpotce/data folder.
+TPOT_PERSISTENCE_CYCLES=30
+
 # T-Pot Type
 #  HIVE: This is the default and offers everything to connect T-Pot sensors.
 #  SENSOR: This needs to be used when running a sensor. Be aware to adjust all other

README.md

@@ -677,7 +677,7 @@ All persistent log files from the honeypots, tools and T-Pot related services ar
 <br><br>
 
 ## Log Persistence
-All log data stored in the [T-Pot Data Folder](#t-pot-data-folder) will be persisted for 30 days by default.
+All log data is stored in the [T-Pot Data Folder](#t-pot-data-folder) and will be persisted for the number of cycles set for `TPOT_PERSISTENCE_CYCLES=<1-999>` in the T-Pot configuration file `~/tpotce/.env`. It defaults to 30.
 <br>
 Elasticsearch indices are handled by the `tpot` Index Lifecycle Policy which can be adjusted directly in Kibana (make sure to "Include managed system policies").
 ![IndexManagement1](doc/kibana_b.png)

docker/tpotinit/Dockerfile

@@ -13,6 +13,7 @@ RUN apk --no-cache -U upgrade && \
         conntrack-tools \
 		cracklib \
         curl \
+        envsubst \
         ethtool \
         figlet \
         git \
@@ -32,7 +33,7 @@ RUN apk --no-cache -U upgrade && \
 # Setup user, logrotate permissions
     addgroup -g 2000 tpot && \
     adduser -S -s /bin/ash -u 2000 -D -g 2000 tpot && \
-    chmod 0600 /opt/tpot/etc/logrotate/logrotate.conf && \
+    chmod 0600 /opt/tpot/etc/logrotate/logrotate.* && \
 #
 # Clean up
     apk del --purge git && \

docker/tpotinit/dist/bin/clean.sh

@@ -10,6 +10,9 @@ myPIGZ=$(which pigz)
 
 # Set persistence
 myPERSISTENCE=$1
+myPERSISTENCE_CYCLES=$2
+myPERSISTENCE_CYCLES="${myPERSISTENCE_CYCLES:=30}"
+export myPERSISTENCE_CYCLES
 
 # Let's create a function to check if folder is empty
 fuEMPTY () {
@@ -18,6 +21,15 @@ fuEMPTY () {
 echo $(ls $myFOLDER | wc -l)
 }
 
+# Let's create a function to setup logrotate config
+fuLOGROTATECONF () {
+  local myLOGROTATECONF="/opt/tpot/etc/logrotate/logrotate.conf"
+  local myLOGROTATETEMP="/opt/tpot/etc/logrotate/logrotate.template"
+  envsubst < $myLOGROTATETEMP > $myLOGROTATECONF
+  chown root:root $myLOGROTATECONF
+  chmod 0600 $myLOGROTATECONF
+}
+
 # Let's create a function to rotate and compress logs
 fuLOGROTATE () {
   local mySTATUS="/data/tpot/etc/logrotate/status"
@@ -43,6 +55,9 @@ fuLOGROTATE () {
   local myTANNERF="/data/tanner/files/"
   local myTANNERFTGZ="/data/tanner/files.tgz"
 
+# Setup logrotate config
+fuLOGROTATECONF
+
 # Ensure correct permissions and ownerships for logrotate to run without issues
 chmod 770 /data/ -R
 chown tpot:tpot /data -R
@@ -408,7 +423,7 @@ fi
 # Check persistence, if enabled compress and rotate logs
 if [ "$myPERSISTENCE" = "on" ];
   then
-    echo "Persistence enabled, now rotating and compressing logs."
+    echo "Persistence enabled for $myPERSISTENCE_CYCLES cycles, now rotating and compressing logs."
     fuLOGROTATE
 fi
 

docker/tpotinit/dist/entrypoint.sh

@@ -114,6 +114,20 @@ validate_ip_or_domain() {
     fi
 }
 
+# Function to validate if TPOT_PERSISTENCE_CYCLES is set and valid
+validate_tpot_persistence_cycles() {
+  # Check if the variable is unset, empty, not a number, or out of the valid range (1–999)
+  if [[ -z "$TPOT_PERSISTENCE_CYCLES" ]] || 
+     [[ ! "$TPOT_PERSISTENCE_CYCLES" =~ ^[0-9]+$ ]] || 
+     (( TPOT_PERSISTENCE_CYCLES < 1 )) || 
+     (( TPOT_PERSISTENCE_CYCLES > 999 )); then
+
+    # Set to default value
+    echo "WARNING! TPOT_PERSISTENCE_CYCLES is not set, invalid or out of bounds. Using default of 30 cycles."
+    TPOT_PERSISTENCE_CYCLES=30
+  fi
+}
+
 create_web_users() {
     echo
     echo "# Creating passwd files based on T-Pot .env config ..."
@@ -203,6 +217,9 @@ for var in TPOT_BLACKHOLE TPOT_PERSISTENCE TPOT_ATTACKMAP_TEXT TPOT_ATTACKMAP_TE
     validate_format "$var"
 done
 
+# Validate TPOT_PERSISTENCE_CYCLES
+validate_tpot_persistence_cycles
+
 if [ "${TPOT_TYPE}" == "HIVE" ];
   then
     # No $ for check_var
@@ -242,7 +259,7 @@ if [ -f "/data/uuid" ];
     echo
     echo "# Data folder is present, just cleaning up, please be patient ..."
     echo
-    /opt/tpot/bin/clean.sh "${TPOT_PERSISTENCE}"
+    /opt/tpot/bin/clean.sh "${TPOT_PERSISTENCE}" "${TPOT_PERSISTENCE_CYCLES}"
     echo
   else
     figlet "Setting up ..."

docker/tpotinit/dist/etc/logrotate/logrotate.template

@@ -0,0 +1,78 @@
+/data/adbhoney/log/*.json
+/data/adbhoney/log/*.log
+/data/beelzebub/log/*.json
+/data/ciscoasa/log/ciscoasa.log
+/data/citrixhoneypot/logs/server.log
+/data/conpot/log/conpot*.json
+/data/conpot/log/conpot*.log
+/data/cowrie/log/cowrie.json
+/data/cowrie/log/cowrie-textlog.log
+/data/cowrie/log/lastlog.txt
+/data/ddospot/log/*.log
+/data/dicompot/log/dicompot.log
+/data/dionaea/log/dionaea.json
+/data/dionaea/log/dionaea.sqlite
+/data/dionaea/dionaea-errors.log
+/data/elasticpot/log/elasticpot.log
+/data/elasticpot/log/elasticpot.json
+/data/elk/log/*.log
+/data/endlessh/log/*.log
+/data/fatt/log/fatt.log
+/data/galah/log/*.json
+/data/glutton/log/*.log
+/data/glutton/log/*.err
+/data/go-pot/log/*.json
+/data/h0neytr4p/log/*.json
+/data/hellpot/log/*.log
+/data/heralding/log/*.log
+/data/heralding/log/*.csv
+/data/heralding/log/*.json
+/data/honeyaml/log/*.log
+/data/honeypots/log/*.log
+/data/honeysap/log/*.log
+/data/honeytrap/log/*.log
+/data/honeytrap/log/*.json
+/data/ipphoney/log/*.json
+/data/log4pot/log/*.log
+/data/mailoney/log/*.log
+/data/medpot/log/*.log
+/data/miniprint/log/*.json
+/data/nginx/log/*.log
+/data/p0f/log/p0f.json
+/data/redishoneypot/log/*.log
+/data/sentrypeer/log/*.json
+/data/suricata/log/*.log
+/data/suricata/log/*.json
+/data/tanner/log/*.json
+/data/wordpot/log/*.log
+{
+	su tpot tpot
+        copytruncate
+        create 770 tpot tpot
+	daily
+        missingok
+        notifempty
+	rotate $myPERSISTENCE_CYCLES
+	compress
+        compresscmd /usr/bin/pigz
+}
+
+/data/adbhoney/downloads.tgz
+/data/cowrie/log/ttylogs.tgz
+/data/cowrie/downloads.tgz
+/data/dionaea/bistreams.tgz
+/data/dionaea/binaries.tgz
+/data/h0neytr4p/payloads.tgz
+/data/honeytrap/attacks.tgz
+/data/honeytrap/downloads.tgz
+/data/miniprint/uploads.tgz
+/data/tanner/files.tgz
+{
+	su tpot tpot
+        copytruncate
+        create 770 tpot tpot
+	daily
+        missingok
+        notifempty
+	rotate $myPERSISTENCE_CYCLES
+}

env.example

@@ -40,6 +40,14 @@ TPOT_BLACKHOLE=DISABLED
 #       if you just do not need any of the logfiles.
 TPOT_PERSISTENCE=on
 
+# T-Pot Persistence Cycles
+#  <1-999>: Set the number of T-Pot restart cycles for logrotate.
+#           Be mindful of this setting as the logs will use up a lot of available disk space.
+#           In case the setting is invalid, T-Pot will default to 30 cycles.
+#  Remember to adjust the Elastic Search Lifecycle Policy (https://github.com/telekom-security/tpotce/?tab=readme-ov-file#log-persistence) 
+#  as this setting only accounts for the honeypot logs in the ~/tpotce/data folder.
+TPOT_PERSISTENCE_CYCLES=30
+
 # T-Pot Type
 #  HIVE: This is the default and offers everything to connect T-Pot sensors.
 #  SENSOR: This needs to be used when running a sensor. Be aware to adjust all other

install.sh

@@ -1,5 +1,67 @@
 #!/usr/bin/env bash
 
+
+print_help() {
+  echo "Usage: $0 [-s y|n] [-t h|s|l|i|m|t] -u <webuser name> -p <password for web user>"
+  echo "  -s: yes or no (optional)"
+  echo "  -t: h (host),s (sensor), l (llm), i(mini),m(mobile),t(tarpit) (optional)"
+  echo "  -u: web username (optional)"
+  echo "  -p: password for web user (optional)"
+  exit 1
+}
+
+validate_s() {
+  if [[ -n "$myQST" ]]; then
+    if [[ "$myQST" =~ ^[yYnN]$ ]]; then
+      return 1 # Valid
+    else
+      print_help
+    fi
+  else
+    print_help
+  fi
+
+}
+
+validate_t() {
+  if [[ -n "$myTPOT_TYPE" ]]; then
+    if [[ "$myTPOT_TYPE" =~ ^[hslimtHSLIMT]$ ]]; then
+      return 1 # Valid
+    else
+      print_help
+    fi
+  else
+    print_help
+  fi
+
+}
+
+while getopts ":s:t:u:p:" opt; do
+  case "$opt" in
+  s)
+    myQST="${OPTARG}"
+    validate_s
+    ;;
+  t)
+    myTPOT_TYPE="${OPTARG}"
+    validate_t
+    ;;
+  u)
+    export myWEB_USER="${OPTARG}"
+    ;;
+  p)
+    export myWEB_PW="${OPTARG}"
+    ;;
+  :)
+    echo "Option -${OPTARG} requires an argument."
+    print_help
+    exit 1
+    ;;
+  \?)
+    print_help
+    ;;
+  esac
+done
 myINSTALL_NOTIFICATION="### Now installing required packages ..."
 myUSER=$(whoami)
 myTPOT_CONF_FILE="/home/${myUSER}/tpotce/.env"
@@ -43,12 +105,13 @@ echo "$myINSTALLER"
 echo
 echo
 echo "### This script will now install T-Pot and all of its dependencies."
-while [ "${myQST}" != "y" ] && [ "${myQST}" != "n" ];
-  do
+if [[ -z "$myQST" ]]; then
+  while [ "${myQST}" != "y" ] && [ "${myQST}" != "n" ]; do
     echo
     read -p "### Install? (y/n) " myQST
     echo
   done
+fi
 if [ "${myQST}" = "n" ];
   then
     echo
@@ -183,7 +246,10 @@ echo "###            Feed data endlessly to attackers, bots and scanners."
 echo "###            Also runs a Denial of Service Honeypot (ddospot)."
 echo
 while true; do
-  read -p "### Install Type? (h/s/l/i/m/t) " myTPOT_TYPE
+  if [[ -z "$myTPOT_TYPE" ]]; then
+    read -p "### Install Type? (h/s/l/i/m/t) " myTPOT_TYPE
+  fi  
+  
   case "${myTPOT_TYPE}" in
     h|H)
       echo
@@ -234,75 +300,71 @@ done
 if [ "${myTPOT_TYPE}" == "HIVE" ];
   # If T-Pot Type is HIVE ask for WebUI username and password
   then
-	# Preparing web user for T-Pot
-	echo
-	echo "### T-Pot User Configuration ..."
-	echo
-	# Asking for web user name
-	myWEB_USER=""
-	while [ 1 != 2 ];
-	  do
-	    myOK=""
-	    read -rp "### Enter your web user name: " myWEB_USER
-	    myWEB_USER=$(echo $myWEB_USER | tr -cd "[:alnum:]_.-")
-	    echo "### Your username is: ${myWEB_USER}"
-	    while [[ ! "${myOK}" =~ [YyNn] ]];
-	      do
-	        read -rp "### Is this correct? (y/n) " myOK
-	      done
-	    if [[ "${myOK}" =~ [Yy] ]] && [ "$myWEB_USER" != "" ];
-	      then
-	        break
-	      else
-	        echo
-	    fi
-	  done
+  # Preparing web user for T-Pot
+  echo
+  echo "### T-Pot User Configuration ..."
+  echo
+  # Asking for web user name
+  if [[ -z "$myWEB_USER" ]]; then
+    myWEB_USER=""
+    while [ 1 != 2 ]; do
+      myOK=""
+      read -rp "### Enter your web user name: " myWEB_USER
+      myWEB_USER=$(echo $myWEB_USER | tr -cd "[:alnum:]_.-")
+      echo "### Your username is: ${myWEB_USER}"
+      while [[ ! "${myOK}" =~ [YyNn] ]]; do    
+        read -rp "### Is this correct? (y/n) " myOK
+      done
+      if [[ "${myOK}" =~ [Yy] ]] && [ "$myWEB_USER" != "" ]; then
+        break
+      else
+        echo
+      fi
+    done
+  fi
 
-	# Asking for web user password
-	myWEB_PW="pass1"
-	myWEB_PW2="pass2"
-	mySECURE=0
-	myOK=""
-	while [ "${myWEB_PW}" != "${myWEB_PW2}"  ] && [ "${mySECURE}" == "0" ]
-	  do
-	    echo
-	    while [ "${myWEB_PW}" == "pass1"  ] || [ "${myWEB_PW}" == "" ]
-	      do
-	        read -rsp "### Enter password for your web user: " myWEB_PW
-	        echo
-	      done
-	    read -rsp "### Repeat password you your web user: " myWEB_PW2
-	    echo
-	    if [ "${myWEB_PW}" != "${myWEB_PW2}" ];
-	      then
-	        echo "### Passwords do not match."
-	        myWEB_PW="pass1"
-	        myWEB_PW2="pass2"
-	    fi
-	    mySECURE=$(printf "%s" "$myWEB_PW" | /usr/sbin/cracklib-check | grep -c "OK")
-	    if [ "$mySECURE" == "0" ] && [ "$myWEB_PW" == "$myWEB_PW2" ];
-	      then
-	        while [[ ! "${myOK}" =~ [YyNn] ]];
-	          do
-	            read -rp "### Keep insecure password? (y/n) " myOK
-	          done
-	        if [[ "${myOK}" =~ [Nn] ]] || [ "$myWEB_PW" == "" ];
-	          then
-	            myWEB_PW="pass1"
-	            myWEB_PW2="pass2"
-	            mySECURE=0
-	            myOK=""
-	        fi
-	    fi
-	done
+  # Asking for web user password
+  if [[ -z "$myWEB_PW" ]]; then
+    myWEB_PW="pass1"
+    myWEB_PW2="pass2"
+    mySECURE=0
+    myOK=""
+    while [ "${myWEB_PW}" != "${myWEB_PW2}" ] && [ "${mySECURE}" == "0" ]; do
+      echo
+      while [ "${myWEB_PW}" == "pass1" ] || [ "${myWEB_PW}" == "" ]; do
+        read -rsp "### Enter password for your web user: " myWEB_PW
+        echo
+      done
+      read -rsp "### Repeat password you your web user: " myWEB_PW2
+      echo
+      if [ "${myWEB_PW}" != "${myWEB_PW2}" ]; then
+        echo "### Passwords do not match."
+        myWEB_PW="pass1"
+        myWEB_PW2="pass2"
+      fi
+      mySECURE=$(printf "%s" "$myWEB_PW" | /usr/sbin/cracklib-check | grep -c "OK")
+      if [ "$mySECURE" == "0" ] && [ "$myWEB_PW" == "$myWEB_PW2" ]; then
+        while [[ ! "${myOK}" =~ [YyNn] ]]; do
+          read -rp "### Keep insecure password? (y/n) " myOK
+        done
+        if [[ "${myOK}" =~ [Nn] ]] || [ "$myWEB_PW" == "" ]; then
+          myWEB_PW="pass1"
+          myWEB_PW2="pass2"
+          mySECURE=0
+          myOK=""
+        fi
+      fi
+    done
+  fi
 
-	# Write username and password to T-Pot config file
-	echo "### Creating base64 encoded htpasswd username and password for T-Pot config file: ${myTPOT_CONF_FILE}"
-	myWEB_USER_ENC=$(htpasswd -b -n "${myWEB_USER}" "${myWEB_PW}")
+
+  # Write username and password to T-Pot config file
+  echo "### Creating base64 encoded htpasswd username and password for T-Pot config file: ${myTPOT_CONF_FILE}"
+  myWEB_USER_ENC=$(htpasswd -b -n "${myWEB_USER}" "${myWEB_PW}")
     myWEB_USER_ENC_B64=$(echo -n "${myWEB_USER_ENC}" | base64 -w0)
     
-	echo
-	sed -i "s|^WEB_USER=.*|WEB_USER=${myWEB_USER_ENC_B64}|" ${myTPOT_CONF_FILE}
+  echo
+  sed -i "s|^WEB_USER=.*|WEB_USER=${myWEB_USER_ENC_B64}|" ${myTPOT_CONF_FILE}
 fi
 
 # Pull docker images