Merge 50beeef63a into d6b3e842fb

- makes logrotate cycles configurable, instead of static 30 days
- adjust .env / env.example for setting cycles
- adjust tpotinit dockerfile to include envsubst
- add logrotate.template
- add checks / validations

.env

@@ -40,6 +40,14 @@ TPOT_BLACKHOLE=DISABLED
 #       if you just do not need any of the logfiles.
 TPOT_PERSISTENCE=on
 
+# T-Pot Persistence Cycles
+#  <1-999>: Set the number of T-Pot restart cycles for logrotate.
+#           Be mindful of this setting as the logs will use up a lot of available disk space.
+#           In case the setting is invalid, T-Pot will default to 30 cycles.
+#  Remember to adjust the Elastic Search Lifecycle Policy (https://github.com/telekom-security/tpotce/?tab=readme-ov-file#log-persistence) 
+#  as this setting only accounts for the honeypot logs in the ~/tpotce/data folder.
+TPOT_PERSISTENCE_CYCLES=30
+
 # T-Pot Type
 #  HIVE: This is the default and offers everything to connect T-Pot sensors.
 #  SENSOR: This needs to be used when running a sensor. Be aware to adjust all other

README.md

@@ -677,7 +677,7 @@ All persistent log files from the honeypots, tools and T-Pot related services ar
 <br><br>
 
 ## Log Persistence
-All log data stored in the [T-Pot Data Folder](#t-pot-data-folder) will be persisted for 30 days by default.
+All log data is stored in the [T-Pot Data Folder](#t-pot-data-folder) and will be persisted for the number of cycles set for `TPOT_PERSISTENCE_CYCLES=<1-999>` in the T-Pot configuration file `~/tpotce/.env`. It defaults to 30.
 <br>
 Elasticsearch indices are handled by the `tpot` Index Lifecycle Policy which can be adjusted directly in Kibana (make sure to "Include managed system policies").
 ![IndexManagement1](doc/kibana_b.png)

docker/tpotinit/Dockerfile

@@ -13,6 +13,7 @@ RUN apk --no-cache -U upgrade && \
         conntrack-tools \
 		cracklib \
         curl \
+        envsubst \
         ethtool \
         figlet \
         git \
@@ -32,7 +33,7 @@ RUN apk --no-cache -U upgrade && \
 # Setup user, logrotate permissions
     addgroup -g 2000 tpot && \
     adduser -S -s /bin/ash -u 2000 -D -g 2000 tpot && \
-    chmod 0600 /opt/tpot/etc/logrotate/logrotate.conf && \
+    chmod 0600 /opt/tpot/etc/logrotate/logrotate.* && \
 #
 # Clean up
     apk del --purge git && \

docker/tpotinit/dist/bin/clean.sh

@@ -10,6 +10,9 @@ myPIGZ=$(which pigz)
 
 # Set persistence
 myPERSISTENCE=$1
+myPERSISTENCE_CYCLES=$2
+myPERSISTENCE_CYCLES="${myPERSISTENCE_CYCLES:=30}"
+export myPERSISTENCE_CYCLES
 
 # Let's create a function to check if folder is empty
 fuEMPTY () {
@@ -18,6 +21,15 @@ fuEMPTY () {
 echo $(ls $myFOLDER | wc -l)
 }
 
+# Let's create a function to setup logrotate config
+fuLOGROTATECONF () {
+  local myLOGROTATECONF="/opt/tpot/etc/logrotate/logrotate.conf"
+  local myLOGROTATETEMP="/opt/tpot/etc/logrotate/logrotate.template"
+  envsubst < $myLOGROTATETEMP > $myLOGROTATECONF
+  chown root:root $myLOGROTATECONF
+  chmod 0600 $myLOGROTATECONF
+}
+
 # Let's create a function to rotate and compress logs
 fuLOGROTATE () {
   local mySTATUS="/data/tpot/etc/logrotate/status"
@@ -43,6 +55,9 @@ fuLOGROTATE () {
   local myTANNERF="/data/tanner/files/"
   local myTANNERFTGZ="/data/tanner/files.tgz"
 
+# Setup logrotate config
+fuLOGROTATECONF
+
 # Ensure correct permissions and ownerships for logrotate to run without issues
 chmod 770 /data/ -R
 chown tpot:tpot /data -R
@@ -408,7 +423,7 @@ fi
 # Check persistence, if enabled compress and rotate logs
 if [ "$myPERSISTENCE" = "on" ];
   then
-    echo "Persistence enabled, now rotating and compressing logs."
+    echo "Persistence enabled for $myPERSISTENCE_CYCLES cycles, now rotating and compressing logs."
     fuLOGROTATE
 fi
 

docker/tpotinit/dist/entrypoint.sh

@@ -114,6 +114,20 @@ validate_ip_or_domain() {
     fi
 }
 
+# Function to validate if TPOT_PERSISTENCE_CYCLES is set and valid
+validate_tpot_persistence_cycles() {
+  # Check if the variable is unset, empty, not a number, or out of the valid range (1–999)
+  if [[ -z "$TPOT_PERSISTENCE_CYCLES" ]] || 
+     [[ ! "$TPOT_PERSISTENCE_CYCLES" =~ ^[0-9]+$ ]] || 
+     (( TPOT_PERSISTENCE_CYCLES < 1 )) || 
+     (( TPOT_PERSISTENCE_CYCLES > 999 )); then
+
+    # Set to default value
+    echo "WARNING! TPOT_PERSISTENCE_CYCLES is not set, invalid or out of bounds. Using default of 30 cycles."
+    TPOT_PERSISTENCE_CYCLES=30
+  fi
+}
+
 create_web_users() {
     echo
     echo "# Creating passwd files based on T-Pot .env config ..."
@@ -203,6 +217,9 @@ for var in TPOT_BLACKHOLE TPOT_PERSISTENCE TPOT_ATTACKMAP_TEXT TPOT_ATTACKMAP_TE
     validate_format "$var"
 done
 
+# Validate TPOT_PERSISTENCE_CYCLES
+validate_tpot_persistence_cycles
+
 if [ "${TPOT_TYPE}" == "HIVE" ];
   then
     # No $ for check_var
@@ -242,7 +259,7 @@ if [ -f "/data/uuid" ];
     echo
     echo "# Data folder is present, just cleaning up, please be patient ..."
     echo
-    /opt/tpot/bin/clean.sh "${TPOT_PERSISTENCE}"
+    /opt/tpot/bin/clean.sh "${TPOT_PERSISTENCE}" "${TPOT_PERSISTENCE_CYCLES}"
     echo
   else
     figlet "Setting up ..."

docker/tpotinit/dist/etc/logrotate/logrotate.template

@@ -0,0 +1,78 @@
+/data/adbhoney/log/*.json
+/data/adbhoney/log/*.log
+/data/beelzebub/log/*.json
+/data/ciscoasa/log/ciscoasa.log
+/data/citrixhoneypot/logs/server.log
+/data/conpot/log/conpot*.json
+/data/conpot/log/conpot*.log
+/data/cowrie/log/cowrie.json
+/data/cowrie/log/cowrie-textlog.log
+/data/cowrie/log/lastlog.txt
+/data/ddospot/log/*.log
+/data/dicompot/log/dicompot.log
+/data/dionaea/log/dionaea.json
+/data/dionaea/log/dionaea.sqlite
+/data/dionaea/dionaea-errors.log
+/data/elasticpot/log/elasticpot.log
+/data/elasticpot/log/elasticpot.json
+/data/elk/log/*.log
+/data/endlessh/log/*.log
+/data/fatt/log/fatt.log
+/data/galah/log/*.json
+/data/glutton/log/*.log
+/data/glutton/log/*.err
+/data/go-pot/log/*.json
+/data/h0neytr4p/log/*.json
+/data/hellpot/log/*.log
+/data/heralding/log/*.log
+/data/heralding/log/*.csv
+/data/heralding/log/*.json
+/data/honeyaml/log/*.log
+/data/honeypots/log/*.log
+/data/honeysap/log/*.log
+/data/honeytrap/log/*.log
+/data/honeytrap/log/*.json
+/data/ipphoney/log/*.json
+/data/log4pot/log/*.log
+/data/mailoney/log/*.log
+/data/medpot/log/*.log
+/data/miniprint/log/*.json
+/data/nginx/log/*.log
+/data/p0f/log/p0f.json
+/data/redishoneypot/log/*.log
+/data/sentrypeer/log/*.json
+/data/suricata/log/*.log
+/data/suricata/log/*.json
+/data/tanner/log/*.json
+/data/wordpot/log/*.log
+{
+	su tpot tpot
+        copytruncate
+        create 770 tpot tpot
+	daily
+        missingok
+        notifempty
+	rotate $myPERSISTENCE_CYCLES
+	compress
+        compresscmd /usr/bin/pigz
+}
+
+/data/adbhoney/downloads.tgz
+/data/cowrie/log/ttylogs.tgz
+/data/cowrie/downloads.tgz
+/data/dionaea/bistreams.tgz
+/data/dionaea/binaries.tgz
+/data/h0neytr4p/payloads.tgz
+/data/honeytrap/attacks.tgz
+/data/honeytrap/downloads.tgz
+/data/miniprint/uploads.tgz
+/data/tanner/files.tgz
+{
+	su tpot tpot
+        copytruncate
+        create 770 tpot tpot
+	daily
+        missingok
+        notifempty
+	rotate $myPERSISTENCE_CYCLES
+}

env.example

@@ -40,6 +40,14 @@ TPOT_BLACKHOLE=DISABLED
 #       if you just do not need any of the logfiles.
 TPOT_PERSISTENCE=on
 
+# T-Pot Persistence Cycles
+#  <1-999>: Set the number of T-Pot restart cycles for logrotate.
+#           Be mindful of this setting as the logs will use up a lot of available disk space.
+#           In case the setting is invalid, T-Pot will default to 30 cycles.
+#  Remember to adjust the Elastic Search Lifecycle Policy (https://github.com/telekom-security/tpotce/?tab=readme-ov-file#log-persistence) 
+#  as this setting only accounts for the honeypot logs in the ~/tpotce/data folder.
+TPOT_PERSISTENCE_CYCLES=30
+
 # T-Pot Type
 #  HIVE: This is the default and offers everything to connect T-Pot sensors.
 #  SENSOR: This needs to be used when running a sensor. Be aware to adjust all other

install.sh

@@ -1,5 +1,67 @@
 #!/usr/bin/env bash
 
+
+print_help() {
+  echo "Usage: $0 [-s y|n] [-t h|s|l|i|m|t] -u <webuser name> -p <password for web user>"
+  echo "  -s: yes or no (optional)"
+  echo "  -t: h (host),s (sensor), l (llm), i(mini),m(mobile),t(tarpit) (optional)"
+  echo "  -u: web username (optional)"
+  echo "  -p: password for web user (optional)"
+  exit 1
+}
+
+validate_s() {
+  if [[ -n "$myQST" ]]; then
+    if [[ "$myQST" =~ ^[yYnN]$ ]]; then
+      return 1 # Valid
+    else
+      print_help
+    fi
+  else
+    print_help
+  fi
+
+}
+
+validate_t() {
+  if [[ -n "$myTPOT_TYPE" ]]; then
+    if [[ "$myTPOT_TYPE" =~ ^[hslimtHSLIMT]$ ]]; then
+      return 1 # Valid
+    else
+      print_help
+    fi
+  else
+    print_help
+  fi
+
+}
+
+while getopts ":s:t:u:p:" opt; do
+  case "$opt" in
+  s)
+    myQST="${OPTARG}"
+    validate_s
+    ;;
+  t)
+    myTPOT_TYPE="${OPTARG}"
+    validate_t
+    ;;
+  u)
+    export myWEB_USER="${OPTARG}"
+    ;;
+  p)
+    export myWEB_PW="${OPTARG}"
+    ;;
+  :)
+    echo "Option -${OPTARG} requires an argument."
+    print_help
+    exit 1
+    ;;
+  \?)
+    print_help
+    ;;
+  esac
+done
 myINSTALL_NOTIFICATION="### Now installing required packages ..."
 myUSER=$(whoami)
 myTPOT_CONF_FILE="/home/${myUSER}/tpotce/.env"
@@ -43,12 +105,13 @@ echo "$myINSTALLER"
 echo
 echo
 echo "### This script will now install T-Pot and all of its dependencies."
-while [ "${myQST}" != "y" ] && [ "${myQST}" != "n" ];
+if [[ -z "$myQST" ]]; then
-  do
+  while [ "${myQST}" != "y" ] && [ "${myQST}" != "n" ]; do
     echo
     read -p "### Install? (y/n) " myQST
     echo
   done
+fi
 if [ "${myQST}" = "n" ];
   then
     echo
@@ -183,7 +246,10 @@ echo "###            Feed data endlessly to attackers, bots and scanners."
 echo "###            Also runs a Denial of Service Honeypot (ddospot)."
 echo
 while true; do
+  if [[ -z "$myTPOT_TYPE" ]]; then
     read -p "### Install Type? (h/s/l/i/m/t) " myTPOT_TYPE
+  fi  
+  
   case "${myTPOT_TYPE}" in
     h|H)
       echo
@@ -239,55 +305,49 @@ if [ "${myTPOT_TYPE}" == "HIVE" ];
   echo "### T-Pot User Configuration ..."
   echo
   # Asking for web user name
+  if [[ -z "$myWEB_USER" ]]; then
     myWEB_USER=""
-	while [ 1 != 2 ];
+    while [ 1 != 2 ]; do
-	  do
       myOK=""
       read -rp "### Enter your web user name: " myWEB_USER
       myWEB_USER=$(echo $myWEB_USER | tr -cd "[:alnum:]_.-")
       echo "### Your username is: ${myWEB_USER}"
-	    while [[ ! "${myOK}" =~ [YyNn] ]];
+      while [[ ! "${myOK}" =~ [YyNn] ]]; do    
-	      do
         read -rp "### Is this correct? (y/n) " myOK
       done
-	    if [[ "${myOK}" =~ [Yy] ]] && [ "$myWEB_USER" != "" ];
+      if [[ "${myOK}" =~ [Yy] ]] && [ "$myWEB_USER" != "" ]; then
-	      then
         break
       else
         echo
       fi
     done
+  fi
 
   # Asking for web user password
+  if [[ -z "$myWEB_PW" ]]; then
     myWEB_PW="pass1"
     myWEB_PW2="pass2"
     mySECURE=0
     myOK=""
-	while [ "${myWEB_PW}" != "${myWEB_PW2}"  ] && [ "${mySECURE}" == "0" ]
+    while [ "${myWEB_PW}" != "${myWEB_PW2}" ] && [ "${mySECURE}" == "0" ]; do
-	  do
       echo
-	    while [ "${myWEB_PW}" == "pass1"  ] || [ "${myWEB_PW}" == "" ]
+      while [ "${myWEB_PW}" == "pass1" ] || [ "${myWEB_PW}" == "" ]; do
-	      do
         read -rsp "### Enter password for your web user: " myWEB_PW
         echo
       done
       read -rsp "### Repeat password you your web user: " myWEB_PW2
       echo
-	    if [ "${myWEB_PW}" != "${myWEB_PW2}" ];
+      if [ "${myWEB_PW}" != "${myWEB_PW2}" ]; then
-	      then
         echo "### Passwords do not match."
         myWEB_PW="pass1"
         myWEB_PW2="pass2"
       fi
       mySECURE=$(printf "%s" "$myWEB_PW" | /usr/sbin/cracklib-check | grep -c "OK")
-	    if [ "$mySECURE" == "0" ] && [ "$myWEB_PW" == "$myWEB_PW2" ];
+      if [ "$mySECURE" == "0" ] && [ "$myWEB_PW" == "$myWEB_PW2" ]; then
-	      then
+        while [[ ! "${myOK}" =~ [YyNn] ]]; do
-	        while [[ ! "${myOK}" =~ [YyNn] ]];
-	          do
           read -rp "### Keep insecure password? (y/n) " myOK
         done
-	        if [[ "${myOK}" =~ [Nn] ]] || [ "$myWEB_PW" == "" ];
+        if [[ "${myOK}" =~ [Nn] ]] || [ "$myWEB_PW" == "" ]; then
-	          then
           myWEB_PW="pass1"
           myWEB_PW2="pass2"
           mySECURE=0
@@ -295,6 +355,8 @@ if [ "${myTPOT_TYPE}" == "HIVE" ];
         fi
       fi
     done
+  fi
+
 
   # Write username and password to T-Pot config file
   echo "### Creating base64 encoded htpasswd username and password for T-Pot config file: ${myTPOT_CONF_FILE}"