From 2c7c5b656d78bf929854fde2cf6e399055cc626f Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Wed, 3 Apr 2019 13:39:40 +0000 Subject: [PATCH 1/5] apt-fast, fix netselect-apt --- iso/installer/install.sh | 28 ++++++++++++++++++++-------- iso/preseed/tpot.seed | 2 +- update.sh | 12 +++++++----- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/iso/installer/install.sh b/iso/installer/install.sh index 132f775f..96b1eb57 100755 --- a/iso/installer/install.sh +++ b/iso/installer/install.sh @@ -13,8 +13,8 @@ myTPOTCOMPOSE="/opt/tpot/etc/tpot.yml" myLSB_STABLE_SUPPORTED="stretch" myLSB_TESTING_SUPPORTED="sid" myREMOTESITES="https://hub.docker.com https://github.com https://pypi.python.org https://debian.org" -myPREINSTALLPACKAGES="apache2-utils curl dialog figlet grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet" -myINSTALLPACKAGES="apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose dstat ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass prips software-properties-common syslinux psmisc pv python-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" +myPREINSTALLPACKAGES="aria2 apache2-utils curl dialog figlet grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet" +myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose dstat ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass prips software-properties-common syslinux psmisc pv python-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" myINFO="\ ######################################## ### T-Pot Installer for Debian (Sid) ### @@ -213,6 +213,8 @@ fi # If not present install them function fuCHECKPACKAGES { export DEBIAN_FRONTEND=noninteractive + echo "### Installing apt-fast" + /bin/bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)" echo -n "### Checking for installer dependencies: " local myPACKAGES="$1" for myDEPS in $myPACKAGES; @@ -221,8 +223,8 @@ function fuCHECKPACKAGES { if [ "$myOK" != "ok" ]; then echo "[ NOW INSTALLING ]" - apt-get update -y - apt-get install -y $myPACKAGES + apt-fast update -y + apt-fast install -y $myPACKAGES break fi done @@ -268,21 +270,31 @@ function fuGET_DEPS { echo "### Determine fastest mirror for your location." echo netselect-apt -n -a amd64 unstable && cp sources.list /etc/apt/ + mySOURCESCHECK=$(cat /etc/apt/sources.list | grep -c unstable) + if [ "$mySOURCESCHECK" == "0" ] + then + echo "### Automatic mirror selection failed, using main mirror." + # Point to Debian (Sid, unstable) + tee /etc/apt/sources.list < Date: Thu, 4 Apr 2019 06:55:59 +0000 Subject: [PATCH 2/5] tweaking --- iso/installer/install.sh | 10 +++++----- update.sh | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/iso/installer/install.sh b/iso/installer/install.sh index 96b1eb57..70db8edd 100755 --- a/iso/installer/install.sh +++ b/iso/installer/install.sh @@ -165,7 +165,7 @@ myCRONJOBS=" 27 3 * * * root systemctl stop tpot && docker stop \$(docker ps -aq) || docker rm \$(docker ps -aq) || reboot # Check for updated packages every sunday, upgrade and reboot -27 16 * * 0 root apt-get autoclean -y && apt-get autoremove -y && apt-get update -y && apt-get upgrade -y && sleep 10 && reboot +27 16 * * 0 root apt-fast autoclean -y && apt-fast autoremove -y && apt-fast update -y && apt-fast upgrade -y && sleep 10 && reboot " myROOTPROMPT='PS1="\[\033[38;5;8m\][\[$(tput sgr0)\]\[\033[38;5;1m\]\u\[$(tput sgr0)\]\[\033[38;5;6m\]@\[$(tput sgr0)\]\[\033[38;5;4m\]\h\[$(tput sgr0)\]\[\033[38;5;6m\]:\[$(tput sgr0)\]\[\033[38;5;5m\]\w\[$(tput sgr0)\]\[\033[38;5;8m\]]\[$(tput sgr0)\]\[\033[38;5;1m\]\\$\[$(tput sgr0)\]\[\033[38;5;15m\] \[$(tput sgr0)\]"' myUSERPROMPT='PS1="\[\033[38;5;8m\][\[$(tput sgr0)\]\[\033[38;5;2m\]\u\[$(tput sgr0)\]\[\033[38;5;6m\]@\[$(tput sgr0)\]\[\033[38;5;4m\]\h\[$(tput sgr0)\]\[\033[38;5;6m\]:\[$(tput sgr0)\]\[\033[38;5;5m\]\w\[$(tput sgr0)\]\[\033[38;5;8m\]]\[$(tput sgr0)\]\[\033[38;5;2m\]\\$\[$(tput sgr0)\]\[\033[38;5;15m\] \[$(tput sgr0)\]"' @@ -296,8 +296,8 @@ EOF echo apt-fast -y install $myINSTALLPACKAGES # Remove exim4 - apt-get -y purge exim4-base mailutils - apt-get -y autoremove + apt-fast -y purge exim4-base mailutils + apt-fast -y autoremove apt-mark hold exim4-base mailutils } @@ -820,8 +820,8 @@ fuBANNER "Update IP" # Let's clean up apt fuBANNER "Clean up" -apt-get autoclean -y -apt-get autoremove -y +apt-fast autoclean -y +apt-fast autoremove -y # Final steps cp /opt/tpot/host/etc/rc.local /etc/rc.local && \ diff --git a/update.sh b/update.sh index b78eced6..54080aa7 100755 --- a/update.sh +++ b/update.sh @@ -181,8 +181,8 @@ echo "### Installing apt-fast" local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose dstat ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass prips software-properties-common syslinux psmisc pv python-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" echo "### Now upgrading packages ..." dpkg --configure -a -apt-get -y autoclean -apt-get -y autoremove +apt-fast -y autoclean +apt-fast -y autoremove apt-fast update apt-fast -y install $myPACKAGES @@ -195,7 +195,7 @@ npm install "https://github.com/taskrabbit/elasticsearch-dump" -g pip install --upgrade pip hash -r pip install --upgrade elasticsearch-curator yq -apt-get -y purge exim4-base mailutils +apt-fast -y purge exim4-base mailutils apt-mark hold exim4-base mailutils echo From 3edc0a721805a116cf0d0274966fbce129a9ae78 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 4 Apr 2019 08:26:33 +0000 Subject: [PATCH 3/5] tweaking --- CHANGELOG.md | 7 +++++++ README.md | 5 +++++ iso/installer/install.sh | 2 +- iso/preseed/tpot.seed | 2 +- update.sh | 2 +- 5 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..c4267cd4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## 20190404 +- **Fix #332** + - If T-Pot, opposed to the requirements, does not have full internet access netselect-apt fails to determine the fastest mirror as it needs ICMP and UDP outgoing. Should netselect-apt fail the default mirrors will be used. +- **Improve install speed with apt-fast** + - Migrating from a stable base install to Debian (Sid) requires downloading lots of packages. Depending on your geo location the download speed was already improved by introducing netselect-apt to determine the fastest mirror. With apt-fast the downloads will be even faster by downloading packages not only in parallel but also with multiple connections per package. diff --git a/README.md b/README.md index 37c5f5a3..4d656d4a 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,11 @@ Furthermore we use the following tools - This feature is beta and is mostly intended to provide you with the latest development advances without the need of reinstalling T-Pot. - **Deprecated tools** - *ctop* will no longer be part of T-Pot. +- **Fix #332** + - If T-Pot, opposed to the requirements, does not have full internet access netselect-apt fails to determine the fastest mirror as it needs ICMP and UDP outgoing. Should netselect-apt fail the default mirrors will be used. +- **Improve install speed with apt-fast** + - Migrating from a stable base install to Debian (Sid) requires downloading lots of packages. Depending on your geo location the download speed was already improved by introducing netselect-apt to determine the fastest mirror. Wit +h apt-fast the downloads will be even faster by downloading packages not only in parallel but also with multiple connections per package. # Technical Concept diff --git a/iso/installer/install.sh b/iso/installer/install.sh index 70db8edd..fdb9a71c 100755 --- a/iso/installer/install.sh +++ b/iso/installer/install.sh @@ -665,7 +665,7 @@ pip install elasticsearch-curator yq # Cloning T-Pot from GitHub fuBANNER "Cloning T-Pot" -git clone https://github.com/dtag-dev-sec/tpotce -b fast /opt/tpot +git clone https://github.com/dtag-dev-sec/tpotce /opt/tpot # Let's create the T-Pot user fuBANNER "Create user" diff --git a/iso/preseed/tpot.seed b/iso/preseed/tpot.seed index dc9c04a9..32aafd12 100755 --- a/iso/preseed/tpot.seed +++ b/iso/preseed/tpot.seed @@ -131,7 +131,7 @@ in-target apt-get -y install grub-pc; \ in-target grub-install --force $(debconf-get partman-auto/disk); \ update-dev; \ in-target update-grub; \ -in-target git clone --depth=1 https://github.com/dtag-dev-sec/tpotce -b fast /opt/tpot; \ +in-target git clone --depth=1 https://github.com/dtag-dev-sec/tpotce /opt/tpot; \ in-target sed -i 's/allow-hotplug/auto/g' /etc/network/interfaces; \ #in-target apt-get -y remove exim4-base; \ #in-target apt-get -y autoremove; \ diff --git a/update.sh b/update.sh index 54080aa7..e5d68cb7 100755 --- a/update.sh +++ b/update.sh @@ -58,7 +58,7 @@ function fuSELFUPDATE () { echo "###### $myBLUE""No updates found in repository.""$myWHITE" return fi - myRESULT=$(git diff --name-only origin/fast | grep update.sh) + myRESULT=$(git diff --name-only origin/master | grep update.sh) if [ "$myRESULT" == "update.sh" ]; then echo "###### $myBLUE""Found newer version, will be pulling updates and restart myself.""$myWHITE" From b59485eea225aa2d1302caaf9902faacf07f3a1d Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 4 Apr 2019 08:31:41 +0000 Subject: [PATCH 4/5] credits apt-fast --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4d656d4a..6470f9eb 100644 --- a/README.md +++ b/README.md @@ -491,6 +491,7 @@ Without open source and the fruitful development community (we are proud to be a ### The developers and development communities of * [adbhoney](https://github.com/huuck/ADBHoney/graphs/contributors) +* [aptfast](https://github.com/ilikenwf/apt-fast/graphs/contributors) * [ciscoasa](https://github.com/Cymmetria/ciscoasa_honeypot/graphs/contributors) * [cockpit](https://github.com/cockpit-project/cockpit/graphs/contributors) * [conpot](https://github.com/mushorg/conpot/graphs/contributors) From ea93cd1db700f622ab33d0c5ec9aaaabd022d302 Mon Sep 17 00:00:00 2001 From: Marco Ochse Date: Thu, 4 Apr 2019 10:36:35 +0200 Subject: [PATCH 5/5] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6470f9eb..4ed7c412 100644 --- a/README.md +++ b/README.md @@ -491,7 +491,7 @@ Without open source and the fruitful development community (we are proud to be a ### The developers and development communities of * [adbhoney](https://github.com/huuck/ADBHoney/graphs/contributors) -* [aptfast](https://github.com/ilikenwf/apt-fast/graphs/contributors) +* [apt-fast](https://github.com/ilikenwf/apt-fast/graphs/contributors) * [ciscoasa](https://github.com/Cymmetria/ciscoasa_honeypot/graphs/contributors) * [cockpit](https://github.com/cockpit-project/cockpit/graphs/contributors) * [conpot](https://github.com/mushorg/conpot/graphs/contributors)