Merge pull request #336 from dtag-dev-sec/fast

fix #332, apt-fast

CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+## 20190404
+- **Fix #332**
+  - If T-Pot, opposed to the requirements, does not have full internet access netselect-apt fails to determine the fastest mirror as it needs ICMP and UDP outgoing. Should netselect-apt fail the default mirrors will be used.
+- **Improve install speed with apt-fast**
+  - Migrating from a stable base install to Debian (Sid) requires downloading lots of packages. Depending on your geo location the download speed was already improved by introducing netselect-apt to determine the fastest mirror. With apt-fast the downloads will be even faster by downloading packages not only in parallel but also with multiple connections per package.

README.md

@@ -112,6 +112,11 @@ Furthermore we use the following tools
   - This feature is beta and is mostly intended to provide you with the latest development advances without the need of reinstalling T-Pot.
 - **Deprecated tools**
   - *ctop* will no longer be part of T-Pot.
+- **Fix #332**
+  - If T-Pot, opposed to the requirements, does not have full internet access netselect-apt fails to determine the fastest mirror as it needs ICMP and UDP outgoing. Should netselect-apt fail the default mirrors will be used.
+- **Improve install speed with apt-fast**
+  - Migrating from a stable base install to Debian (Sid) requires downloading lots of packages. Depending on your geo location the download speed was already improved by introducing netselect-apt to determine the fastest mirror. Wit
+h apt-fast the downloads will be even faster by downloading packages not only in parallel but also with multiple connections per package.
 
 <a name="concept"></a>
 # Technical Concept
@@ -486,6 +491,7 @@ Without open source and the fruitful development community (we are proud to be a
 ### The developers and development communities of
 
 * [adbhoney](https://github.com/huuck/ADBHoney/graphs/contributors)
+* [apt-fast](https://github.com/ilikenwf/apt-fast/graphs/contributors)
 * [ciscoasa](https://github.com/Cymmetria/ciscoasa_honeypot/graphs/contributors)
 * [cockpit](https://github.com/cockpit-project/cockpit/graphs/contributors)
 * [conpot](https://github.com/mushorg/conpot/graphs/contributors)

iso/installer/install.sh

@@ -13,8 +13,8 @@ myTPOTCOMPOSE="/opt/tpot/etc/tpot.yml"
 myLSB_STABLE_SUPPORTED="stretch"
 myLSB_TESTING_SUPPORTED="sid"
 myREMOTESITES="https://hub.docker.com https://github.com https://pypi.python.org https://debian.org"
-myPREINSTALLPACKAGES="apache2-utils curl dialog figlet grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet"
-myINSTALLPACKAGES="apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose dstat ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass prips software-properties-common syslinux psmisc pv python-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
+myPREINSTALLPACKAGES="aria2 apache2-utils curl dialog figlet grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet"
+myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose dstat ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass prips software-properties-common syslinux psmisc pv python-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
 myINFO="\
 ########################################
 ### T-Pot Installer for Debian (Sid) ###
@@ -165,7 +165,7 @@ myCRONJOBS="
 27 3 * * *      root    systemctl stop tpot && docker stop \$(docker ps -aq) || docker rm \$(docker ps -aq) || reboot
 
 # Check for updated packages every sunday, upgrade and reboot
-27 16 * * 0     root    apt-get autoclean -y && apt-get autoremove -y && apt-get update -y && apt-get upgrade -y && sleep 10 && reboot
+27 16 * * 0     root    apt-fast autoclean -y && apt-fast autoremove -y && apt-fast update -y && apt-fast upgrade -y && sleep 10 && reboot
 "
 myROOTPROMPT='PS1="\[\033[38;5;8m\][\[$(tput sgr0)\]\[\033[38;5;1m\]\u\[$(tput sgr0)\]\[\033[38;5;6m\]@\[$(tput sgr0)\]\[\033[38;5;4m\]\h\[$(tput sgr0)\]\[\033[38;5;6m\]:\[$(tput sgr0)\]\[\033[38;5;5m\]\w\[$(tput sgr0)\]\[\033[38;5;8m\]]\[$(tput sgr0)\]\[\033[38;5;1m\]\\$\[$(tput sgr0)\]\[\033[38;5;15m\] \[$(tput sgr0)\]"'
 myUSERPROMPT='PS1="\[\033[38;5;8m\][\[$(tput sgr0)\]\[\033[38;5;2m\]\u\[$(tput sgr0)\]\[\033[38;5;6m\]@\[$(tput sgr0)\]\[\033[38;5;4m\]\h\[$(tput sgr0)\]\[\033[38;5;6m\]:\[$(tput sgr0)\]\[\033[38;5;5m\]\w\[$(tput sgr0)\]\[\033[38;5;8m\]]\[$(tput sgr0)\]\[\033[38;5;2m\]\\$\[$(tput sgr0)\]\[\033[38;5;15m\] \[$(tput sgr0)\]"'
@@ -213,6 +213,8 @@ fi
 # If not present install them
 function fuCHECKPACKAGES {
   export DEBIAN_FRONTEND=noninteractive
+  echo "### Installing apt-fast"
+  /bin/bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)"
   echo -n "### Checking for installer dependencies: "
   local myPACKAGES="$1"
   for myDEPS in $myPACKAGES;
@@ -221,8 +223,8 @@ function fuCHECKPACKAGES {
       if [ "$myOK" != "ok" ];
         then
           echo "[ NOW INSTALLING ]"
-          apt-get update -y
-          apt-get install -y $myPACKAGES
+          apt-fast update -y
+          apt-fast install -y $myPACKAGES
           break
       fi
   done
@@ -268,24 +270,34 @@ function fuGET_DEPS {
   echo "### Determine fastest mirror for your location."
   echo
   netselect-apt -n -a amd64 unstable && cp sources.list /etc/apt/
+  mySOURCESCHECK=$(cat /etc/apt/sources.list | grep -c unstable)
+  if [ "$mySOURCESCHECK" == "0" ]
+    then
+      echo "### Automatic mirror selection failed, using main mirror."
+      # Point to Debian (Sid, unstable)
+      tee /etc/apt/sources.list <<EOF
+      deb http://deb.debian.org/debian unstable main contrib non-free
+      deb-src http://deb.debian.org/debian unstable main contrib non-free
+EOF
+  fi
   echo
   echo "### Getting update information."
   echo
-  apt-get -y update
+  apt-fast -y update
   echo
   echo "### Upgrading packages."
   echo
   # Downlaod and upgrade packages, but silently keep existing configs
   echo "docker.io docker.io/restart       boolean true" | debconf-set-selections -v
   echo "debconf debconf/frontend select noninteractive" | debconf-set-selections -v
-  apt-get -y dist-upgrade -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes
+  apt-fast -y dist-upgrade -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes
   echo
   echo "### Installing T-Pot dependencies."
   echo
-  apt-get -y install $myINSTALLPACKAGES
+  apt-fast -y install $myINSTALLPACKAGES
   # Remove exim4
-  apt-get -y purge exim4-base mailutils
-  apt-get -y autoremove
+  apt-fast -y purge exim4-base mailutils
+  apt-fast -y autoremove
   apt-mark hold exim4-base mailutils
 }
 
@@ -808,8 +820,8 @@ fuBANNER "Update IP"
 
 # Let's clean up apt
 fuBANNER "Clean up"
-apt-get autoclean -y
-apt-get autoremove -y
+apt-fast autoclean -y
+apt-fast autoremove -y
 
 # Final steps
 cp /opt/tpot/host/etc/rc.local /etc/rc.local && \

update.sh

@@ -176,24 +176,26 @@ echo
 
 function fuUPDATER () {
 export DEBIAN_FRONTEND=noninteractive
-local myPACKAGES="apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose dstat ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass prips software-properties-common syslinux psmisc pv python-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
+echo "### Installing apt-fast"
+/bin/bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)"
+local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose dstat ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass prips software-properties-common syslinux psmisc pv python-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant"
 echo "### Now upgrading packages ..."
 dpkg --configure -a
-apt-get -y autoclean
-apt-get -y autoremove
-apt-get update
-apt-get -y install $myPACKAGES
+apt-fast -y autoclean
+apt-fast -y autoremove
+apt-fast update
+apt-fast -y install $myPACKAGES
 
 # Some updates require interactive attention, and the following settings will override that.
 echo "docker.io docker.io/restart       boolean true" | debconf-set-selections -v
 echo "debconf debconf/frontend select noninteractive" | debconf-set-selections -v
-apt-get -y dist-upgrade -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes
+apt-fast -y dist-upgrade -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes
 dpkg --configure -a
 npm install "https://github.com/taskrabbit/elasticsearch-dump" -g
 pip install --upgrade pip
 hash -r
 pip install --upgrade elasticsearch-curator yq
-apt-get -y purge exim4-base mailutils
+apt-fast -y purge exim4-base mailutils
 apt-mark hold exim4-base mailutils
 echo