download instead of git pull

download translation maps rather than running a git pull
translation maps will now be bzip2 compressed to reduce traffic to a minimum
fixes #432

docker/elk/logstash/Dockerfile

@@ -1,22 +1,25 @@
 FROM alpine
-
+#
 # Include dist
 ADD dist/ /root/dist/
-
+#
 # Setup env and apt
 RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     apk -U --no-cache add \
              aria2 \
              bash \
-             curl \
+             bzip2 \
-             git \
+	     curl \
              libc6-compat \
              libzmq \
              nss \
              openjdk8-jre && \
-
+#
 # Get and install packages
-    git clone --depth=1 https://github.com/dtag-dev-sec/listbot /etc/listbot && \
+    mkdir -p /etc/listbot && \
+    aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/cve.yaml.bz2 && \
+    aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/iprep.yaml.bz2 && \
+    bunzip2 *.bz2 && \
     cd /root/dist/ && \
     mkdir -p /usr/share/logstash/ && \
     aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.6.2.tar.gz && \
@@ -25,7 +28,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
     aria2c -s 16 -x 16 -o GeoLite2-ASN.tar.gz http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz && \
     tar xvfz GeoLite2-ASN.tar.gz --strip-components=1 -C /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor && \
-
+#
 # Add and move files
     cd /root/dist/ && \
     cp update.sh /usr/bin/ && \
@@ -33,23 +36,22 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
     mkdir -p /etc/logstash/conf.d && \
     cp logstash.conf /etc/logstash/conf.d/ && \
     cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.3.2-java/lib/logstash/outputs/elasticsearch/ && \
-
+#
 # Setup user, groups and configs
     addgroup -g 2000 logstash && \
     adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \
     chown -R logstash:logstash /usr/share/logstash && \
     chown -R logstash:logstash /etc/listbot && \
     chmod 755 /usr/bin/update.sh && \
-
+#
 # Clean up
-    apk del --purge aria2 && \
     rm -rf /root/* && \
     rm -rf /tmp/* && \
     rm -rf /var/cache/apk/*
-
+#
 # Healthcheck
 HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
-
+#
 # Start logstash
 #USER logstash:logstash
 CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution

docker/elk/logstash/dist/update.sh

@@ -6,7 +6,31 @@ function fuCLEANUP {
 }
 trap fuCLEANUP EXIT
 
-# Download updated translation maps
+# Check internet availability 
-cd /etc/listbot 
+function fuCHECKINET () {
-git pull --all --depth=1
+mySITES=$1
-cd /
+error=0
+for i in $mySITES;
+  do
+    curl --connect-timeout 5 -Is $i 2>&1 > /dev/null
+      if [ $? -ne 0 ];
+        then
+          let error+=1
+      fi;
+  done;
+  echo $error
+}
+
+# Check for connectivity and download latest translation maps
+myCHECK=$(fuCHECKINET "raw.githubusercontent.com")
+if [ "$myCHECK" == "0" ];
+  then
+    echo "Connection to Github looks good, now downloading latest translation maps."
+    cd /etc/listbot 
+    aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/cve.yaml.bz2 && \
+    aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/iprep.yaml.bz2 && \
+    bunzip2 -f *.bz2
+    cd /
+  else
+    echo "Cannot reach Github, starting Logstash without latest translation maps."
+fi