From 5a4724bcba8acd33ec2c9a847cecff8fd0783651 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 31 Jan 2020 14:21:55 +0000 Subject: [PATCH 01/31] elk 7.x dev test --- docker/elk/docker-compose.yml | 8 +-- docker/elk/elasticsearch/Dockerfile | 10 +-- docker/elk/elasticsearch/Dockerfile.new | 46 ++++++++++++++ docker/elk/elasticsearch/Dockerfile.old | 44 +++++++++++++ .../elk/elasticsearch/dist/elasticsearch.yml | 5 +- docker/elk/elasticsearch/docker-compose.yml | 2 +- docker/elk/head/docker-compose.yml | 2 +- docker/elk/kibana/Dockerfile | 29 +++++---- docker/elk/kibana/Dockerfile.old | 63 +++++++++++++++++++ docker/elk/kibana/docker-compose.yml | 2 +- docker/elk/logstash/Dockerfile | 10 +-- docker/elk/logstash/Dockerfile.old | 56 +++++++++++++++++ .../dist/elasticsearch-template-es5x.json | 53 ---------------- .../dist/elasticsearch-template-es6x.json | 48 -------------- .../dist/elasticsearch-template-es7x.json | 46 ++++++++++++++ docker/elk/logstash/dist/logstash.conf | 4 +- docker/elk/logstash/docker-compose.yml | 2 +- etc/compose/nextgen.yml | 8 +-- 18 files changed, 302 insertions(+), 136 deletions(-) create mode 100644 docker/elk/elasticsearch/Dockerfile.new create mode 100644 docker/elk/elasticsearch/Dockerfile.old create mode 100644 docker/elk/kibana/Dockerfile.old create mode 100644 docker/elk/logstash/Dockerfile.old delete mode 100644 docker/elk/logstash/dist/elasticsearch-template-es5x.json delete mode 100644 docker/elk/logstash/dist/elasticsearch-template-es6x.json create mode 100644 docker/elk/logstash/dist/elasticsearch-template-es7x.json diff --git a/docker/elk/docker-compose.yml b/docker/elk/docker-compose.yml index 4b068b01..10fd8b67 100644 --- a/docker/elk/docker-compose.yml +++ b/docker/elk/docker-compose.yml @@ -24,7 +24,7 @@ services: mem_limit: 4g ports: - "127.0.0.1:64298:9200" - image: "dtagdevsec/elasticsearch:1903" + image: "dtagdevsec/elasticsearch:2006" volumes: - /data:/data @@ -39,7 +39,7 @@ services: condition: service_healthy ports: - "127.0.0.1:64296:5601" - image: "dtagdevsec/kibana:1903" + image: "dtagdevsec/kibana:2006" ## Logstash service logstash: @@ -51,7 +51,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/logstash:1903" + image: "dtagdevsec/logstash:2006" volumes: - /data:/data - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf @@ -66,5 +66,5 @@ services: condition: service_healthy ports: - "127.0.0.1:64302:9100" - image: "dtagdevsec/head:1903" + image: "dtagdevsec/head:2006" read_only: true diff --git a/docker/elk/elasticsearch/Dockerfile b/docker/elk/elasticsearch/Dockerfile index f1eb0183..45108a4d 100644 --- a/docker/elk/elasticsearch/Dockerfile +++ b/docker/elk/elasticsearch/Dockerfile @@ -1,5 +1,8 @@ FROM alpine # +# VARS +ENV ES_VER=7.5.2 \ + JAVA_HOME=/usr/lib/jvm/java-11-openjdk # Include dist ADD dist/ /root/dist/ # @@ -10,13 +13,13 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ bash \ curl \ nss \ - openjdk8-jre && \ + openjdk11-jre && \ # # Get and install packages cd /root/dist/ && \ mkdir -p /usr/share/elasticsearch/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && \ - tar xvfz elasticsearch-6.8.6.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VER-linux-x86_64.tar.gz && \ + tar xvfz elasticsearch-$ES_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ # # Add and move files cd /root/dist/ && \ @@ -40,5 +43,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' # # Start ELK USER elasticsearch:elasticsearch -ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/elasticsearch/Dockerfile.new b/docker/elk/elasticsearch/Dockerfile.new new file mode 100644 index 00000000..45108a4d --- /dev/null +++ b/docker/elk/elasticsearch/Dockerfile.new @@ -0,0 +1,46 @@ +FROM alpine +# +# VARS +ENV ES_VER=7.5.2 \ + JAVA_HOME=/usr/lib/jvm/java-11-openjdk +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + bash \ + curl \ + nss \ + openjdk11-jre && \ +# +# Get and install packages + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VER-linux-x86_64.tar.gz && \ + tar xvfz elasticsearch-$ES_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ +# +# Add and move files + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/config && \ + cp elasticsearch.yml /usr/share/elasticsearch/config/ && \ +# +# Setup user, groups and configs + addgroup -g 2000 elasticsearch && \ + adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \ + chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \ + rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ +# +# Clean up + apk del --purge aria2 && \ + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' +# +# Start ELK +USER elasticsearch:elasticsearch +CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/elasticsearch/Dockerfile.old b/docker/elk/elasticsearch/Dockerfile.old new file mode 100644 index 00000000..f1eb0183 --- /dev/null +++ b/docker/elk/elasticsearch/Dockerfile.old @@ -0,0 +1,44 @@ +FROM alpine +# +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + bash \ + curl \ + nss \ + openjdk8-jre && \ +# +# Get and install packages + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && \ + tar xvfz elasticsearch-6.8.6.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ +# +# Add and move files + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/config && \ + cp elasticsearch.yml /usr/share/elasticsearch/config/ && \ +# +# Setup user, groups and configs + addgroup -g 2000 elasticsearch && \ + adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \ + chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \ + rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ +# +# Clean up + apk del --purge aria2 && \ + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' +# +# Start ELK +USER elasticsearch:elasticsearch +ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk +CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/elasticsearch/dist/elasticsearch.yml b/docker/elk/elasticsearch/dist/elasticsearch.yml index 2466b212..e8b4e479 100644 --- a/docker/elk/elasticsearch/dist/elasticsearch.yml +++ b/docker/elk/elasticsearch/dist/elasticsearch.yml @@ -1,11 +1,14 @@ cluster.name: tpotcluster node.name: "tpotcluster-node-01" xpack.ml.enabled: false +xpack.security.enabled: false path: logs: /data/elk/log data: /data/elk/data http.host: 0.0.0.0 http.cors.enabled: true http.cors.allow-origin: "*" +cluster.initial_master_nodes: +- "tpotcluster-node-01" discovery.zen.ping.unicast.hosts: - - localhost +- localhost diff --git a/docker/elk/elasticsearch/docker-compose.yml b/docker/elk/elasticsearch/docker-compose.yml index a09ec66c..3f51dcb5 100644 --- a/docker/elk/elasticsearch/docker-compose.yml +++ b/docker/elk/elasticsearch/docker-compose.yml @@ -24,6 +24,6 @@ services: mem_limit: 2g ports: - "127.0.0.1:64298:9200" - image: "dtagdevsec/elasticsearch:1903" + image: "dtagdevsec/elasticsearch:2006" volumes: - /data:/data diff --git a/docker/elk/head/docker-compose.yml b/docker/elk/head/docker-compose.yml index 35865584..5cfaafdb 100644 --- a/docker/elk/head/docker-compose.yml +++ b/docker/elk/head/docker-compose.yml @@ -12,5 +12,5 @@ services: # condition: service_healthy ports: - "127.0.0.1:64302:9100" - image: "dtagdevsec/head:1903" + image: "dtagdevsec/head:2006" read_only: true diff --git a/docker/elk/kibana/Dockerfile b/docker/elk/kibana/Dockerfile index 5088f540..6e6fd837 100644 --- a/docker/elk/kibana/Dockerfile +++ b/docker/elk/kibana/Dockerfile @@ -1,5 +1,8 @@ FROM node:10.15.2-alpine # +# VARS +ENV KB_VER=7.5.2 +# # Include dist ADD dist/ /root/dist/ # @@ -12,20 +15,20 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Get and install packages cd /root/dist/ && \ mkdir -p /usr/share/kibana/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-6.8.6-linux-x86_64.tar.gz && \ - tar xvfz kibana-6.8.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-$KB_VER-linux-x86_64.tar.gz && \ + tar xvfz kibana-$KB_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \ # # Kibana's bundled node does not work in alpine rm /usr/share/kibana/node/bin/node && \ - ln -s /usr/bin/node /usr/share/kibana/node/bin/node && \ + ln -s /usr/local/bin/node /usr/share/kibana/node/bin/node && \ # # Add and move files cd /root/dist/ && \ - cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \ - cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \ +# cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \ +# cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \ +# cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \ +# cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \ +# cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \ # # Setup user, groups and configs sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \ @@ -33,17 +36,19 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \ sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \ sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \ - sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ - sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ - sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ +# sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ +# sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ +# sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ rm -rf /usr/share/kibana/optimize/bundles/* && \ - /usr/share/kibana/bin/kibana --optimize && \ + /usr/share/kibana/bin/kibana --optimize --allow-root && \ addgroup -g 2000 kibana && \ adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \ chown -R kibana:kibana /usr/share/kibana/ && \ diff --git a/docker/elk/kibana/Dockerfile.old b/docker/elk/kibana/Dockerfile.old new file mode 100644 index 00000000..5088f540 --- /dev/null +++ b/docker/elk/kibana/Dockerfile.old @@ -0,0 +1,63 @@ +FROM node:10.15.2-alpine +# +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + curl && \ +# +# Get and install packages + cd /root/dist/ && \ + mkdir -p /usr/share/kibana/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-6.8.6-linux-x86_64.tar.gz && \ + tar xvfz kibana-6.8.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \ +# +# Kibana's bundled node does not work in alpine + rm /usr/share/kibana/node/bin/node && \ + ln -s /usr/bin/node /usr/share/kibana/node/bin/node && \ +# +# Add and move files + cd /root/dist/ && \ + cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \ + cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \ + cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \ + cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \ + cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \ +# +# Setup user, groups and configs + sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \ + sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ + sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ + sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ + echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + rm -rf /usr/share/kibana/optimize/bundles/* && \ + /usr/share/kibana/bin/kibana --optimize && \ + addgroup -g 2000 kibana && \ + adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \ + chown -R kibana:kibana /usr/share/kibana/ && \ +# +# Clean up + apk del --purge aria2 && \ + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:5601' +# +# Start kibana +STOPSIGNAL SIGKILL +USER kibana:kibana +CMD ["/usr/share/kibana/bin/kibana"] diff --git a/docker/elk/kibana/docker-compose.yml b/docker/elk/kibana/docker-compose.yml index 3044a84f..2f464089 100644 --- a/docker/elk/kibana/docker-compose.yml +++ b/docker/elk/kibana/docker-compose.yml @@ -12,4 +12,4 @@ services: # condition: service_healthy ports: - "127.0.0.1:64296:5601" - image: "dtagdevsec/kibana:1903" + image: "dtagdevsec/kibana:2006" diff --git a/docker/elk/logstash/Dockerfile b/docker/elk/logstash/Dockerfile index 9d39fedf..1eb89a43 100644 --- a/docker/elk/logstash/Dockerfile +++ b/docker/elk/logstash/Dockerfile @@ -1,5 +1,7 @@ FROM alpine # +# VARS +ENV LS_VER=7.5.2 # Include dist ADD dist/ /root/dist/ # @@ -13,7 +15,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ libc6-compat \ libzmq \ nss \ - openjdk8-jre && \ + openjdk11-jre && \ # # Get and install packages mkdir -p /etc/listbot && \ @@ -23,8 +25,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ bunzip2 *.bz2 && \ cd /root/dist/ && \ mkdir -p /usr/share/logstash/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \ - tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER.tar.gz && \ + tar xvfz logstash-$LS_VER.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \ /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \ # @@ -34,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ chmod u+x /usr/bin/update.sh && \ mkdir -p /etc/logstash/conf.d && \ cp logstash.conf /etc/logstash/conf.d/ && \ - cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \ + cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.2.3-java/lib/logstash/outputs/elasticsearch/ && \ # # Setup user, groups and configs addgroup -g 2000 logstash && \ diff --git a/docker/elk/logstash/Dockerfile.old b/docker/elk/logstash/Dockerfile.old new file mode 100644 index 00000000..9d39fedf --- /dev/null +++ b/docker/elk/logstash/Dockerfile.old @@ -0,0 +1,56 @@ +FROM alpine +# +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + bash \ + bzip2 \ + curl \ + libc6-compat \ + libzmq \ + nss \ + openjdk8-jre && \ +# +# Get and install packages + mkdir -p /etc/listbot && \ + cd /etc/listbot && \ + aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/cve.yaml.bz2 && \ + aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/iprep.yaml.bz2 && \ + bunzip2 *.bz2 && \ + cd /root/dist/ && \ + mkdir -p /usr/share/logstash/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \ + tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ + /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \ + /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \ +# +# Add and move files + cd /root/dist/ && \ + cp update.sh /usr/bin/ && \ + chmod u+x /usr/bin/update.sh && \ + mkdir -p /etc/logstash/conf.d && \ + cp logstash.conf /etc/logstash/conf.d/ && \ + cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \ +# +# Setup user, groups and configs + addgroup -g 2000 logstash && \ + adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \ + chown -R logstash:logstash /usr/share/logstash && \ + chown -R logstash:logstash /etc/listbot && \ + chmod 755 /usr/bin/update.sh && \ +# +# Clean up + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600' +# +# Start logstash +#USER logstash:logstash +CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution diff --git a/docker/elk/logstash/dist/elasticsearch-template-es5x.json b/docker/elk/logstash/dist/elasticsearch-template-es5x.json deleted file mode 100644 index f02dfdb8..00000000 --- a/docker/elk/logstash/dist/elasticsearch-template-es5x.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "template" : "logstash-*", - "version" : 50001, - "settings" : { - "index.refresh_interval" : "5s", - "index.number_of_shards" : "1", - "index.number_of_replicas" : "0", - "mapping" : { - "total_fields" : { - "limit" : "2000" - } - } - }, - "mappings" : { - "_default_" : { - "_all" : {"enabled" : true, "norms" : false}, - "dynamic_templates" : [ { - "message_field" : { - "path_match" : "message", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", - "norms" : false - } - } - }, { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", "norms" : false, - "fields" : { - "keyword" : { "type": "keyword", "ignore_above": 256 } - } - } - } - } ], - "properties" : { - "@timestamp": { "type": "date", "include_in_all": false }, - "@version": { "type": "keyword", "include_in_all": false }, - "geoip" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - } - } - } - } -} diff --git a/docker/elk/logstash/dist/elasticsearch-template-es6x.json b/docker/elk/logstash/dist/elasticsearch-template-es6x.json deleted file mode 100644 index aff190b9..00000000 --- a/docker/elk/logstash/dist/elasticsearch-template-es6x.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "template" : "logstash-*", - "version" : 60001, - "settings" : { - "index.refresh_interval" : "5s", - "index.number_of_shards" : "1", - "index.number_of_replicas" : "0", - "index.mapping.total_fields.limit": "2000" - }, - "mappings" : { - "_default_" : { - "dynamic_templates" : [ { - "message_field" : { - "path_match" : "message", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", - "norms" : false - } - } - }, { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", "norms" : false, - "fields" : { - "keyword" : { "type": "keyword", "ignore_above": 256 } - } - } - } - } ], - "properties" : { - "@timestamp": { "type": "date"}, - "@version": { "type": "keyword"}, - "geoip" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - } - } - } - } -} diff --git a/docker/elk/logstash/dist/elasticsearch-template-es7x.json b/docker/elk/logstash/dist/elasticsearch-template-es7x.json new file mode 100644 index 00000000..71793ff7 --- /dev/null +++ b/docker/elk/logstash/dist/elasticsearch-template-es7x.json @@ -0,0 +1,46 @@ +{ + "index_patterns" : "logstash-*", + "version" : 60001, + "settings" : { + "index.refresh_interval" : "5s", + "number_of_shards": 1, + "index.number_of_replicas" : "0", + "index.mapping.total_fields.limit": "2000" + }, + "mappings" : { + "dynamic_templates" : [ { + "message_field" : { + "path_match" : "message", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", + "norms" : false + } + } + }, { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", "norms" : false, + "fields" : { + "keyword" : { "type": "keyword", "ignore_above": 256 } + } + } + } + } ], + "properties" : { + "@timestamp": { "type": "date"}, + "@version": { "type": "keyword"}, + "geoip" : { + "dynamic": true, + "properties" : { + "ip": { "type": "ip" }, + "location" : { "type" : "geo_point" }, + "latitude" : { "type" : "half_float" }, + "longitude" : { "type" : "half_float" } + } + } + } + } +} diff --git a/docker/elk/logstash/dist/logstash.conf b/docker/elk/logstash/dist/logstash.conf index 3d118364..0beb0c10 100644 --- a/docker/elk/logstash/dist/logstash.conf +++ b/docker/elk/logstash/dist/logstash.conf @@ -413,12 +413,12 @@ if "_grokparsefailure" in [tags] { drop {} } geoip { cache_size => 10000 source => "src_ip" - database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-City.mmdb" } geoip { cache_size => 10000 source => "src_ip" - database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-ASN.mmdb" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-ASN.mmdb" } translate { refresh_interval => 86400 diff --git a/docker/elk/logstash/docker-compose.yml b/docker/elk/logstash/docker-compose.yml index c213a098..5b74feb3 100644 --- a/docker/elk/logstash/docker-compose.yml +++ b/docker/elk/logstash/docker-compose.yml @@ -12,7 +12,7 @@ services: # condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/logstash:1903" + image: "dtagdevsec/logstash:2006" volumes: - /data:/data - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 0bb76907..509d97e6 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -496,7 +496,7 @@ services: mem_limit: 4g ports: - "127.0.0.1:64298:9200" - image: "dtagdevsec/elasticsearch:1903" + image: "dtagdevsec/elasticsearch:2006" volumes: - /data:/data @@ -509,7 +509,7 @@ services: condition: service_healthy ports: - "127.0.0.1:64296:5601" - image: "dtagdevsec/kibana:1903" + image: "dtagdevsec/kibana:2006" ## Logstash service logstash: @@ -520,7 +520,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/logstash:1903" + image: "dtagdevsec/logstash:2006" volumes: - /data:/data @@ -533,7 +533,7 @@ services: condition: service_healthy ports: - "127.0.0.1:64302:9100" - image: "dtagdevsec/head:1903" + image: "dtagdevsec/head:2006" read_only: true # Ewsposter service From 64729f5064e1fa5991cc8242a994a5de3335d7e1 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 31 Jan 2020 15:50:34 +0000 Subject: [PATCH 02/31] remove ilm support, breaks existing index at upgrade --- docker/elk/elasticsearch/dist/elasticsearch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/elk/elasticsearch/dist/elasticsearch.yml b/docker/elk/elasticsearch/dist/elasticsearch.yml index e8b4e479..62085d47 100644 --- a/docker/elk/elasticsearch/dist/elasticsearch.yml +++ b/docker/elk/elasticsearch/dist/elasticsearch.yml @@ -2,6 +2,7 @@ cluster.name: tpotcluster node.name: "tpotcluster-node-01" xpack.ml.enabled: false xpack.security.enabled: false +xpack.ilm.enabled: false path: logs: /data/elk/log data: /data/elk/data From 2d249ac6b1e7da4c2b7dacd251bc0dd587a44cf3 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 31 Jan 2020 17:43:04 +0000 Subject: [PATCH 03/31] tweak export script for new references --- bin/export_kibana-objects.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/export_kibana-objects.sh b/bin/export_kibana-objects.sh index d7f94465..6cea65f5 100755 --- a/bin/export_kibana-objects.sh +++ b/bin/export_kibana-objects.sh @@ -32,7 +32,7 @@ trap fuCLEANUP EXIT # Export index patterns mkdir -p patterns echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index pattern fields." $myCOL0 -curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern/'$myINDEXID'' | jq '. | {attributes}' > patterns/$myINDEXID.json & +curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern/'$myINDEXID'' | jq '. | {attributes, references}' > patterns/$myINDEXID.json & echo # Export dashboards @@ -41,7 +41,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"das for i in $myDASHBOARDS; do echo $myCOL1"###### "$i $myCOL0 - curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard/'$i'' | jq '. | {attributes}' > dashboards/$i.json & + curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard/'$i'' | jq '. | {attributes, references}' > dashboards/$i.json & done; echo @@ -51,7 +51,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1 for i in $myVISUALIZATIONS; do echo $myCOL1"###### "$i $myCOL0 - curl -s -XGET ''$myKIBANA'api/saved_objects/visualization/'$i'' | jq '. | {attributes}' > visualizations/$i.json & + curl -s -XGET ''$myKIBANA'api/saved_objects/visualization/'$i'' | jq '. | {attributes, references}' > visualizations/$i.json & done; echo @@ -61,7 +61,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searc for i in $mySEARCHES; do echo $myCOL1"###### "$i $myCOL0 - curl -s -XGET ''$myKIBANA'api/saved_objects/search/'$i'' | jq '. | {attributes}' > searches/$i.json & + curl -s -XGET ''$myKIBANA'api/saved_objects/search/'$i'' | jq '. | {attributes, references}' > searches/$i.json & done; echo From 984ba958fb92bfd4efac280aa7b23b353930b60b Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Sat, 1 Feb 2020 14:08:23 +0000 Subject: [PATCH 04/31] logstash template not upgraded with daily index enabled logstash will not be able to put new events into ES simple solution, just deleting logstash template upon logstash start and leave it to logstash to upload the latest template . --- docker/elk/logstash/dist/update.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/elk/logstash/dist/update.sh b/docker/elk/logstash/dist/update.sh index 9619ac93..55281998 100644 --- a/docker/elk/logstash/dist/update.sh +++ b/docker/elk/logstash/dist/update.sh @@ -34,3 +34,11 @@ if [ "$myCHECK" == "0" ]; else echo "Cannot reach Github, starting Logstash without latest translation maps." fi + +# Make sure logstash can put latest logstash template by deleting the old one first +echo "Removing logstash template." +curl -XDELETE http://elasticsearch:9200/_template/logstash +echo +echo "Checking if empty." +curl -XGET http://elasticsearch:9200/_template/logstash +echo From b9da9f04af48a933d25ad2c5fcbac54fb981d5dc Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 3 Feb 2020 12:18:43 +0000 Subject: [PATCH 05/31] adjust default field --- docker/elk/logstash/dist/elasticsearch-template-es7x.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/elk/logstash/dist/elasticsearch-template-es7x.json b/docker/elk/logstash/dist/elasticsearch-template-es7x.json index 71793ff7..41e5c801 100644 --- a/docker/elk/logstash/dist/elasticsearch-template-es7x.json +++ b/docker/elk/logstash/dist/elasticsearch-template-es7x.json @@ -3,9 +3,12 @@ "version" : 60001, "settings" : { "index.refresh_interval" : "5s", - "number_of_shards": 1, + "number_of_shards" : 1, "index.number_of_replicas" : "0", - "index.mapping.total_fields.limit": "2000" + "index.mapping.total_fields.limit" : "2000", + "index.query": { + "default_field": "fields.*" + } }, "mappings" : { "dynamic_templates" : [ { From 5ce5911ec1a7beac071f9c6c8fbfecaf0d2ba4a4 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 3 Feb 2020 12:59:21 +0000 Subject: [PATCH 06/31] cleanup --- docker/elk/docker-compose.yml | 2 +- docker/elk/elasticsearch/Dockerfile.new | 46 ------------------ docker/elk/elasticsearch/Dockerfile.old | 44 ----------------- docker/elk/kibana/Dockerfile.old | 63 ------------------------- docker/elk/logstash/Dockerfile.old | 56 ---------------------- 5 files changed, 1 insertion(+), 210 deletions(-) delete mode 100644 docker/elk/elasticsearch/Dockerfile.new delete mode 100644 docker/elk/elasticsearch/Dockerfile.old delete mode 100644 docker/elk/kibana/Dockerfile.old delete mode 100644 docker/elk/logstash/Dockerfile.old diff --git a/docker/elk/docker-compose.yml b/docker/elk/docker-compose.yml index 10fd8b67..896b2cd5 100644 --- a/docker/elk/docker-compose.yml +++ b/docker/elk/docker-compose.yml @@ -54,7 +54,7 @@ services: image: "dtagdevsec/logstash:2006" volumes: - /data:/data - - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf +# - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf ## Elasticsearch-head service head: diff --git a/docker/elk/elasticsearch/Dockerfile.new b/docker/elk/elasticsearch/Dockerfile.new deleted file mode 100644 index 45108a4d..00000000 --- a/docker/elk/elasticsearch/Dockerfile.new +++ /dev/null @@ -1,46 +0,0 @@ -FROM alpine -# -# VARS -ENV ES_VER=7.5.2 \ - JAVA_HOME=/usr/lib/jvm/java-11-openjdk -# Include dist -ADD dist/ /root/dist/ -# -# Setup env and apt -RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ - apk -U --no-cache add \ - aria2 \ - bash \ - curl \ - nss \ - openjdk11-jre && \ -# -# Get and install packages - cd /root/dist/ && \ - mkdir -p /usr/share/elasticsearch/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VER-linux-x86_64.tar.gz && \ - tar xvfz elasticsearch-$ES_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ -# -# Add and move files - cd /root/dist/ && \ - mkdir -p /usr/share/elasticsearch/config && \ - cp elasticsearch.yml /usr/share/elasticsearch/config/ && \ -# -# Setup user, groups and configs - addgroup -g 2000 elasticsearch && \ - adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \ - chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \ - rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ -# -# Clean up - apk del --purge aria2 && \ - rm -rf /root/* && \ - rm -rf /tmp/* && \ - rm -rf /var/cache/apk/* -# -# Healthcheck -HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' -# -# Start ELK -USER elasticsearch:elasticsearch -CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/elasticsearch/Dockerfile.old b/docker/elk/elasticsearch/Dockerfile.old deleted file mode 100644 index f1eb0183..00000000 --- a/docker/elk/elasticsearch/Dockerfile.old +++ /dev/null @@ -1,44 +0,0 @@ -FROM alpine -# -# Include dist -ADD dist/ /root/dist/ -# -# Setup env and apt -RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ - apk -U --no-cache add \ - aria2 \ - bash \ - curl \ - nss \ - openjdk8-jre && \ -# -# Get and install packages - cd /root/dist/ && \ - mkdir -p /usr/share/elasticsearch/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && \ - tar xvfz elasticsearch-6.8.6.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ -# -# Add and move files - cd /root/dist/ && \ - mkdir -p /usr/share/elasticsearch/config && \ - cp elasticsearch.yml /usr/share/elasticsearch/config/ && \ -# -# Setup user, groups and configs - addgroup -g 2000 elasticsearch && \ - adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \ - chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \ - rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ -# -# Clean up - apk del --purge aria2 && \ - rm -rf /root/* && \ - rm -rf /tmp/* && \ - rm -rf /var/cache/apk/* -# -# Healthcheck -HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' -# -# Start ELK -USER elasticsearch:elasticsearch -ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/kibana/Dockerfile.old b/docker/elk/kibana/Dockerfile.old deleted file mode 100644 index 5088f540..00000000 --- a/docker/elk/kibana/Dockerfile.old +++ /dev/null @@ -1,63 +0,0 @@ -FROM node:10.15.2-alpine -# -# Include dist -ADD dist/ /root/dist/ -# -# Setup env and apt -RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ - apk -U --no-cache add \ - aria2 \ - curl && \ -# -# Get and install packages - cd /root/dist/ && \ - mkdir -p /usr/share/kibana/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-6.8.6-linux-x86_64.tar.gz && \ - tar xvfz kibana-6.8.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \ -# -# Kibana's bundled node does not work in alpine - rm /usr/share/kibana/node/bin/node && \ - ln -s /usr/bin/node /usr/share/kibana/node/bin/node && \ -# -# Add and move files - cd /root/dist/ && \ - cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \ - cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \ -# -# Setup user, groups and configs - sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \ - sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \ - sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \ - sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \ - sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \ - sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ - sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ - sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ - echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ - echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ - echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ - echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ - echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ - echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ - rm -rf /usr/share/kibana/optimize/bundles/* && \ - /usr/share/kibana/bin/kibana --optimize && \ - addgroup -g 2000 kibana && \ - adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \ - chown -R kibana:kibana /usr/share/kibana/ && \ -# -# Clean up - apk del --purge aria2 && \ - rm -rf /root/* && \ - rm -rf /tmp/* && \ - rm -rf /var/cache/apk/* -# -# Healthcheck -HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:5601' -# -# Start kibana -STOPSIGNAL SIGKILL -USER kibana:kibana -CMD ["/usr/share/kibana/bin/kibana"] diff --git a/docker/elk/logstash/Dockerfile.old b/docker/elk/logstash/Dockerfile.old deleted file mode 100644 index 9d39fedf..00000000 --- a/docker/elk/logstash/Dockerfile.old +++ /dev/null @@ -1,56 +0,0 @@ -FROM alpine -# -# Include dist -ADD dist/ /root/dist/ -# -# Setup env and apt -RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ - apk -U --no-cache add \ - aria2 \ - bash \ - bzip2 \ - curl \ - libc6-compat \ - libzmq \ - nss \ - openjdk8-jre && \ -# -# Get and install packages - mkdir -p /etc/listbot && \ - cd /etc/listbot && \ - aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/cve.yaml.bz2 && \ - aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/iprep.yaml.bz2 && \ - bunzip2 *.bz2 && \ - cd /root/dist/ && \ - mkdir -p /usr/share/logstash/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \ - tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ - /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \ - /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \ -# -# Add and move files - cd /root/dist/ && \ - cp update.sh /usr/bin/ && \ - chmod u+x /usr/bin/update.sh && \ - mkdir -p /etc/logstash/conf.d && \ - cp logstash.conf /etc/logstash/conf.d/ && \ - cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \ -# -# Setup user, groups and configs - addgroup -g 2000 logstash && \ - adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \ - chown -R logstash:logstash /usr/share/logstash && \ - chown -R logstash:logstash /etc/listbot && \ - chmod 755 /usr/bin/update.sh && \ -# -# Clean up - rm -rf /root/* && \ - rm -rf /tmp/* && \ - rm -rf /var/cache/apk/* -# -# Healthcheck -HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600' -# -# Start logstash -#USER logstash:logstash -CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution From ad861200de78d0ff25361b3758a7ce53ce99e13b Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 3 Feb 2020 14:46:43 +0000 Subject: [PATCH 07/31] update mailoney --- docker/mailoney/docker-compose.yml | 2 +- etc/compose/nextgen.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/mailoney/docker-compose.yml b/docker/mailoney/docker-compose.yml index d0faba6f..c5979e6b 100644 --- a/docker/mailoney/docker-compose.yml +++ b/docker/mailoney/docker-compose.yml @@ -20,7 +20,7 @@ services: - mailoney_local ports: - "25:25" - image: "dtagdevsec/mailoney:1903" + image: "dtagdevsec/mailoney:2006" read_only: true volumes: - /data/mailoney/log:/opt/mailoney/logs diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 509d97e6..60b534dd 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -288,7 +288,7 @@ services: - mailoney_local ports: - "25:25" - image: "dtagdevsec/mailoney:1903" + image: "dtagdevsec/mailoney:2006" read_only: true volumes: - /data/mailoney/log:/opt/mailoney/logs From a49d56080902d3e85f7f2ace766a987c80fcb9f0 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Wed, 5 Feb 2020 15:24:32 +0000 Subject: [PATCH 08/31] up java mem limit --- etc/compose/nextgen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 60b534dd..bb6381d2 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -482,7 +482,7 @@ services: restart: always environment: - bootstrap.memory_lock=true - - ES_JAVA_OPTS=-Xms1024m -Xmx1024m + - ES_JAVA_OPTS=-Xms2048m -Xmx2048m - ES_TMPDIR=/tmp cap_add: - IPC_LOCK From f11ad6b52329d29c781a47071ebac9fe90a99811 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 14 Feb 2020 15:28:06 +0000 Subject: [PATCH 09/31] tweaking ELK 7.6.0 is not ready for production, however it works if APM is enabled (disabled in config, so image wont build as precaution) Remove SISSDEN from ewsposter, suricata Bump suricata to 5.0.1 Alpine now support suricata incl. enabled JA3 support, move back to Alpine install --- README.md | 7 +- bin/hpfeeds_optin.sh | 21 +-- docker/elk/elasticsearch/Dockerfile | 2 +- .../elk/elasticsearch/dist/elasticsearch.yml | 1 + docker/elk/kibana/Dockerfile | 6 +- docker/elk/logstash/Dockerfile | 4 +- docker/ews/Dockerfile | 2 +- docker/ews/dist/sissden.pem | 70 --------- docker/ews/docker-compose.yml | 2 +- docker/suricata/Dockerfile | 113 +------------- docker/suricata/Dockerfile.old | 139 ++++++++++++++++++ docker/suricata/dist/capture-filter.bpf | 3 +- docker/suricata/docker-compose.yml | 2 +- 13 files changed, 161 insertions(+), 211 deletions(-) delete mode 100644 docker/ews/dist/sissden.pem create mode 100644 docker/suricata/Dockerfile.old diff --git a/README.md b/README.md index e6600dfb..32b7924f 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Furthermore we use the following tools - **Release Cycle** - As far as possible we will integrate changes now faster into the master branch, eliminating the need for monolithic releases. The update feature will be continuously improved on that behalf. However this might not account for all feature changes. - **HPFEEDS Opt-In** - - If you want to share your T-Pot data with a 3rd party HPFEEDS broker such as [SISSDEN](https://sissden.eu) you can do so by creating an account at the SISSDEN portal and run `hpfeeds_optin.sh` on T-Pot. + - If you want to share your T-Pot data with a 3rd party HPFEEDS broker such as you can do so by runnning `hpfeeds_optin.sh` on T-Pot. - **Update Feature** - For the ones who like to live on the bleeding edge of T-Pot development there is now an update script available in `/opt/tpot/update.sh`. - This feature is beta and is mostly intended to provide you with the latest development advances without the need of reinstalling T-Pot. @@ -487,9 +487,8 @@ We encourage you not to disable the data submission as it is the main purpose of ## Opt-In HPFEEDS Data Submission -As an Opt-In it is now possible to also share T-Pot data with 3rd party HPFEEDS brokers, such as [SISSDEN](https://sissden.eu). -If you want to share your T-Pot data you simply have to register an account with a 3rd party broker with its own benefits towards the community. Once registered you will receive your credentials to share events with the broker. In T-Pot you simply run `hpfeeds_optin.sh` which will ask for your credentials, in case of SISSDEN this is just `Ident` and `Secret`, everything else is pre-configured. -It will automatically update `/opt/tpot/etc/tpot.yml` to deliver events to your desired broker. +As an Opt-In it is now possible to also share T-Pot data with 3rd party HPFEEDS brokers. +If you want to share your T-Pot data you simply have to register an account with a 3rd party broker with its own benefits towards the community. You simply run `hpfeeds_optin.sh` which will ask for your credentials. It will automatically update `/opt/tpot/etc/tpot.yml` to deliver events to your desired broker. The script can accept a config file as an argument, e.g. `./hpfeeds_optin.sh --conf=hpfeeds.cfg` diff --git a/bin/hpfeeds_optin.sh b/bin/hpfeeds_optin.sh index dceb9c29..b3821522 100755 --- a/bin/hpfeeds_optin.sh +++ b/bin/hpfeeds_optin.sh @@ -10,20 +10,6 @@ fi myTPOTYMLFILE="/opt/tpot/etc/tpot.yml" -function fuSISSDEN () { -echo -echo "You chose SISSDEN, you just need to provide ident and secret" -echo -myENABLE="true" -myHOST="hpfeeds.sissden.eu" -myPORT="10000" -myCHANNEL="t-pot.events" -myCERT="/opt/ewsposter/sissden.pem" -read -p "Ident: " myIDENT -read -p "Secret: " mySECRET -myFORMAT="json" -} - function fuGENERIC () { echo echo "You chose generic, please provide all the details of the broker" @@ -119,8 +105,7 @@ echo echo echo "Please choose your broker" echo "---------------------------" -echo "[1] - SISSDEN" -echo "[2] - Generic (enter details manually)" +echo "[1] - Generic (enter details manually)" echo "[0] - Opt out of HPFEEDS" echo "[q] - Do not agree end exit" echo @@ -130,10 +115,6 @@ while [ 1 != 2 ] echo $mySELECT case "$mySELECT" in [1]) - fuSISSDEN - break - ;; - [2]) fuGENERIC break ;; diff --git a/docker/elk/elasticsearch/Dockerfile b/docker/elk/elasticsearch/Dockerfile index 45108a4d..7e0bc58f 100644 --- a/docker/elk/elasticsearch/Dockerfile +++ b/docker/elk/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ FROM alpine # # VARS -ENV ES_VER=7.5.2 \ +ENV ES_VER=7.6.0 \ JAVA_HOME=/usr/lib/jvm/java-11-openjdk # Include dist ADD dist/ /root/dist/ diff --git a/docker/elk/elasticsearch/dist/elasticsearch.yml b/docker/elk/elasticsearch/dist/elasticsearch.yml index 62085d47..f0f24d57 100644 --- a/docker/elk/elasticsearch/dist/elasticsearch.yml +++ b/docker/elk/elasticsearch/dist/elasticsearch.yml @@ -9,6 +9,7 @@ path: http.host: 0.0.0.0 http.cors.enabled: true http.cors.allow-origin: "*" +indices.query.bool.max_clause_count: 2000 cluster.initial_master_nodes: - "tpotcluster-node-01" discovery.zen.ping.unicast.hosts: diff --git a/docker/elk/kibana/Dockerfile b/docker/elk/kibana/Dockerfile index 6e6fd837..12804d6c 100644 --- a/docker/elk/kibana/Dockerfile +++ b/docker/elk/kibana/Dockerfile @@ -1,7 +1,7 @@ -FROM node:10.15.2-alpine +FROM node:10.18.0-alpine # # VARS -ENV KB_VER=7.5.2 +ENV KB_VER=7.6.0 # # Include dist ADD dist/ /root/dist/ @@ -47,6 +47,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "elasticsearch.requestTimeout: 60000" >> /usr/share/kibana/config/kibana.yml && \ + echo "elasticsearch.shardTimeout: 60000" >> /usr/share/kibana/config/kibana.yml && \ rm -rf /usr/share/kibana/optimize/bundles/* && \ /usr/share/kibana/bin/kibana --optimize --allow-root && \ addgroup -g 2000 kibana && \ diff --git a/docker/elk/logstash/Dockerfile b/docker/elk/logstash/Dockerfile index 1eb89a43..2e654923 100644 --- a/docker/elk/logstash/Dockerfile +++ b/docker/elk/logstash/Dockerfile @@ -1,7 +1,7 @@ FROM alpine # # VARS -ENV LS_VER=7.5.2 +ENV LS_VER=7.6.0 # Include dist ADD dist/ /root/dist/ # @@ -36,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ chmod u+x /usr/bin/update.sh && \ mkdir -p /etc/logstash/conf.d && \ cp logstash.conf /etc/logstash/conf.d/ && \ - cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.2.3-java/lib/logstash/outputs/elasticsearch/ && \ + cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.3.0-java/lib/logstash/outputs/elasticsearch/ && \ # # Setup user, groups and configs addgroup -g 2000 logstash && \ diff --git a/docker/ews/Dockerfile b/docker/ews/Dockerfile index ead17016..87397ddb 100644 --- a/docker/ews/Dockerfile +++ b/docker/ews/Dockerfile @@ -32,7 +32,7 @@ RUN apk -U --no-cache add \ # # Supply configs mv /root/dist/ews.cfg /opt/ewsposter/ && \ - mv /root/dist/*.pem /opt/ewsposter/ && \ +# mv /root/dist/*.pem /opt/ewsposter/ && \ # # Clean up apk del build-base \ diff --git a/docker/ews/dist/sissden.pem b/docker/ews/dist/sissden.pem deleted file mode 100644 index ae822f5f..00000000 --- a/docker/ews/dist/sissden.pem +++ /dev/null @@ -1,70 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIGBDCCA+ygAwIBAgIBATANBgkqhkiG9w0BAQsFADCBnTEYMBYGA1UEAwwPU0lT -U0RFTiBSb290IENBMQswCQYDVQQGEwJQTDERMA8GA1UEBwwIV2Fyc3phd2ExLjAs -BgNVBAoMJU5hdWtvd2EgaSBBa2FkZW1pY2thIFNpZWMgS29tcHV0ZXJvd2ExEDAO -BgNVBAsMB1NJU1NERU4xHzAdBgkqhkiG9w0BCQEWEGFkbWluQHNpc3NkZW4uZXUw -HhcNMTcwNDExMTMxNDE2WhcNMjcwNDA5MTMxNDE2WjCBjTEbMBkGA1UEAwwSU0lT -U0RFTiBTZXJ2aWNlIENBMQswCQYDVQQGEwJQTDEfMB0GCSqGSIb3DQEJARYQYWRt -aW5Ac2lzc2Rlbi5ldTEuMCwGA1UECgwlTmF1a293YSBpIEFrYWRlbWlja2EgU2ll -YyBLb21wdXRlcm93YTEQMA4GA1UECwwHU0lTU0RFTjCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBAPFLjU6cLQoGz1s73QMPiRxYISCMUh3CXFe52Uim9a60 -nkBDLfjMFW87MNhFCcE2xmxwdPPTz4+f5+DsEV3eZf0y63NxWx+RFV+UpODuEW5n -tWPFUDxmgKx6iAR/tyeLVNqmgtCnWzSthE0cg71dlil6onWvkMc+Wn5Kv6aXoz4e -5YVVhNsymhhrR0BntospY8EvtPm70hHAzOty957/zixOQ/MM+4SHRsWXTlKqv0K2 -udWpkUy1Ihs3bpea2KAvn9bBWejFwy7K4q3LyhSyqwpVCYjNi+s+9z4ipSMfvAlT -FvHrMrODv/Iz/TQOfypYSlpX2gBP9WKLgOQj3wulJnMDQlvG1XNgOAqKfEF52YGF -eUu21UraRgDAguIIhWxRwgXenmRo8ngWjfk9Q8734PzzXt8cwzbxJWiJLMew1SiW -I+Kg8uYNGNT4mdBeUMo92S17ZNMXVnkt1TYfxT0A0ZlTCrhXPiWITtsVZXAdqFtl -j5hASmEcRYNgXEUQHBn13O9IinEmks2PEcqbbbKbs2Je0DS/JvxBkqES51UdsaVQ -zITKw3deCk0pISG8WDWZ97LEeDCvAKA5l/ooKjDwfS5vWw11mTUCOdhCoF0m8Lao -TwE1fzzNbSaqMsT6JF/n0ACabfuvF2aqCmWsZC/Hpw8LQQS62zOouCLdcqizL9+z -AgMBAAGjXTBbMAwGA1UdEwQFMAMBAf8wCwYDVR0PBAQDAgHuMB0GA1UdDgQWBBQ4 -nurxBppBA5PTNvFFU/vhDr/NFzAfBgNVHSMEGDAWgBSDpRyQSgaBD5XvyFOA8YHH -tbUAbzANBgkqhkiG9w0BAQsFAAOCAgEAIvA2gkYsIVH7FGuoIo9RIxgwy7G/SHNC -Xllz6hyTx10UwbttJ+o4gdNt8WPuGnkmywFgsjL1//bFw2+fUO5IRvWKSmXzwx9N -faRJAjQT4JNx2uOW0ctw4USngPrLjXr3UrIQQlJFtZnEyT9u5VJXX8zkhfNJudyJ -N88YVrPEf6Gh1Q0P+yCX0rDEb3PlP2jsYyXZtcYA5kDQ6Qq7jpLT/zrjJdaPTmzh -2NUe7jJOBfZxPCoeev7meafY2vVOgqRqMz1+DZRoOgwq+ysczzRaXmd5a2p9Tabc -L1w5FXKNJQ4apszA0cEScI+4mBIIQ7VFT3GO098GOcYsC2MelRkgONAIyamm66AP -tvLQAKoiK/xz3sEHN4zaZvN/YVHaSYZEXUP0QHdyL62P62a92aCNyrHpzKURhEDA -n8cs6icxKrS4xuVa517m53zun0brjrfeltfbO7z1A2TstFYu9BHKzRuhwV9cGRHP -EDcb7PkfA/08sDHsyfsWtzIysNo3hwCmQ6gtOW5xlrGplFfwSsXmPG4SR3ByW379 -RA5h3zzrO0g7iCvbLclqHoqLTJTMS+6U43qXjnQ7DJ+mcbhRGcMHcZVKqO3QmLm+ -mmkDNzNYfTgY52D5mXJqUK50750mQ8dwMSkD2TufSAPmAPUp90LdQ8u9CIv6gQ+x -A08hDHJ1cdY= ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIGHDCCBASgAwIBAgIJAPZqsOOroxaHMA0GCSqGSIb3DQEBCwUAMIGdMRgwFgYD -VQQDDA9TSVNTREVOIFJvb3QgQ0ExCzAJBgNVBAYTAlBMMREwDwYDVQQHDAhXYXJz -emF3YTEuMCwGA1UECgwlTmF1a293YSBpIEFrYWRlbWlja2EgU2llYyBLb21wdXRl -cm93YTEQMA4GA1UECwwHU0lTU0RFTjEfMB0GCSqGSIb3DQEJARYQYWRtaW5Ac2lz -c2Rlbi5ldTAeFw0xNzA0MTExMzA3NTZaFw0yNzA0MDkxMzA3NTZaMIGdMRgwFgYD -VQQDDA9TSVNTREVOIFJvb3QgQ0ExCzAJBgNVBAYTAlBMMREwDwYDVQQHDAhXYXJz -emF3YTEuMCwGA1UECgwlTmF1a293YSBpIEFrYWRlbWlja2EgU2llYyBLb21wdXRl -cm93YTEQMA4GA1UECwwHU0lTU0RFTjEfMB0GCSqGSIb3DQEJARYQYWRtaW5Ac2lz -c2Rlbi5ldTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANKT77EYYEhV -tJUnfnvQtGttfgqIzKIV2W6nPK9aDsKRTX5BVDHF6P5ZAF1u/52ATwdyTK7+LD66 -Q/nCzyyA2kqTgdruX6VGucpD2DVVSVF6nZhV9PcISNaMXytoG2HHlqrim53E/rVa -rskColfs7oCxama6lPKZ/rqrJlVjA1Pl5ZtxR0IORjpOyZjSbSzKQwLp/JxHPMCU -2cVirS7aEu5UGj+Q7Ibg0AEyoAu5tnHBKun4hmIoo7LtKWNEe1TdboxOSboGJ5wd -UTEmNH+7izZ5FAogTUINjubkf2zZ65xEnN7DT/zFS30vYU1EclqCTp96EKPANogV -ZeBKntEN6M5azM6Q6+nFI56TV5DWHTIXm85zzeDj5JM7TQlIGTh8A5APHpr0YyUP -AiIUrixV2lqSDrjewey5qQcWV6WbjMS72OFKh/x7+UJICJhoUw+KwnPmWSq1WAlt -n7C+W0raSQzt7puI30LUkInKL6iEQebMoYg0eDRI5vsRIpbo+PzflIuk/Vea/D1Y -twgRc8ujoKI9GpPJyP4yO4nY7BkShLqKJ251lEJZnxq8LiFVi8aN6ZHt//OGEtVs -6L97cPzqFx7qx8vnyLBFk23lb8pilHK1G0nqxCCjakTruT/JgkLXnZcLu/IDSqd3 -QLjJL0rmU9q6+RTH8A782pcBUNzeLKnlAgMBAAGjXTBbMAwGA1UdEwQFMAMBAf8w -CwYDVR0PBAQDAgHuMB0GA1UdDgQWBBSDpRyQSgaBD5XvyFOA8YHHtbUAbzAfBgNV -HSMEGDAWgBSDpRyQSgaBD5XvyFOA8YHHtbUAbzANBgkqhkiG9w0BAQsFAAOCAgEA -IA0U6znfPykr5PoQlXb/Wr4L5mY/ZtNAJsvJ8jwNMsj3ZlqLOJfnHHoG5LHkb2b/ -xfM1Ee2ojmYBt4VDARqrHLLbup38Ivqt0aEco3Qx/WqbIR4IlvZBF+/qKF/wIUuc -CuBYNIy12PcLzafT+SJosj1BJ+XiUCj/RsVXIT5CxsdXIABWC+5b3T3/PrAtKk+C -sVjA/ck1KAHDd+3VUyRjLAAekYWA9C/hek3YwWQ3OvmyHos5gxifqMMDj6bx5qgv -AuIs4mYJlBlHE19GxRmo2TDwE0eZiUoUdavdRBbl9v7dex+AF2GegmnC1ouYc9kv -9moNBcuPFXuJMCOCU44aTpgEKRm3QTZTvVcUza251T+4kgT2wlFyzPqQ8hcpih4t -knlqHhNc9ibL3/qzWr093AgC9uNaNRqmqu1WAu3vs9g3DVb/RSMrUG/V0YS1GgPq -E+nVJ1AIJoee8YaxHztRfjPsmu1R3pp633lfcRPUKCkz52dZDFRPuQP36DuJzl2M -itTra0MtDUuRCsuJfVGe1op2wFprswLI0qy7O9N21D4Ab8g0ik+lhmpOf5DpYxmx -C2Xpe4d/5Xlg3wIYhEs5MnfeEy4lSMA4cxwJs11gVYHba62L7/5lqzpPmHdRYHu3 -Vf0pM/6zniQpy58Pf9+9CNU15I3iWF5K3zmevFArd6s= ------END CERTIFICATE----- diff --git a/docker/ews/docker-compose.yml b/docker/ews/docker-compose.yml index 2b5d2110..1900e1d3 100644 --- a/docker/ews/docker-compose.yml +++ b/docker/ews/docker-compose.yml @@ -23,7 +23,7 @@ services: - EWS_HPFEEDS_FORMAT=json env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/ewsposter:1903" + image: "dtagdevsec/ewsposter:2006" volumes: - /data:/data - /data/ews/conf/ews.ip:/opt/ewsposter/ews.ip diff --git a/docker/suricata/Dockerfile b/docker/suricata/Dockerfile index d7f776a8..65c3ab9c 100644 --- a/docker/suricata/Dockerfile +++ b/docker/suricata/Dockerfile @@ -1,90 +1,17 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ # # Install packages -#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ -RUN apk -U add \ +RUN apk -U --no-cache add \ ca-certificates \ curl \ file \ - geoip \ - hiredis \ - jansson \ - libcap-ng \ - libmagic \ - libmaxminddb \ - libnet \ - libnetfilter_queue \ - libnfnetlink \ - libpcap \ - luajit \ - lz4-libs \ - musl \ - nspr \ - nss \ - pcre \ - yaml \ - wget \ - automake \ - autoconf \ - build-base \ - cargo \ - file-dev \ - geoip-dev \ - hiredis-dev \ - jansson-dev \ - libtool \ - libcap-ng-dev \ - luajit-dev \ - libmaxminddb-dev \ - libpcap-dev \ - libnet-dev \ - libnetfilter_queue-dev \ - libnfnetlink-dev \ - lz4-dev \ - nss-dev \ - nspr-dev \ - pcre-dev \ - python3 \ - rust \ - yaml-dev && \ -# -# We need latest libhtp[-dev] which is only available in community - apk -U add --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \ - libhtp \ - libhtp-dev && \ -# -# Upgrade pip, install suricata-update to meet deps, however we will not be using it -# to reduce image (no python needed) and use the update script. - pip3 install --no-cache-dir --upgrade pip && \ - pip3 install --no-cache-dir suricata-update && \ -# -# Get and build Suricata - mkdir -p /opt/builder/ && \ - wget https://www.openinfosecfoundation.org/download/suricata-5.0.0.tar.gz && \ - tar xvfz suricata-5.0.0.tar.gz --strip-components=1 -C /opt/builder/ && \ - rm suricata-5.0.0.tar.gz && \ - cd /opt/builder && \ - ./configure \ - --prefix=/usr \ - --sysconfdir=/etc \ - --mandir=/usr/share/man \ - --localstatedir=/var \ - --enable-non-bundled-htp \ - --enable-nfqueue \ - --enable-rust \ - --disable-gccmarch-native \ - --enable-hiredis \ - --enable-geoip \ - --enable-gccprotect \ - --enable-pie \ - --enable-luajit && \ - make && \ - make check && \ - make install && \ - make install-full && \ + libcap \ + wget && \ + apk -U add --repository http://dl-cdn.alpinelinux.org/alpine/edge/community \ + suricata && \ # # Setup user, groups and configs addgroup -g 2000 suri && \ @@ -92,8 +19,6 @@ RUN apk -U add \ chmod 644 /etc/suricata/*.config && \ cp /root/dist/suricata.yaml /etc/suricata/suricata.yaml && \ cp /root/dist/*.bpf /etc/suricata/ && \ - mkdir -p /etc/suricata/rules && \ - cp /opt/builder/rules/* /etc/suricata/rules/ && \ # # Download the latest EmergingThreats ruleset, replace rulebase and enable all rules cp /root/dist/update.sh /usr/bin/ && \ @@ -101,32 +26,6 @@ RUN apk -U add \ update.sh OPEN && \ # # Clean up - apk del --purge \ - automake \ - autoconf \ - build-base \ - cargo \ - file-dev \ - geoip-dev \ - hiredis-dev \ - jansson-dev \ - libtool \ - libhtp-dev \ - libcap-ng-dev \ - luajit-dev \ - libpcap-dev \ - libmaxminddb-dev \ - libnet-dev \ - libnetfilter_queue-dev \ - libnfnetlink-dev \ - lz4-dev \ - nss-dev \ - nspr-dev \ - pcre-dev \ - python3 \ - rust \ - yaml-dev && \ - rm -rf /opt/builder && \ rm -rf /root/* && \ rm -rf /tmp/* && \ rm -rf /var/cache/apk/* diff --git a/docker/suricata/Dockerfile.old b/docker/suricata/Dockerfile.old new file mode 100644 index 00000000..59c2687a --- /dev/null +++ b/docker/suricata/Dockerfile.old @@ -0,0 +1,139 @@ +FROM alpine +# +# VARS +ENV VER=5.0.2 +# +# Include dist +ADD dist/ /root/dist/ +# +# Install packages +#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ +RUN apk -U add \ + ca-certificates \ + curl \ + file \ + geoip \ + hiredis \ + jansson \ + libcap-ng \ + libmagic \ + libmaxminddb \ + libnet \ + libnetfilter_queue \ + libnfnetlink \ + libpcap \ + luajit \ + lz4-libs \ + musl \ + nspr \ + nss \ + pcre \ + yaml \ + wget \ + automake \ + autoconf \ + build-base \ + cargo \ + file-dev \ + geoip-dev \ + hiredis-dev \ + jansson-dev \ + libtool \ + libcap-ng-dev \ + luajit-dev \ + libmaxminddb-dev \ + libpcap-dev \ + libnet-dev \ + libnetfilter_queue-dev \ + libnfnetlink-dev \ + lz4-dev \ + nss-dev \ + nspr-dev \ + pcre-dev \ + python3 \ + rust \ + yaml-dev && \ +# +# We need latest libhtp[-dev] which is only available in community + apk -U add --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \ + libhtp \ + libhtp-dev && \ +# +# Upgrade pip, install suricata-update to meet deps, however we will not be using it +# to reduce image (no python needed) and use the update script. + pip3 install --no-cache-dir --upgrade pip && \ + pip3 install --no-cache-dir suricata-update && \ +# +# Get and build Suricata + mkdir -p /opt/builder/ && \ + wget https://www.openinfosecfoundation.org/download/suricata-$VER.tar.gz && \ + tar xvfz suricata-$VER.tar.gz --strip-components=1 -C /opt/builder/ && \ + rm suricata-$VER.tar.gz && \ + cd /opt/builder && \ + ./configure \ + --prefix=/usr \ + --sysconfdir=/etc \ + --mandir=/usr/share/man \ + --localstatedir=/var \ + --enable-non-bundled-htp \ + --enable-nfqueue \ + --enable-rust \ + --disable-gccmarch-native \ + --enable-hiredis \ + --enable-geoip \ + --enable-gccprotect \ + --enable-pie \ + --enable-luajit && \ + make && \ + make check && \ + make install && \ + make install-full && \ +# +# Setup user, groups and configs + addgroup -g 2000 suri && \ + adduser -S -H -u 2000 -D -g 2000 suri && \ + chmod 644 /etc/suricata/*.config && \ + cp /root/dist/suricata.yaml /etc/suricata/suricata.yaml && \ + cp /root/dist/*.bpf /etc/suricata/ && \ + mkdir -p /etc/suricata/rules && \ + cp /opt/builder/rules/* /etc/suricata/rules/ && \ +# +# Download the latest EmergingThreats ruleset, replace rulebase and enable all rules + cp /root/dist/update.sh /usr/bin/ && \ + chmod 755 /usr/bin/update.sh && \ + update.sh OPEN && \ +# +# Clean up + apk del --purge \ + automake \ + autoconf \ + build-base \ + cargo \ + file-dev \ + geoip-dev \ + hiredis-dev \ + jansson-dev \ + libtool \ + libhtp-dev \ + libcap-ng-dev \ + luajit-dev \ + libpcap-dev \ + libmaxminddb-dev \ + libnet-dev \ + libnetfilter_queue-dev \ + libnfnetlink-dev \ + lz4-dev \ + nss-dev \ + nspr-dev \ + pcre-dev \ + python3 \ + rust \ + yaml-dev && \ + rm -rf /opt/builder && \ + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Start suricata +STOPSIGNAL SIGINT +CMD SURICATA_CAPTURE_FILTER=$(update.sh $OINKCODE) && exec suricata -v -F $SURICATA_CAPTURE_FILTER -i $(/sbin/ip address | grep '^2: ' | awk '{ print $2 }' | tr -d [:punct:]) diff --git a/docker/suricata/dist/capture-filter.bpf b/docker/suricata/dist/capture-filter.bpf index 9f798d44..4b289a00 100644 --- a/docker/suricata/dist/capture-filter.bpf +++ b/docker/suricata/dist/capture-filter.bpf @@ -1,4 +1,3 @@ not (host sicherheitstacho.eu or community.sicherheitstacho.eu) and not (host deb.debian.org) and -not (host index.docker.io or docker.io) and -not (host hpfeeds.sissden.eu) +not (host index.docker.io or docker.io) diff --git a/docker/suricata/docker-compose.yml b/docker/suricata/docker-compose.yml index b1913e58..4568fba9 100644 --- a/docker/suricata/docker-compose.yml +++ b/docker/suricata/docker-compose.yml @@ -15,6 +15,6 @@ services: - NET_ADMIN - SYS_NICE - NET_RAW - image: "dtagdevsec/suricata:1903" + image: "dtagdevsec/suricata:2006" volumes: - /data/suricata/log:/var/log/suricata From aa4a93684dcbf40d96c2a17fb1e53bee04fea23b Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 14 Feb 2020 15:30:55 +0000 Subject: [PATCH 10/31] bump more images to 20.06 --- etc/compose/nextgen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index bb6381d2..7db4c4b1 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -455,7 +455,7 @@ services: - NET_ADMIN - SYS_NICE - NET_RAW - image: "dtagdevsec/suricata:1903" + image: "dtagdevsec/suricata:2006" volumes: - /data/suricata/log:/var/log/suricata @@ -553,7 +553,7 @@ services: - EWS_HPFEEDS_FORMAT=json env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/ewsposter:1903" + image: "dtagdevsec/ewsposter:2006" volumes: - /data:/data - /data/ews/conf/ews.ip:/opt/ewsposter/ews.ip From faa56672465514049043411988cefbafae6d1943 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 14 Feb 2020 17:22:30 +0000 Subject: [PATCH 11/31] bump adbhoney, cowrie, honeytrap to 20.06 --- docker/adbhoney/Dockerfile | 5 +++-- docker/adbhoney/docker-compose.yml | 2 +- docker/cowrie/Dockerfile | 9 +++++---- docker/cowrie/docker-compose.yml | 2 +- docker/honeytrap/Dockerfile | 3 ++- docker/honeytrap/docker-compose.yml | 2 +- etc/compose/nextgen.yml | 4 ++-- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docker/adbhoney/Dockerfile b/docker/adbhoney/Dockerfile index 20c0d111..88f2456a 100644 --- a/docker/adbhoney/Dockerfile +++ b/docker/adbhoney/Dockerfile @@ -4,7 +4,8 @@ FROM alpine ADD dist/ /root/dist/ # # Install packages -RUN apk -U add \ +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U add \ git \ libcap \ python3 \ @@ -20,7 +21,7 @@ RUN apk -U add \ addgroup -g 2000 adbhoney && \ adduser -S -H -s /bin/ash -u 2000 -D -g 2000 adbhoney && \ chown -R adbhoney:adbhoney /opt/adbhoney && \ - setcap cap_net_bind_service=+ep /usr/bin/python3.7 && \ + setcap cap_net_bind_service=+ep /usr/bin/python3.8 && \ # # Clean up apk del --purge git \ diff --git a/docker/adbhoney/docker-compose.yml b/docker/adbhoney/docker-compose.yml index 56d6a497..58e62f11 100644 --- a/docker/adbhoney/docker-compose.yml +++ b/docker/adbhoney/docker-compose.yml @@ -14,7 +14,7 @@ services: - adbhoney_local ports: - "5555:5555" - image: "dtagdevsec/adbhoney:1903" + image: "dtagdevsec/adbhoney:2006" read_only: true volumes: - /data/adbhoney/log:/opt/adbhoney/log diff --git a/docker/cowrie/Dockerfile b/docker/cowrie/Dockerfile index d4e56572..0158c127 100644 --- a/docker/cowrie/Dockerfile +++ b/docker/cowrie/Dockerfile @@ -4,7 +4,8 @@ FROM alpine ADD dist/ /root/dist/ # # Get and install dependencies & packages -RUN apk -U add \ +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U add \ bash \ build-base \ git \ @@ -29,16 +30,16 @@ RUN apk -U add \ # Install cowrie mkdir -p /home/cowrie && \ cd /home/cowrie && \ - git clone --depth=1 https://github.com/micheloosterhof/cowrie -b v2.0.0 && \ + git clone --depth=1 https://github.com/micheloosterhof/cowrie -b v2.0.2 && \ cd cowrie && \ mkdir -p log && \ pip3 install --upgrade pip && \ pip3 install --upgrade -r requirements.txt && \ # # Setup configs - setcap cap_net_bind_service=+ep /usr/bin/python3.7 && \ + setcap cap_net_bind_service=+ep /usr/bin/python3.8 && \ cp /root/dist/cowrie.cfg /home/cowrie/cowrie/cowrie.cfg && \ - chown cowrie:cowrie -R /home/cowrie/* /usr/lib/python3.7/site-packages/twisted/plugins && \ + chown cowrie:cowrie -R /home/cowrie/* /usr/lib/python3.8/site-packages/twisted/plugins && \ # # Start Cowrie once to prevent dropin.cache errors upon container start caused by read-only filesystem su - cowrie -c "export PYTHONPATH=/home/cowrie/cowrie:/home/cowrie/cowrie/src && \ diff --git a/docker/cowrie/docker-compose.yml b/docker/cowrie/docker-compose.yml index 7fc90ebf..181a9bd7 100644 --- a/docker/cowrie/docker-compose.yml +++ b/docker/cowrie/docker-compose.yml @@ -18,7 +18,7 @@ services: ports: - "22:22" - "23:23" - image: "dtagdevsec/cowrie:1903" + image: "dtagdevsec/cowrie:2006" read_only: true volumes: - /data/cowrie/downloads:/home/cowrie/cowrie/dl diff --git a/docker/honeytrap/Dockerfile b/docker/honeytrap/Dockerfile index 6546b6c8..ff560283 100644 --- a/docker/honeytrap/Dockerfile +++ b/docker/honeytrap/Dockerfile @@ -27,7 +27,8 @@ RUN apt-get update -y && \ # # Install honeytrap from source cd /root/ && \ - git clone https://github.com/armedpot/honeytrap && \ +# git clone https://github.com/armedpot/honeytrap && \ + git clone https://github.com/t3chn0m4g3/honeytrap-1 /root/honeytrap && \ cd /root/honeytrap/ && \ autoreconf -vfi && \ ./configure \ diff --git a/docker/honeytrap/docker-compose.yml b/docker/honeytrap/docker-compose.yml index 15eb3d73..7573b3d5 100644 --- a/docker/honeytrap/docker-compose.yml +++ b/docker/honeytrap/docker-compose.yml @@ -12,7 +12,7 @@ services: network_mode: "host" cap_add: - NET_ADMIN - image: "dtagdevsec/honeytrap:1903" + image: "dtagdevsec/honeytrap:2006" read_only: true volumes: - /data/honeytrap/attacks:/opt/honeytrap/var/attacks diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 7db4c4b1..171b539a 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -34,7 +34,7 @@ services: - adbhoney_local ports: - "5555:5555" - image: "dtagdevsec/adbhoney:1903" + image: "dtagdevsec/adbhoney:2006" read_only: true volumes: - /data/adbhoney/log:/opt/adbhoney/log @@ -166,7 +166,7 @@ services: ports: - "22:22" - "23:23" - image: "dtagdevsec/cowrie:1903" + image: "dtagdevsec/cowrie:2006" read_only: true volumes: - /data/cowrie/downloads:/home/cowrie/cowrie/dl From 153f7be9dcb69a8025e61614b2d3ea11b65e2b6b Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 14 Feb 2020 17:26:53 +0000 Subject: [PATCH 12/31] cleanup --- docker/honeytrap/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/honeytrap/Dockerfile b/docker/honeytrap/Dockerfile index ff560283..6546b6c8 100644 --- a/docker/honeytrap/Dockerfile +++ b/docker/honeytrap/Dockerfile @@ -27,8 +27,7 @@ RUN apt-get update -y && \ # # Install honeytrap from source cd /root/ && \ -# git clone https://github.com/armedpot/honeytrap && \ - git clone https://github.com/t3chn0m4g3/honeytrap-1 /root/honeytrap && \ + git clone https://github.com/armedpot/honeytrap && \ cd /root/honeytrap/ && \ autoreconf -vfi && \ ./configure \ From 545209dce6ab57102fca4507e3beb1e30d542ce0 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Sat, 15 Feb 2020 15:40:47 +0000 Subject: [PATCH 13/31] fix for honeytrap --- docker/honeytrap/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/honeytrap/Dockerfile b/docker/honeytrap/Dockerfile index 6546b6c8..fbd0c110 100644 --- a/docker/honeytrap/Dockerfile +++ b/docker/honeytrap/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stretch-slim +FROM debian:buster-slim ENV DEBIAN_FRONTEND noninteractive # # Include dist @@ -26,8 +26,8 @@ RUN apt-get update -y && \ wget && \ # # Install honeytrap from source - cd /root/ && \ - git clone https://github.com/armedpot/honeytrap && \ +# git clone https://github.com/armedpot/honeytrap && \ + git clone https://github.com/t3chn0m4g3/honeytrap /root/honeytrap && \ cd /root/honeytrap/ && \ autoreconf -vfi && \ ./configure \ From bd0e6936eb5e4ad5d6b049fd469fa3b5fa7b5884 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Fri, 21 Feb 2020 11:38:29 +0000 Subject: [PATCH 14/31] bump heralding to latest master fixed by https://github.com/johnnykv/heralding/issues/129#event-3058184614 --- docker/heralding/Dockerfile | 5 ++--- docker/heralding/docker-compose.yml | 2 +- etc/compose/nextgen.yml | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/heralding/Dockerfile b/docker/heralding/Dockerfile index b03a7ec0..da95c7ee 100644 --- a/docker/heralding/Dockerfile +++ b/docker/heralding/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.10 +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ @@ -23,7 +23,6 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ cd /opt/ && \ git clone --depth=1 https://github.com/johnnykv/heralding && \ cd heralding && \ - sed -i 's/asyncssh/asyncssh==1.18.0/' requirements.txt && \ pip3 install --no-cache-dir -r requirements.txt && \ pip3 install --no-cache-dir . && \ # @@ -32,7 +31,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ adduser -S -H -s /bin/ash -u 2000 -D -g 2000 heralding && \ mkdir -p /var/log/heralding/ /etc/heralding && \ mv /root/dist/heralding.yml /etc/heralding/ && \ - setcap cap_net_bind_service=+ep /usr/bin/python3.7 && \ + setcap cap_net_bind_service=+ep /usr/bin/python3.8 && \ chown -R heralding:heralding /var/log/heralding && \ # # Clean up diff --git a/docker/heralding/docker-compose.yml b/docker/heralding/docker-compose.yml index 6f5188e1..15f92661 100644 --- a/docker/heralding/docker-compose.yml +++ b/docker/heralding/docker-compose.yml @@ -30,7 +30,7 @@ services: - "3389:3389" - "5432:5432" - "5900:5900" - image: "dtagdevsec/heralding:1903" + image: "dtagdevsec/heralding:2006" read_only: true volumes: - /data/heralding/log:/var/log/heralding diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 171b539a..64838709 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -250,7 +250,7 @@ services: - "1080:1080" - "5432:5432" - "5900:5900" - image: "dtagdevsec/heralding:1903" + image: "dtagdevsec/heralding:2006" read_only: true volumes: - /data/heralding/log:/var/log/heralding From 6bfcf8b1c400ae1a310790c2cca185e7d7858e48 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 24 Feb 2020 16:43:34 +0000 Subject: [PATCH 15/31] tweaking --- docker/adbhoney/Dockerfile | 2 +- docker/ciscoasa/Dockerfile | 6 +++--- docker/ciscoasa/docker-compose.yml | 2 +- docker/citrixhoneypot/Dockerfile | 5 +++-- docker/citrixhoneypot/docker-compose.yml | 2 +- docker/conpot/docker-compose.yml | 10 +++++----- docker/cyberchef/docker-compose.yml | 2 +- iso/installer/install.sh | 4 ++-- iso/preseed/tpot.seed | 2 +- update.sh | 2 +- 10 files changed, 19 insertions(+), 18 deletions(-) diff --git a/docker/adbhoney/Dockerfile b/docker/adbhoney/Dockerfile index 88f2456a..635858c8 100644 --- a/docker/adbhoney/Dockerfile +++ b/docker/adbhoney/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ diff --git a/docker/ciscoasa/Dockerfile b/docker/ciscoasa/Dockerfile index 5bdcd20f..e5e110ba 100644 --- a/docker/ciscoasa/Dockerfile +++ b/docker/ciscoasa/Dockerfile @@ -1,10 +1,11 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ # # Setup env and apt -RUN apk -U upgrade && \ +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U upgrade && \ apk add build-base \ git \ libffi \ @@ -23,7 +24,6 @@ RUN apk -U upgrade && \ cd /opt/ && \ git clone --depth=1 https://github.com/cymmetria/ciscoasa_honeypot && \ cd ciscoasa_honeypot && \ - pip3 install --no-cache-dir --upgrade pip && \ pip3 install --no-cache-dir -r requirements.txt && \ cp /root/dist/asa_server.py /opt/ciscoasa_honeypot && \ chown -R ciscoasa:ciscoasa /opt/ciscoasa_honeypot && \ diff --git a/docker/ciscoasa/docker-compose.yml b/docker/ciscoasa/docker-compose.yml index 2d4e2d1f..bf85bc48 100644 --- a/docker/ciscoasa/docker-compose.yml +++ b/docker/ciscoasa/docker-compose.yml @@ -13,7 +13,7 @@ services: ports: - "5000:5000/udp" - "8443:8443" - image: "dtagdevsec/ciscoasa:1903" + image: "dtagdevsec/ciscoasa:2006" read_only: true volumes: - /data/ciscoasa/log:/var/log/ciscoasa diff --git a/docker/citrixhoneypot/Dockerfile b/docker/citrixhoneypot/Dockerfile index f1e43882..d72a0ef4 100644 --- a/docker/citrixhoneypot/Dockerfile +++ b/docker/citrixhoneypot/Dockerfile @@ -1,7 +1,8 @@ -FROM alpine +FROM alpine:latest # # Install packages -RUN apk -U add \ +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U add \ git \ libcap \ openssl \ diff --git a/docker/citrixhoneypot/docker-compose.yml b/docker/citrixhoneypot/docker-compose.yml index c61cd140..16eea88f 100644 --- a/docker/citrixhoneypot/docker-compose.yml +++ b/docker/citrixhoneypot/docker-compose.yml @@ -14,7 +14,7 @@ services: - citrixhoneypot_local ports: - "443:443" - image: "dtagdevsec/citrixhoneypot:1903" + image: "dtagdevsec/citrixhoneypot:2006" read_only: true volumes: - /data/citrixhoneypot/logs:/opt/citrixhoneypot/logs diff --git a/docker/conpot/docker-compose.yml b/docker/conpot/docker-compose.yml index 0187c19e..4b315497 100644 --- a/docker/conpot/docker-compose.yml +++ b/docker/conpot/docker-compose.yml @@ -35,7 +35,7 @@ services: - "2121:21" - "44818:44818" - "47808:47808" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -58,7 +58,7 @@ services: ports: # - "161:161" - "2404:2404" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -80,7 +80,7 @@ services: - conpot_local_guardian_ast ports: - "10001:10001" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -102,7 +102,7 @@ services: - conpot_local_ipmi ports: - "623:623" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -125,7 +125,7 @@ services: ports: - "1025:1025" - "50100:50100" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot diff --git a/docker/cyberchef/docker-compose.yml b/docker/cyberchef/docker-compose.yml index 0bdf0e77..6bb8c3b9 100644 --- a/docker/cyberchef/docker-compose.yml +++ b/docker/cyberchef/docker-compose.yml @@ -14,5 +14,5 @@ services: - cyberchef_local ports: - "127.0.0.1:64299:8000" - image: "dtagdevsec/cyberchef:1903" + image: "dtagdevsec/cyberchef:2006" read_only: true diff --git a/iso/installer/install.sh b/iso/installer/install.sh index cb92db9b..48cccd4e 100755 --- a/iso/installer/install.sh +++ b/iso/installer/install.sh @@ -13,8 +13,8 @@ myTPOTCOMPOSE="/opt/tpot/etc/tpot.yml" myLSB_STABLE_SUPPORTED="stretch buster" myLSB_TESTING_SUPPORTED="sid" myREMOTESITES="https://hub.docker.com https://github.com https://pypi.python.org https://debian.org" -myPREINSTALLPACKAGES="aria2 apache2-utils curl dialog figlet fuse grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet" -myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" +myPREINSTALLPACKAGES="aria2 apache2-utils cracklib-check curl dialog figlet fuse grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet" +myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux cracklib-check curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" myINFO="\ ######################################## ### T-Pot Installer for Debian (Sid) ### diff --git a/iso/preseed/tpot.seed b/iso/preseed/tpot.seed index 32aafd12..12835a5c 100755 --- a/iso/preseed/tpot.seed +++ b/iso/preseed/tpot.seed @@ -109,7 +109,7 @@ tasksel tasksel/first multiselect ssh-server ######################## ### Package Installation ######################## -d-i pkgsel/include string apache2-utils curl dialog figlet git grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet +d-i pkgsel/include string apache2-utils cracklib-check curl dialog figlet git grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet popularity-contest popularity-contest/participate boolean false ################# diff --git a/update.sh b/update.sh index 50dbbdea..ee4f6a8c 100755 --- a/update.sh +++ b/update.sh @@ -183,7 +183,7 @@ function fuUPDATER () { export DEBIAN_FRONTEND=noninteractive echo "### Installing apt-fast" /bin/bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)" -local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" +local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux cracklib-check curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" echo "### Now upgrading packages ..." dpkg --configure -a apt-fast -y autoclean From 70dca02ce41724496cc1d79ca203d883dd561ece Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Tue, 25 Feb 2020 16:59:22 +0000 Subject: [PATCH 16/31] tweaking --- docker/{ => deprecated}/glastopf/Dockerfile | 0 docker/{ => deprecated}/glastopf/README.md | 0 .../glastopf/dist/glastopf.cfg | 0 .../glastopf/dist/requirements.txt | 0 .../glastopf/doc/dashboard.png | Bin .../glastopf/docker-compose.yml | 0 docker/dionaea/docker-compose.yml | 2 +- docker/elasticpot/Dockerfile | 2 +- docker/elasticpot/docker-compose.yml | 2 +- docker/ews/Dockerfile | 7 ++++--- docker/fatt/Dockerfile | 2 +- docker/fatt/docker-compose.yml | 2 +- docker/glutton/Dockerfile | 5 +++-- docker/glutton/docker-compose.yml | 2 +- docker/heimdall/docker-compose.yml | 2 +- docker/honeypy/Dockerfile | 3 ++- docker/honeypy/docker-compose.yml | 2 +- docker/hpfeeds/Dockerfile | 17 +++++++++-------- etc/curator/actions.yml | 6 +++--- 19 files changed, 29 insertions(+), 25 deletions(-) rename docker/{ => deprecated}/glastopf/Dockerfile (100%) rename docker/{ => deprecated}/glastopf/README.md (100%) rename docker/{ => deprecated}/glastopf/dist/glastopf.cfg (100%) rename docker/{ => deprecated}/glastopf/dist/requirements.txt (100%) rename docker/{ => deprecated}/glastopf/doc/dashboard.png (100%) rename docker/{ => deprecated}/glastopf/docker-compose.yml (100%) diff --git a/docker/glastopf/Dockerfile b/docker/deprecated/glastopf/Dockerfile similarity index 100% rename from docker/glastopf/Dockerfile rename to docker/deprecated/glastopf/Dockerfile diff --git a/docker/glastopf/README.md b/docker/deprecated/glastopf/README.md similarity index 100% rename from docker/glastopf/README.md rename to docker/deprecated/glastopf/README.md diff --git a/docker/glastopf/dist/glastopf.cfg b/docker/deprecated/glastopf/dist/glastopf.cfg similarity index 100% rename from docker/glastopf/dist/glastopf.cfg rename to docker/deprecated/glastopf/dist/glastopf.cfg diff --git a/docker/glastopf/dist/requirements.txt b/docker/deprecated/glastopf/dist/requirements.txt similarity index 100% rename from docker/glastopf/dist/requirements.txt rename to docker/deprecated/glastopf/dist/requirements.txt diff --git a/docker/glastopf/doc/dashboard.png b/docker/deprecated/glastopf/doc/dashboard.png similarity index 100% rename from docker/glastopf/doc/dashboard.png rename to docker/deprecated/glastopf/doc/dashboard.png diff --git a/docker/glastopf/docker-compose.yml b/docker/deprecated/glastopf/docker-compose.yml similarity index 100% rename from docker/glastopf/docker-compose.yml rename to docker/deprecated/glastopf/docker-compose.yml diff --git a/docker/dionaea/docker-compose.yml b/docker/dionaea/docker-compose.yml index ac6218a1..052b1976 100644 --- a/docker/dionaea/docker-compose.yml +++ b/docker/dionaea/docker-compose.yml @@ -27,7 +27,7 @@ services: - "5060:5060/udp" - "5061:5061" - "27017:27017" - image: "dtagdevsec/dionaea:1903" + image: "dtagdevsec/dionaea:2006" read_only: true volumes: - /data/dionaea/roots/ftp:/opt/dionaea/var/dionaea/roots/ftp diff --git a/docker/elasticpot/Dockerfile b/docker/elasticpot/Dockerfile index 3e06caf8..e52055bb 100644 --- a/docker/elasticpot/Dockerfile +++ b/docker/elasticpot/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ diff --git a/docker/elasticpot/docker-compose.yml b/docker/elasticpot/docker-compose.yml index 5e336ee8..a8fd3547 100644 --- a/docker/elasticpot/docker-compose.yml +++ b/docker/elasticpot/docker-compose.yml @@ -14,7 +14,7 @@ services: - elasticpot_local ports: - "9200:9200" - image: "dtagdevsec/elasticpot:1903" + image: "dtagdevsec/elasticpot:2006" read_only: true volumes: - /data/elasticpot/log:/opt/ElasticpotPY/log diff --git a/docker/ews/Dockerfile b/docker/ews/Dockerfile index 87397ddb..acd4767d 100644 --- a/docker/ews/Dockerfile +++ b/docker/ews/Dockerfile @@ -1,10 +1,11 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ # # Install packages -RUN apk -U --no-cache add \ +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ build-base \ git \ libffi-dev \ @@ -46,4 +47,4 @@ RUN apk -U --no-cache add \ # Run ewsposter STOPSIGNAL SIGINT USER ews:ews -CMD sleep 10 && exec /usr/bin/python3 -u /opt/ewsposter/ews.py -l $(shuf -i 10-60 -n 1) +CMD sleep 10 && exec /usr/bin/python3 -u /opt/ewsposter/ews.py -l $(shuf -i 10-15 -n 1) diff --git a/docker/fatt/Dockerfile b/docker/fatt/Dockerfile index 387149c0..793679be 100644 --- a/docker/fatt/Dockerfile +++ b/docker/fatt/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Include dist #ADD dist/ /root/dist/ diff --git a/docker/fatt/docker-compose.yml b/docker/fatt/docker-compose.yml index 75cbe6a6..1550ed3a 100644 --- a/docker/fatt/docker-compose.yml +++ b/docker/fatt/docker-compose.yml @@ -12,6 +12,6 @@ services: - NET_ADMIN - SYS_NICE - NET_RAW - image: "dtagdevsec/fatt:1903" + image: "dtagdevsec/fatt:2006" volumes: - /data/fatt/log:/opt/fatt/log diff --git a/docker/glutton/Dockerfile b/docker/glutton/Dockerfile index 38514bc6..34c51835 100644 --- a/docker/glutton/Dockerfile +++ b/docker/glutton/Dockerfile @@ -1,10 +1,11 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ # # Setup apk -RUN apk -U --no-cache add \ +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ build-base \ git \ go \ diff --git a/docker/glutton/docker-compose.yml b/docker/glutton/docker-compose.yml index 5377a6aa..68843e9d 100644 --- a/docker/glutton/docker-compose.yml +++ b/docker/glutton/docker-compose.yml @@ -13,7 +13,7 @@ services: network_mode: "host" cap_add: - NET_ADMIN - image: "dtagdevsec/glutton:1903" + image: "dtagdevsec/glutton:2006" read_only: true volumes: - /data/glutton/log:/var/log/glutton diff --git a/docker/heimdall/docker-compose.yml b/docker/heimdall/docker-compose.yml index 6fb399a3..98346f10 100644 --- a/docker/heimdall/docker-compose.yml +++ b/docker/heimdall/docker-compose.yml @@ -26,7 +26,7 @@ services: ports: - "64297:64297" - "127.0.0.1:64304:64304" - image: "dtagdevsec/nginx:1903" + image: "dtagdevsec/nginx:2006" read_only: true volumes: - /data/nginx/cert/:/etc/nginx/cert/:ro diff --git a/docker/honeypy/Dockerfile b/docker/honeypy/Dockerfile index b61c6886..80d95870 100644 --- a/docker/honeypy/Dockerfile +++ b/docker/honeypy/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ @@ -28,6 +28,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ sed -i 's/bytes/size/g' /opt/honeypy/loggers/file/honeypy_file.py && \ sed -i 's/date_time/timestamp/g' /opt/honeypy/loggers/file/honeypy_file.py && \ sed -i 's/data,/data.decode("hex"),/g' /opt/honeypy/loggers/file/honeypy_file.py && \ + sed -i 's/urllib3/urllib3 == 1.21.1/g' /opt/honeypy/requirements.txt && \ virtualenv env && \ cp /root/dist/services.cfg /opt/honeypy/etc && \ cp /root/dist/honeypy.cfg /opt/honeypy/etc && \ diff --git a/docker/honeypy/docker-compose.yml b/docker/honeypy/docker-compose.yml index 3550622a..dd12fa2d 100644 --- a/docker/honeypy/docker-compose.yml +++ b/docker/honeypy/docker-compose.yml @@ -20,7 +20,7 @@ services: - "2324:2324" - "4096:4096" - "9200:9200" - image: "dtagdevsec/honeypy:1903" + image: "dtagdevsec/honeypy:2006" read_only: true volumes: - /data/honeypy/log:/opt/honeypy/log diff --git a/docker/hpfeeds/Dockerfile b/docker/hpfeeds/Dockerfile index 57c078fe..b0743132 100644 --- a/docker/hpfeeds/Dockerfile +++ b/docker/hpfeeds/Dockerfile @@ -1,10 +1,11 @@ ### This is only for testing purposes, do NOT use for production -FROM alpine - +FROM alpine:latest +# ADD dist/ /root/dist/ - +# # Install packages -RUN apk -U --no-cache add \ +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ build-base \ coreutils \ git \ @@ -15,7 +16,7 @@ RUN apk -U --no-cache add \ python \ python-dev \ sqlite && \ - +# # Install php sandbox from git git clone --depth=1 https://github.com/rep/hpfeeds /opt/hpfeeds && \ cd /opt/hpfeeds/broker && \ @@ -23,10 +24,10 @@ RUN apk -U --no-cache add \ cp /root/dist/adduser.sql . && \ cd /opt/hpfeeds/broker && timeout 5 python broker.py || : && \ sqlite3 db.sqlite3 < adduser.sql && \ - +# #python setup.py build && \ #python setup.py install && \ - +# # Clean up apk del --purge autoconf \ build-base \ @@ -35,7 +36,7 @@ RUN apk -U --no-cache add \ python-dev && \ rm -rf /root/* && \ rm -rf /var/cache/apk/* - +# # Set workdir and start glastopf WORKDIR /opt/hpfeeds/broker CMD python broker.py diff --git a/etc/curator/actions.yml b/etc/curator/actions.yml index fe48bfb9..aaece424 100644 --- a/etc/curator/actions.yml +++ b/etc/curator/actions.yml @@ -16,11 +16,11 @@ actions: disable_action: False filters: - filtertype: pattern - kind: prefix - value: logstash- + kind: timestring + value: '%Y.%m.%d' - filtertype: age source: name direction: older timestring: '%Y.%m.%d' unit: days - unit_count: 90 + unit_count: 60 From bf7d1299ca214d3cb13caf6c45cafedf0b990e4e Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Wed, 26 Feb 2020 14:22:48 +0000 Subject: [PATCH 17/31] tweaking --- docker/{ => deprecated}/hpfeeds/Dockerfile | 0 docker/deprecated/hpfeeds/Dockerfile.old | 42 +++++++++++++ .../{ => deprecated}/hpfeeds/dist/adduser.sql | 0 .../hpfeeds/docker-compose.yml | 0 docker/{ => deprecated}/nginx/Dockerfile | 2 +- .../nginx/dist/conf/nginx.conf | 0 .../nginx/dist/conf/ssl/dhparam4096.pem | 0 .../nginx/dist/conf/ssl/gen-cert.sh | 0 .../nginx/dist/conf/ssl/gen-dhparam.sh | 0 .../nginx/dist/conf/tpotweb.conf | 0 .../nginx/dist/html/error.html | 0 .../nginx/dist/html/favicon.ico | Bin .../nginx/dist/html/navbar.html | 0 .../nginx/dist/html/style.css | 0 .../nginx/dist/html/tpotweb.html | 0 .../{ => deprecated}/nginx/docker-compose.yml | 0 docker/honeypy/Dockerfile | 2 +- docker/mailoney/Dockerfile | 2 +- docker/medpot/Dockerfile | 2 +- docker/medpot/docker-compose.yml | 2 +- docker/p0f/Dockerfile | 2 +- docker/p0f/docker-compose.yml | 2 +- docker/rdpy/Dockerfile | 2 +- docker/rdpy/docker-compose.yml | 2 +- docker/spiderfoot/Dockerfile | 34 +++++++---- docker/spiderfoot/Dockerfile.old | 56 ++++++++++++++++++ docker/spiderfoot/docker-compose.yml | 2 +- docker/tanner/docker-compose.yml | 12 ++-- 28 files changed, 136 insertions(+), 28 deletions(-) rename docker/{ => deprecated}/hpfeeds/Dockerfile (100%) create mode 100644 docker/deprecated/hpfeeds/Dockerfile.old rename docker/{ => deprecated}/hpfeeds/dist/adduser.sql (100%) rename docker/{ => deprecated}/hpfeeds/docker-compose.yml (100%) rename docker/{ => deprecated}/nginx/Dockerfile (97%) rename docker/{ => deprecated}/nginx/dist/conf/nginx.conf (100%) rename docker/{ => deprecated}/nginx/dist/conf/ssl/dhparam4096.pem (100%) rename docker/{ => deprecated}/nginx/dist/conf/ssl/gen-cert.sh (100%) rename docker/{ => deprecated}/nginx/dist/conf/ssl/gen-dhparam.sh (100%) rename docker/{ => deprecated}/nginx/dist/conf/tpotweb.conf (100%) rename docker/{ => deprecated}/nginx/dist/html/error.html (100%) rename docker/{ => deprecated}/nginx/dist/html/favicon.ico (100%) rename docker/{ => deprecated}/nginx/dist/html/navbar.html (100%) rename docker/{ => deprecated}/nginx/dist/html/style.css (100%) rename docker/{ => deprecated}/nginx/dist/html/tpotweb.html (100%) rename docker/{ => deprecated}/nginx/docker-compose.yml (100%) create mode 100644 docker/spiderfoot/Dockerfile.old diff --git a/docker/hpfeeds/Dockerfile b/docker/deprecated/hpfeeds/Dockerfile similarity index 100% rename from docker/hpfeeds/Dockerfile rename to docker/deprecated/hpfeeds/Dockerfile diff --git a/docker/deprecated/hpfeeds/Dockerfile.old b/docker/deprecated/hpfeeds/Dockerfile.old new file mode 100644 index 00000000..b0743132 --- /dev/null +++ b/docker/deprecated/hpfeeds/Dockerfile.old @@ -0,0 +1,42 @@ +### This is only for testing purposes, do NOT use for production +FROM alpine:latest +# +ADD dist/ /root/dist/ +# +# Install packages +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + build-base \ + coreutils \ + git \ + libffi \ + libffi-dev \ + py-gevent \ + py-pip \ + python \ + python-dev \ + sqlite && \ +# +# Install php sandbox from git + git clone --depth=1 https://github.com/rep/hpfeeds /opt/hpfeeds && \ + cd /opt/hpfeeds/broker && \ + sed -i -e '87d;88d' database.py && \ + cp /root/dist/adduser.sql . && \ + cd /opt/hpfeeds/broker && timeout 5 python broker.py || : && \ + sqlite3 db.sqlite3 < adduser.sql && \ +# + #python setup.py build && \ + #python setup.py install && \ +# +# Clean up + apk del --purge autoconf \ + build-base \ + coreutils \ + libffi-dev \ + python-dev && \ + rm -rf /root/* && \ + rm -rf /var/cache/apk/* +# +# Set workdir and start glastopf +WORKDIR /opt/hpfeeds/broker +CMD python broker.py diff --git a/docker/hpfeeds/dist/adduser.sql b/docker/deprecated/hpfeeds/dist/adduser.sql similarity index 100% rename from docker/hpfeeds/dist/adduser.sql rename to docker/deprecated/hpfeeds/dist/adduser.sql diff --git a/docker/hpfeeds/docker-compose.yml b/docker/deprecated/hpfeeds/docker-compose.yml similarity index 100% rename from docker/hpfeeds/docker-compose.yml rename to docker/deprecated/hpfeeds/docker-compose.yml diff --git a/docker/nginx/Dockerfile b/docker/deprecated/nginx/Dockerfile similarity index 97% rename from docker/nginx/Dockerfile rename to docker/deprecated/nginx/Dockerfile index 1056490e..47696e7b 100644 --- a/docker/nginx/Dockerfile +++ b/docker/deprecated/nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ diff --git a/docker/nginx/dist/conf/nginx.conf b/docker/deprecated/nginx/dist/conf/nginx.conf similarity index 100% rename from docker/nginx/dist/conf/nginx.conf rename to docker/deprecated/nginx/dist/conf/nginx.conf diff --git a/docker/nginx/dist/conf/ssl/dhparam4096.pem b/docker/deprecated/nginx/dist/conf/ssl/dhparam4096.pem similarity index 100% rename from docker/nginx/dist/conf/ssl/dhparam4096.pem rename to docker/deprecated/nginx/dist/conf/ssl/dhparam4096.pem diff --git a/docker/nginx/dist/conf/ssl/gen-cert.sh b/docker/deprecated/nginx/dist/conf/ssl/gen-cert.sh similarity index 100% rename from docker/nginx/dist/conf/ssl/gen-cert.sh rename to docker/deprecated/nginx/dist/conf/ssl/gen-cert.sh diff --git a/docker/nginx/dist/conf/ssl/gen-dhparam.sh b/docker/deprecated/nginx/dist/conf/ssl/gen-dhparam.sh similarity index 100% rename from docker/nginx/dist/conf/ssl/gen-dhparam.sh rename to docker/deprecated/nginx/dist/conf/ssl/gen-dhparam.sh diff --git a/docker/nginx/dist/conf/tpotweb.conf b/docker/deprecated/nginx/dist/conf/tpotweb.conf similarity index 100% rename from docker/nginx/dist/conf/tpotweb.conf rename to docker/deprecated/nginx/dist/conf/tpotweb.conf diff --git a/docker/nginx/dist/html/error.html b/docker/deprecated/nginx/dist/html/error.html similarity index 100% rename from docker/nginx/dist/html/error.html rename to docker/deprecated/nginx/dist/html/error.html diff --git a/docker/nginx/dist/html/favicon.ico b/docker/deprecated/nginx/dist/html/favicon.ico similarity index 100% rename from docker/nginx/dist/html/favicon.ico rename to docker/deprecated/nginx/dist/html/favicon.ico diff --git a/docker/nginx/dist/html/navbar.html b/docker/deprecated/nginx/dist/html/navbar.html similarity index 100% rename from docker/nginx/dist/html/navbar.html rename to docker/deprecated/nginx/dist/html/navbar.html diff --git a/docker/nginx/dist/html/style.css b/docker/deprecated/nginx/dist/html/style.css similarity index 100% rename from docker/nginx/dist/html/style.css rename to docker/deprecated/nginx/dist/html/style.css diff --git a/docker/nginx/dist/html/tpotweb.html b/docker/deprecated/nginx/dist/html/tpotweb.html similarity index 100% rename from docker/nginx/dist/html/tpotweb.html rename to docker/deprecated/nginx/dist/html/tpotweb.html diff --git a/docker/nginx/docker-compose.yml b/docker/deprecated/nginx/docker-compose.yml similarity index 100% rename from docker/nginx/docker-compose.yml rename to docker/deprecated/nginx/docker-compose.yml diff --git a/docker/honeypy/Dockerfile b/docker/honeypy/Dockerfile index 80d95870..3c07c6ac 100644 --- a/docker/honeypy/Dockerfile +++ b/docker/honeypy/Dockerfile @@ -38,7 +38,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ addgroup -g 2000 honeypy && \ adduser -S -H -s /bin/ash -u 2000 -D -g 2000 honeypy && \ chown -R honeypy:honeypy /opt/honeypy && \ - setcap cap_net_bind_service=+ep /opt/honeypy/env/bin/python2 && \ + setcap cap_net_bind_service=+ep /opt/honeypy/env/bin/python && \ # # Clean up apk del --purge build-base \ diff --git a/docker/mailoney/Dockerfile b/docker/mailoney/Dockerfile index f23d24ea..c924714f 100644 --- a/docker/mailoney/Dockerfile +++ b/docker/mailoney/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Install packages RUN apk -U --no-cache add \ diff --git a/docker/medpot/Dockerfile b/docker/medpot/Dockerfile index f530af8c..05ea54d6 100644 --- a/docker/medpot/Dockerfile +++ b/docker/medpot/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Setup apk RUN apk -U --no-cache add \ diff --git a/docker/medpot/docker-compose.yml b/docker/medpot/docker-compose.yml index 6d147739..a5565475 100644 --- a/docker/medpot/docker-compose.yml +++ b/docker/medpot/docker-compose.yml @@ -14,7 +14,7 @@ services: - medpot_local ports: - "2575:2575" - image: "dtagdevsec/medpot:1903" + image: "dtagdevsec/medpot:2006" read_only: true volumes: - /data/medpot/log/:/var/log/medpot diff --git a/docker/p0f/Dockerfile b/docker/p0f/Dockerfile index 1c5b1f30..6568b41f 100644 --- a/docker/p0f/Dockerfile +++ b/docker/p0f/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Add source ADD . /opt/p0f diff --git a/docker/p0f/docker-compose.yml b/docker/p0f/docker-compose.yml index 658ed924..0b1329b8 100644 --- a/docker/p0f/docker-compose.yml +++ b/docker/p0f/docker-compose.yml @@ -8,7 +8,7 @@ services: container_name: p0f restart: always network_mode: "host" - image: "dtagdevsec/p0f:1903" + image: "dtagdevsec/p0f:2006" read_only: true volumes: - /data/p0f/log:/var/log/p0f diff --git a/docker/rdpy/Dockerfile b/docker/rdpy/Dockerfile index a671e1e0..a871b906 100644 --- a/docker/rdpy/Dockerfile +++ b/docker/rdpy/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:latest # # Include dist ADD dist/ /root/dist/ diff --git a/docker/rdpy/docker-compose.yml b/docker/rdpy/docker-compose.yml index 8f2da9a9..c991c270 100644 --- a/docker/rdpy/docker-compose.yml +++ b/docker/rdpy/docker-compose.yml @@ -22,7 +22,7 @@ services: - rdpy_local ports: - "3389:3389" - image: "dtagdevsec/rdpy:1903" + image: "dtagdevsec/rdpy:2006" read_only: true volumes: - /data/rdpy/log:/var/log/rdpy diff --git a/docker/spiderfoot/Dockerfile b/docker/spiderfoot/Dockerfile index 4b1e4c8b..e95f8cd9 100644 --- a/docker/spiderfoot/Dockerfile +++ b/docker/spiderfoot/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.10 +FROM alpine:latest # # Get and install dependencies & packages RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ @@ -6,45 +6,55 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ build-base \ curl \ git \ + jpeg-dev \ libffi-dev \ libxml2 \ libxml2-dev \ libxslt \ libxslt-dev \ + musl \ + musl-dev \ + openjpeg-dev \ openssl \ openssl-dev \ - python \ - python-dev \ + python3 \ + python3-dev \ py-cffi \ py-pillow \ py-future \ - py-pip \ - swig && \ + py3-pip \ + swig \ + tinyxml \ + tinyxml-dev \ + zlib-dev && \ # # Setup user addgroup -g 2000 spiderfoot && \ adduser -S -s /bin/ash -u 2000 -D -g 2000 spiderfoot && \ # # Install spiderfoot -# git clone --depth=1 https://github.com/smicallef/spiderfoot -b v2.12.0-final /home/spiderfoot && \ git clone --depth=1 https://github.com/smicallef/spiderfoot /home/spiderfoot && \ cd /home/spiderfoot && \ - pip install --no-cache-dir openxmllib wheel && \ - pip install --no-cache-dir -r requirements.txt && \ + pip3 install --no-cache-dir wheel && \ + pip3 install --no-cache-dir -r requirements.txt && \ chown -R spiderfoot:spiderfoot /home/spiderfoot && \ sed -i "s#'__docroot': ''#'__docroot': '\/spiderfoot'#" /home/spiderfoot/sf.py && \ sed -i 's#raise cherrypy.HTTPRedirect("\/")#raise cherrypy.HTTPRedirect("\/spiderfoot")#' /home/spiderfoot/sfwebui.py && \ # # Clean up apk del --purge build-base \ + curl \ + gcc \ git \ libffi-dev \ libxml2-dev \ libxslt-dev \ + musl-dev \ openssl-dev \ - python-dev \ - py-pip \ - py-setuptools && \ + python3-dev \ + py3-pip \ + swig \ + tinyxml-dev && \ rm -rf /var/cache/apk/* # # Healthcheck @@ -53,4 +63,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:8080' # Set user, workdir and start spiderfoot USER spiderfoot:spiderfoot WORKDIR /home/spiderfoot -CMD ["/usr/bin/python", "sf.py", "0.0.0.0:8080"] +CMD ["/usr/bin/python3.8", "sf.py","-l", "0.0.0.0:8080"] diff --git a/docker/spiderfoot/Dockerfile.old b/docker/spiderfoot/Dockerfile.old new file mode 100644 index 00000000..86f0b0ea --- /dev/null +++ b/docker/spiderfoot/Dockerfile.old @@ -0,0 +1,56 @@ +FROM alpine:latest +# +# Get and install dependencies & packages +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + build-base \ + curl \ + git \ + libffi-dev \ + libxml2 \ + libxml2-dev \ + libxslt \ + libxslt-dev \ + openssl \ + openssl-dev \ + python \ + python-dev \ + py-cffi \ + py-pillow \ + py-future \ + py-pip \ + swig && \ +# +# Setup user + addgroup -g 2000 spiderfoot && \ + adduser -S -s /bin/ash -u 2000 -D -g 2000 spiderfoot && \ +# +# Install spiderfoot +# git clone --depth=1 https://github.com/smicallef/spiderfoot -b v2.12.0-final /home/spiderfoot && \ + git clone --depth=1 https://github.com/smicallef/spiderfoot /home/spiderfoot && \ + cd /home/spiderfoot && \ + pip install --no-cache-dir openxmllib wheel && \ + pip install --no-cache-dir -r requirements.txt && \ + chown -R spiderfoot:spiderfoot /home/spiderfoot && \ + sed -i "s#'__docroot': ''#'__docroot': '\/spiderfoot'#" /home/spiderfoot/sf.py && \ + sed -i 's#raise cherrypy.HTTPRedirect("\/")#raise cherrypy.HTTPRedirect("\/spiderfoot")#' /home/spiderfoot/sfwebui.py && \ +# +# Clean up + apk del --purge build-base \ + git \ + libffi-dev \ + libxml2-dev \ + libxslt-dev \ + openssl-dev \ + python-dev \ + py-pip \ + py-setuptools && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:8080' +# +# Set user, workdir and start spiderfoot +USER spiderfoot:spiderfoot +WORKDIR /home/spiderfoot +CMD ["/usr/bin/python", "sf.py", "0.0.0.0:8080"] diff --git a/docker/spiderfoot/docker-compose.yml b/docker/spiderfoot/docker-compose.yml index 475386ad..efc808c9 100644 --- a/docker/spiderfoot/docker-compose.yml +++ b/docker/spiderfoot/docker-compose.yml @@ -14,6 +14,6 @@ services: - spiderfoot_local ports: - "127.0.0.1:64303:8080" - image: "dtagdevsec/spiderfoot:1903" + image: "dtagdevsec/spiderfoot:2006" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db diff --git a/docker/tanner/docker-compose.yml b/docker/tanner/docker-compose.yml index 703946d3..01a007a9 100644 --- a/docker/tanner/docker-compose.yml +++ b/docker/tanner/docker-compose.yml @@ -14,7 +14,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/redis:1903" + image: "dtagdevsec/redis:2006" read_only: true # PHP Sandbox service @@ -26,7 +26,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/phpox:1903" + image: "dtagdevsec/phpox:2006" read_only: true # Tanner API Service @@ -40,7 +40,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" read_only: true volumes: - /data/tanner/log:/var/log/tanner @@ -59,7 +59,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" command: tannerweb read_only: true volumes: @@ -78,7 +78,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" command: tanner read_only: true volumes: @@ -100,6 +100,6 @@ services: - tanner_local ports: - "80:80" - image: "dtagdevsec/snare:1903" + image: "dtagdevsec/snare:2006" depends_on: - tanner From 21c68f75e20d732e0fbd5951e352ae4da884084f Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Wed, 26 Feb 2020 14:43:02 +0000 Subject: [PATCH 18/31] tweaking --- etc/compose/nextgen.yml | 42 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 64838709..71222829 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -50,7 +50,7 @@ services: ports: - "5000:5000/udp" - "8443:8443" - image: "dtagdevsec/ciscoasa:1903" + image: "dtagdevsec/ciscoasa:2006" read_only: true volumes: - /data/ciscoasa/log:/var/log/ciscoasa @@ -63,7 +63,7 @@ services: - citrixhoneypot_local ports: - "443:443" - image: "dtagdevsec/citrixhoneypot:1903" + image: "dtagdevsec/citrixhoneypot:2006" read_only: true volumes: - /data/citrixhoneypot/logs:/opt/citrixhoneypot/logs @@ -85,7 +85,7 @@ services: ports: - "161:161" - "2404:2404" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -106,7 +106,7 @@ services: - conpot_local_guardian_ast ports: - "10001:10001" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -127,7 +127,7 @@ services: - conpot_local_ipmi ports: - "623:623" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -149,7 +149,7 @@ services: ports: - "1025:1025" - "50100:50100" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -198,7 +198,7 @@ services: - "5060:5060/udp" - "5061:5061" - "27017:27017" - image: "dtagdevsec/dionaea:1903" + image: "dtagdevsec/dionaea:2006" read_only: true volumes: - /data/dionaea/roots/ftp:/opt/dionaea/var/dionaea/roots/ftp @@ -220,7 +220,7 @@ services: network_mode: "host" cap_add: - NET_ADMIN - image: "dtagdevsec/glutton:1903" + image: "dtagdevsec/glutton:2006" read_only: true volumes: - /data/glutton/log:/var/log/glutton @@ -269,7 +269,7 @@ services: - "2324:2324" - "4096:4096" - "9200:9200" - image: "dtagdevsec/honeypy:1903" + image: "dtagdevsec/honeypy:2006" read_only: true volumes: - /data/honeypy/log:/opt/honeypy/log @@ -301,7 +301,7 @@ services: - medpot_local ports: - "2575:2575" - image: "dtagdevsec/medpot:1903" + image: "dtagdevsec/medpot:2006" read_only: true volumes: - /data/medpot/log/:/var/log/medpot @@ -322,7 +322,7 @@ services: - rdpy_local ports: - "3389:3389" - image: "dtagdevsec/rdpy:1903" + image: "dtagdevsec/rdpy:2006" read_only: true volumes: - /data/rdpy/log:/var/log/rdpy @@ -335,7 +335,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/redis:1903" + image: "dtagdevsec/redis:2006" read_only: true ## PHP Sandbox service @@ -345,7 +345,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/phpox:1903" + image: "dtagdevsec/phpox:2006" read_only: true ## Tanner API Service @@ -357,7 +357,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" read_only: true volumes: - /data/tanner/log:/var/log/tanner @@ -374,7 +374,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" command: tannerweb read_only: true volumes: @@ -391,7 +391,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" command: tanner read_only: true volumes: @@ -411,7 +411,7 @@ services: - tanner_local ports: - "80:80" - image: "dtagdevsec/snare:1903" + image: "dtagdevsec/snare:2006" depends_on: - tanner @@ -429,7 +429,7 @@ services: - NET_ADMIN - SYS_NICE - NET_RAW - image: "dtagdevsec/fatt:1903" + image: "dtagdevsec/fatt:2006" volumes: - /data/fatt/log:/opt/fatt/log @@ -438,7 +438,7 @@ services: container_name: p0f restart: always network_mode: "host" - image: "dtagdevsec/p0f:1903" + image: "dtagdevsec/p0f:2006" read_only: true volumes: - /data/p0f/log:/var/log/p0f @@ -472,7 +472,7 @@ services: - cyberchef_local ports: - "127.0.0.1:64299:8000" - image: "dtagdevsec/cyberchef:1903" + image: "dtagdevsec/cyberchef:2006" read_only: true #### ELK @@ -599,6 +599,6 @@ services: - spiderfoot_local ports: - "127.0.0.1:64303:8080" - image: "dtagdevsec/spiderfoot:1903" + image: "dtagdevsec/spiderfoot:2006" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db From 53e9470d58a77fde1cfe5cff96cc5fb30a67e991 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 27 Feb 2020 10:35:50 +0000 Subject: [PATCH 19/31] cleanup --- docker/cowrie/Dockerfile.old | 70 ------------------- docker/deprecated/hpfeeds/Dockerfile.old | 42 ----------- docker/glutton/Dockerfile.old | 54 -------------- docker/heralding/Dockerfile.old | 54 -------------- docker/mailoney/Dockerfile.old | 52 -------------- docker/spiderfoot/Dockerfile.old | 56 --------------- ...{Dockerfile.old => Dockerfile.from.source} | 0 docker/tanner/docker-compose.yml | 36 +++++----- docker/tanner/redis/Dockerfile | 2 +- docker/tanner/tanner/Dockerfile | 2 +- 10 files changed, 20 insertions(+), 348 deletions(-) delete mode 100644 docker/cowrie/Dockerfile.old delete mode 100644 docker/deprecated/hpfeeds/Dockerfile.old delete mode 100644 docker/glutton/Dockerfile.old delete mode 100644 docker/heralding/Dockerfile.old delete mode 100644 docker/mailoney/Dockerfile.old delete mode 100644 docker/spiderfoot/Dockerfile.old rename docker/suricata/{Dockerfile.old => Dockerfile.from.source} (100%) diff --git a/docker/cowrie/Dockerfile.old b/docker/cowrie/Dockerfile.old deleted file mode 100644 index fab410f1..00000000 --- a/docker/cowrie/Dockerfile.old +++ /dev/null @@ -1,70 +0,0 @@ -FROM alpine - -# Include dist -ADD dist/ /root/dist/ - -# Get and install dependencies & packages -RUN apk -U --no-cache add \ - bash \ - build-base \ - git \ - gmp-dev \ - libcap \ - libffi-dev \ - mpc1-dev \ - mpfr-dev \ - openssl \ - openssl-dev \ - python \ - python-dev \ - py-bcrypt \ - py-mysqldb \ - py-pip \ - py-requests \ - py-setuptools && \ - -# Setup user - addgroup -g 2000 cowrie && \ - adduser -S -s /bin/ash -u 2000 -D -g 2000 cowrie && \ - -# Install cowrie - mkdir -p /home/cowrie && \ - cd /home/cowrie && \ - git clone --depth=1 https://github.com/micheloosterhof/cowrie -b 1.5.3 && \ - cd cowrie && \ - mkdir -p log && \ - pip install --upgrade pip && \ - pip install --upgrade -r requirements.txt && \ - -# Setup configs - setcap cap_net_bind_service=+ep /usr/bin/python2.7 && \ - cp /root/dist/cowrie.cfg /home/cowrie/cowrie/cowrie.cfg && \ - chown cowrie:cowrie -R /home/cowrie/* /usr/lib/python2.7/site-packages/twisted/plugins && \ - -# Start Cowrie once to prevent dropin.cache errors upon container start caused by read-only filesystem - su - cowrie -c "export PYTHONPATH=/home/cowrie/cowrie:/home/cowrie/cowrie/src && \ - cd /home/cowrie/cowrie && \ - /usr/bin/twistd --uid=2000 --gid=2000 -y cowrie.tac --pidfile cowrie.pid cowrie &" && \ - sleep 10 && \ - -# Clean up - apk del --purge build-base \ - git \ - gmp-dev \ - libcap \ - libffi-dev \ - mpc1-dev \ - mpfr-dev \ - openssl-dev \ - python-dev \ - py-mysqldb \ - py-pip && \ - rm -rf /root/* && \ - rm -rf /var/cache/apk/* && \ - rm -rf /home/cowrie/cowrie/cowrie.pid - -# Start cowrie -ENV PYTHONPATH /home/cowrie/cowrie:/home/cowrie/cowrie/src -WORKDIR /home/cowrie/cowrie -USER cowrie:cowrie -CMD ["/usr/bin/twistd", "--nodaemon", "-y", "cowrie.tac", "--pidfile", "/tmp/cowrie/cowrie.pid", "cowrie"] diff --git a/docker/deprecated/hpfeeds/Dockerfile.old b/docker/deprecated/hpfeeds/Dockerfile.old deleted file mode 100644 index b0743132..00000000 --- a/docker/deprecated/hpfeeds/Dockerfile.old +++ /dev/null @@ -1,42 +0,0 @@ -### This is only for testing purposes, do NOT use for production -FROM alpine:latest -# -ADD dist/ /root/dist/ -# -# Install packages -RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ - apk -U --no-cache add \ - build-base \ - coreutils \ - git \ - libffi \ - libffi-dev \ - py-gevent \ - py-pip \ - python \ - python-dev \ - sqlite && \ -# -# Install php sandbox from git - git clone --depth=1 https://github.com/rep/hpfeeds /opt/hpfeeds && \ - cd /opt/hpfeeds/broker && \ - sed -i -e '87d;88d' database.py && \ - cp /root/dist/adduser.sql . && \ - cd /opt/hpfeeds/broker && timeout 5 python broker.py || : && \ - sqlite3 db.sqlite3 < adduser.sql && \ -# - #python setup.py build && \ - #python setup.py install && \ -# -# Clean up - apk del --purge autoconf \ - build-base \ - coreutils \ - libffi-dev \ - python-dev && \ - rm -rf /root/* && \ - rm -rf /var/cache/apk/* -# -# Set workdir and start glastopf -WORKDIR /opt/hpfeeds/broker -CMD python broker.py diff --git a/docker/glutton/Dockerfile.old b/docker/glutton/Dockerfile.old deleted file mode 100644 index 84d0ad89..00000000 --- a/docker/glutton/Dockerfile.old +++ /dev/null @@ -1,54 +0,0 @@ -FROM alpine -# -# Include dist -ADD dist/ /root/dist/ -# -# Setup apk -RUN apk -U --no-cache add \ - build-base \ - git \ - go \ - g++ \ - iptables-dev \ - libnetfilter_queue-dev \ - libcap \ - libpcap-dev && \ -# -# Setup go, glutton - export GOPATH=/opt/go/ && \ - go get -d github.com/mushorg/glutton && \ - cd /opt/go/src/github.com/satori/ && \ - rm -rf go.uuid && \ - git clone https://github.com/satori/go.uuid && \ - cd go.uuid && \ - git checkout v1.2.0 && \ - mv /root/dist/system.go /opt/go/src/github.com/mushorg/glutton/ && \ - cd /opt/go/src/github.com/mushorg/glutton/ && \ - make build && \ - cd / && \ - mkdir -p /opt/glutton && \ - mv /opt/go/src/github.com/mushorg/glutton/bin /opt/glutton/ && \ - mv /opt/go/src/github.com/mushorg/glutton/config /opt/glutton/ && \ - mv /opt/go/src/github.com/mushorg/glutton/rules /opt/glutton/ && \ - setcap cap_net_admin,cap_net_raw=+ep /opt/glutton/bin/server && \ - setcap cap_net_admin,cap_net_raw=+ep /sbin/xtables-multi && \ -# -# Setup user, groups and configs - addgroup -g 2000 glutton && \ - adduser -S -s /bin/ash -u 2000 -D -g 2000 glutton && \ - mkdir -p /var/log/glutton && \ - mv /root/dist/rules.yaml /opt/glutton/rules/ && \ -# -# Clean up - apk del --purge build-base \ - git \ - go \ - g++ && \ - rm -rf /var/cache/apk/* \ - /opt/go \ - /root/dist -# -# Start glutton -WORKDIR /opt/glutton -USER glutton:glutton -CMD exec bin/server -i $(/sbin/ip address | grep '^2: ' | awk '{ print $2 }' | tr -d [:punct:]) -l /var/log/glutton/glutton.log > /dev/null 2>&1 diff --git a/docker/heralding/Dockerfile.old b/docker/heralding/Dockerfile.old deleted file mode 100644 index 5ab3adcb..00000000 --- a/docker/heralding/Dockerfile.old +++ /dev/null @@ -1,54 +0,0 @@ -FROM alpine - -# Include dist -ADD dist/ /root/dist/ - -# Install packages -RUN apk -U --no-cache add \ - build-base \ - git \ - libcap \ - libffi-dev \ - openssl-dev \ - libzmq \ - postgresql-dev \ - python3 \ - python3-dev \ - py-virtualenv && \ - pip3 install --no-cache-dir --upgrade pip && \ - -# Setup heralding - mkdir -p /opt && \ - cd /opt/ && \ - git clone --depth=1 https://github.com/johnnykv/heralding && \ - cd heralding && \ - pip3 install --no-cache-dir -r requirements.txt && \ - pip3 install --no-cache-dir . && \ - -# Setup user, groups and configs - addgroup -g 2000 heralding && \ - adduser -S -H -s /bin/ash -u 2000 -D -g 2000 heralding && \ - mkdir -p /var/log/heralding/ /etc/heralding && \ - mv /root/dist/heralding.yml /etc/heralding/ && \ - setcap cap_net_bind_service=+ep /usr/bin/python3.6 && \ - chown -R heralding:heralding /var/log/heralding && \ - -# Clean up - apk del --purge \ - build-base \ - git \ - libcap \ - libffi-dev \ - libressl-dev \ - postgresql-dev \ - python3-dev \ - py-virtualenv && \ - rm -rf /root/* \ - /var/cache/apk/* \ - /opt/heralding - -# Start elasticpot -STOPSIGNAL SIGINT -WORKDIR /tmp/heralding/ -USER heralding:heralding -CMD exec heralding -c /etc/heralding/heralding.yml -l /var/log/heralding/heralding.log diff --git a/docker/mailoney/Dockerfile.old b/docker/mailoney/Dockerfile.old deleted file mode 100644 index 99c37142..00000000 --- a/docker/mailoney/Dockerfile.old +++ /dev/null @@ -1,52 +0,0 @@ -FROM alpine -# -# Install packages -RUN apk -U --no-cache add \ - autoconf \ - automake \ - build-base \ - git \ - libcap \ - libtool \ - py-pip \ - python \ - python-dev && \ -# -# Install libemu - git clone --depth=1 https://github.com/buffer/libemu /root/libemu/ && \ - cd /root/libemu/ && \ - autoreconf -vi && \ - ./configure && \ - make && \ - make install && \ -# -# Install libemu python wrapper - pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir \ - hpfeeds \ - pylibemu && \ -# -# Install mailoney from git - git clone --depth=1 https://github.com/awhitehatter/mailoney /opt/mailoney && \ -# -# Setup user, groups and configs - addgroup -g 2000 mailoney && \ - adduser -S -H -s /bin/ash -u 2000 -D -g 2000 mailoney && \ - chown -R mailoney:mailoney /opt/mailoney && \ - setcap cap_net_bind_service=+ep /usr/bin/python2.7 && \ -# -# Clean up - apk del --purge autoconf \ - automake \ - build-base \ - git \ - py-pip \ - python-dev && \ - rm -rf /root/* && \ - rm -rf /var/cache/apk/* -# -# Set workdir and start mailoney -STOPSIGNAL SIGINT -USER mailoney:mailoney -WORKDIR /opt/mailoney/ -CMD ["/usr/bin/python","mailoney.py","-i","0.0.0.0","-p","25","-s","mailrelay.local","-t","schizo_open_relay"] diff --git a/docker/spiderfoot/Dockerfile.old b/docker/spiderfoot/Dockerfile.old deleted file mode 100644 index 86f0b0ea..00000000 --- a/docker/spiderfoot/Dockerfile.old +++ /dev/null @@ -1,56 +0,0 @@ -FROM alpine:latest -# -# Get and install dependencies & packages -RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ - apk -U --no-cache add \ - build-base \ - curl \ - git \ - libffi-dev \ - libxml2 \ - libxml2-dev \ - libxslt \ - libxslt-dev \ - openssl \ - openssl-dev \ - python \ - python-dev \ - py-cffi \ - py-pillow \ - py-future \ - py-pip \ - swig && \ -# -# Setup user - addgroup -g 2000 spiderfoot && \ - adduser -S -s /bin/ash -u 2000 -D -g 2000 spiderfoot && \ -# -# Install spiderfoot -# git clone --depth=1 https://github.com/smicallef/spiderfoot -b v2.12.0-final /home/spiderfoot && \ - git clone --depth=1 https://github.com/smicallef/spiderfoot /home/spiderfoot && \ - cd /home/spiderfoot && \ - pip install --no-cache-dir openxmllib wheel && \ - pip install --no-cache-dir -r requirements.txt && \ - chown -R spiderfoot:spiderfoot /home/spiderfoot && \ - sed -i "s#'__docroot': ''#'__docroot': '\/spiderfoot'#" /home/spiderfoot/sf.py && \ - sed -i 's#raise cherrypy.HTTPRedirect("\/")#raise cherrypy.HTTPRedirect("\/spiderfoot")#' /home/spiderfoot/sfwebui.py && \ -# -# Clean up - apk del --purge build-base \ - git \ - libffi-dev \ - libxml2-dev \ - libxslt-dev \ - openssl-dev \ - python-dev \ - py-pip \ - py-setuptools && \ - rm -rf /var/cache/apk/* -# -# Healthcheck -HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:8080' -# -# Set user, workdir and start spiderfoot -USER spiderfoot:spiderfoot -WORKDIR /home/spiderfoot -CMD ["/usr/bin/python", "sf.py", "0.0.0.0:8080"] diff --git a/docker/suricata/Dockerfile.old b/docker/suricata/Dockerfile.from.source similarity index 100% rename from docker/suricata/Dockerfile.old rename to docker/suricata/Dockerfile.from.source diff --git a/docker/tanner/docker-compose.yml b/docker/tanner/docker-compose.yml index 01a007a9..e50649f9 100644 --- a/docker/tanner/docker-compose.yml +++ b/docker/tanner/docker-compose.yml @@ -49,23 +49,23 @@ services: - tanner_redis # Tanner WEB Service - tanner_web: - build: ./tanner - container_name: tanner_web - restart: always - stop_signal: SIGKILL - tmpfs: - - /tmp/tanner:uid=2000,gid=2000 - tty: true - networks: - - tanner_local - image: "dtagdevsec/tanner:2006" - command: tannerweb - read_only: true - volumes: - - /data/tanner/log:/var/log/tanner - depends_on: - - tanner_redis +# tanner_web: +# build: ./tanner +# container_name: tanner_web +# restart: always +# stop_signal: SIGKILL +# tmpfs: +# - /tmp/tanner:uid=2000,gid=2000 +# tty: true +# networks: +# - tanner_local +# image: "dtagdevsec/tanner:2006" +# command: tannerweb +# read_only: true +# volumes: +# - /data/tanner/log:/var/log/tanner +# depends_on: +# - tanner_redis # Tanner Service tanner: @@ -86,7 +86,7 @@ services: - /data/tanner/files:/opt/tanner/files depends_on: - tanner_api - - tanner_web +# - tanner_web - tanner_phpox # Snare Service diff --git a/docker/tanner/redis/Dockerfile b/docker/tanner/redis/Dockerfile index a86215e3..13566b81 100644 --- a/docker/tanner/redis/Dockerfile +++ b/docker/tanner/redis/Dockerfile @@ -13,7 +13,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ rm -rf /tmp/* /var/tmp/* && \ rm -rf /var/cache/apk/* # -# Start conpot +# Start redis STOPSIGNAL SIGKILL USER nobody:nobody CMD redis-server /etc/redis.conf diff --git a/docker/tanner/tanner/Dockerfile b/docker/tanner/tanner/Dockerfile index ebddeb37..f02e203a 100644 --- a/docker/tanner/tanner/Dockerfile +++ b/docker/tanner/tanner/Dockerfile @@ -56,7 +56,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ rm -rf /tmp/* /var/tmp/* && \ rm -rf /var/cache/apk/* # -# Start conpot +# Start tanner STOPSIGNAL SIGKILL USER tanner:tanner WORKDIR /opt/tanner From ee1342ce2a321becf09ff74b08b8ae6f12590ac8 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 27 Feb 2020 11:29:42 +0000 Subject: [PATCH 20/31] remove tanner_web from nextgen --- etc/compose/nextgen.yml | 72 ++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 71222829..5671110e 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -50,7 +50,7 @@ services: ports: - "5000:5000/udp" - "8443:8443" - image: "dtagdevsec/ciscoasa:2006" + image: "dtagdevsec/ciscoasa:1903" read_only: true volumes: - /data/ciscoasa/log:/var/log/ciscoasa @@ -63,7 +63,7 @@ services: - citrixhoneypot_local ports: - "443:443" - image: "dtagdevsec/citrixhoneypot:2006" + image: "dtagdevsec/citrixhoneypot:1903" read_only: true volumes: - /data/citrixhoneypot/logs:/opt/citrixhoneypot/logs @@ -85,7 +85,7 @@ services: ports: - "161:161" - "2404:2404" - image: "dtagdevsec/conpot:2006" + image: "dtagdevsec/conpot:1903" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -106,7 +106,7 @@ services: - conpot_local_guardian_ast ports: - "10001:10001" - image: "dtagdevsec/conpot:2006" + image: "dtagdevsec/conpot:1903" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -127,7 +127,7 @@ services: - conpot_local_ipmi ports: - "623:623" - image: "dtagdevsec/conpot:2006" + image: "dtagdevsec/conpot:1903" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -149,7 +149,7 @@ services: ports: - "1025:1025" - "50100:50100" - image: "dtagdevsec/conpot:2006" + image: "dtagdevsec/conpot:1903" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -198,7 +198,7 @@ services: - "5060:5060/udp" - "5061:5061" - "27017:27017" - image: "dtagdevsec/dionaea:2006" + image: "dtagdevsec/dionaea:1903" read_only: true volumes: - /data/dionaea/roots/ftp:/opt/dionaea/var/dionaea/roots/ftp @@ -220,7 +220,7 @@ services: network_mode: "host" cap_add: - NET_ADMIN - image: "dtagdevsec/glutton:2006" + image: "dtagdevsec/glutton:1903" read_only: true volumes: - /data/glutton/log:/var/log/glutton @@ -269,7 +269,7 @@ services: - "2324:2324" - "4096:4096" - "9200:9200" - image: "dtagdevsec/honeypy:2006" + image: "dtagdevsec/honeypy:1903" read_only: true volumes: - /data/honeypy/log:/opt/honeypy/log @@ -301,7 +301,7 @@ services: - medpot_local ports: - "2575:2575" - image: "dtagdevsec/medpot:2006" + image: "dtagdevsec/medpot:1903" read_only: true volumes: - /data/medpot/log/:/var/log/medpot @@ -322,7 +322,7 @@ services: - rdpy_local ports: - "3389:3389" - image: "dtagdevsec/rdpy:2006" + image: "dtagdevsec/rdpy:1903" read_only: true volumes: - /data/rdpy/log:/var/log/rdpy @@ -335,7 +335,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/redis:2006" + image: "dtagdevsec/redis:1903" read_only: true ## PHP Sandbox service @@ -345,7 +345,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/phpox:2006" + image: "dtagdevsec/phpox:1903" read_only: true ## Tanner API Service @@ -357,7 +357,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:2006" + image: "dtagdevsec/tanner:1903" read_only: true volumes: - /data/tanner/log:/var/log/tanner @@ -366,21 +366,21 @@ services: - tanner_redis ## Tanner WEB Service - tanner_web: - container_name: tanner_web - restart: always - tmpfs: - - /tmp/tanner:uid=2000,gid=2000 - tty: true - networks: - - tanner_local - image: "dtagdevsec/tanner:2006" - command: tannerweb - read_only: true - volumes: - - /data/tanner/log:/var/log/tanner - depends_on: - - tanner_redis +# tanner_web: +# container_name: tanner_web +# restart: always +# tmpfs: +# - /tmp/tanner:uid=2000,gid=2000 +# tty: true +# networks: +# - tanner_local +# image: "dtagdevsec/tanner:1903" +# command: tannerweb +# read_only: true +# volumes: +# - /data/tanner/log:/var/log/tanner +# depends_on: +# - tanner_redis ## Tanner Service tanner: @@ -391,7 +391,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:2006" + image: "dtagdevsec/tanner:1903" command: tanner read_only: true volumes: @@ -399,7 +399,7 @@ services: - /data/tanner/files:/opt/tanner/files depends_on: - tanner_api - - tanner_web +# - tanner_web - tanner_phpox ## Snare Service @@ -411,7 +411,7 @@ services: - tanner_local ports: - "80:80" - image: "dtagdevsec/snare:2006" + image: "dtagdevsec/snare:1903" depends_on: - tanner @@ -429,7 +429,7 @@ services: - NET_ADMIN - SYS_NICE - NET_RAW - image: "dtagdevsec/fatt:2006" + image: "dtagdevsec/fatt:1903" volumes: - /data/fatt/log:/opt/fatt/log @@ -438,7 +438,7 @@ services: container_name: p0f restart: always network_mode: "host" - image: "dtagdevsec/p0f:2006" + image: "dtagdevsec/p0f:1903" read_only: true volumes: - /data/p0f/log:/var/log/p0f @@ -472,7 +472,7 @@ services: - cyberchef_local ports: - "127.0.0.1:64299:8000" - image: "dtagdevsec/cyberchef:2006" + image: "dtagdevsec/cyberchef:1903" read_only: true #### ELK @@ -599,6 +599,6 @@ services: - spiderfoot_local ports: - "127.0.0.1:64303:8080" - image: "dtagdevsec/spiderfoot:2006" + image: "dtagdevsec/spiderfoot:1903" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db From 5d7a6f3270eb574c57ca5ba0c9573f44c5985e89 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 2 Mar 2020 15:23:05 +0000 Subject: [PATCH 21/31] tweaking --- docker/spiderfoot/Dockerfile | 4 +-- docker/tanner/tanner/Dockerfile | 4 ++- docker/tanner/tanner/dist/config.py | 5 ++-- etc/compose/nextgen.yml | 42 ++++++++++++++--------------- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/docker/spiderfoot/Dockerfile b/docker/spiderfoot/Dockerfile index e95f8cd9..e56f3bf7 100644 --- a/docker/spiderfoot/Dockerfile +++ b/docker/spiderfoot/Dockerfile @@ -43,7 +43,6 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # # Clean up apk del --purge build-base \ - curl \ gcc \ git \ libffi-dev \ @@ -58,7 +57,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ rm -rf /var/cache/apk/* # # Healthcheck -HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:8080' +#HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:8080' +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:8080/spiderfoot/' # # Set user, workdir and start spiderfoot USER spiderfoot:spiderfoot diff --git a/docker/tanner/tanner/Dockerfile b/docker/tanner/tanner/Dockerfile index f02e203a..5c710b82 100644 --- a/docker/tanner/tanner/Dockerfile +++ b/docker/tanner/tanner/Dockerfile @@ -18,8 +18,10 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # # Setup Tanner git clone --depth=1 https://github.com/mushorg/tanner /opt/tanner && \ - cp /root/dist/config.py /opt/tanner/tanner/ && \ cd /opt/tanner/ && \ + git fetch origin pull/362/head:test && \ + git checkout test && \ + cp /root/dist/config.py /opt/tanner/tanner/ && \ pip3 install --no-cache-dir setuptools && \ pip3 install --no-cache-dir -r requirements.txt && \ python3 setup.py install && \ diff --git a/docker/tanner/tanner/dist/config.py b/docker/tanner/tanner/dist/config.py index 44db36e0..54a836aa 100644 --- a/docker/tanner/tanner/dist/config.py +++ b/docker/tanner/tanner/dist/config.py @@ -15,7 +15,7 @@ config_template = {'DATA': {'db_config': '/opt/tanner/db/db_config.json', }, 'TANNER': {'host': '0.0.0.0', 'port': 8090}, 'WEB': {'host': '0.0.0.0', 'port': 8091}, - 'API': {'host': '0.0.0.0', 'port': 8092}, + 'API': {'host': '0.0.0.0', 'port': 8092, 'auth': False, 'auth_signature': 'tanner_api_auth'}, 'PHPOX': {'host': '0.0.0.0', 'port': 8088}, 'REDIS': {'host': 'tanner_redis', 'port': 6379, 'poolsize': 80, 'timeout': 1}, 'EMULATORS': {'root_dir': '/opt/tanner'}, @@ -33,7 +33,8 @@ config_template = {'DATA': {'db_config': '/opt/tanner/db/db_config.json', 'LOCALLOG': {'enabled': True, 'PATH': '/var/log/tanner/tanner_report.json'}, 'CLEANLOG': {'enabled': False}, 'REMOTE_DOCKERFILE': {'GITHUB': "https://raw.githubusercontent.com/mushorg/tanner/master/docker/" - "tanner/template_injection/Dockerfile"} + "tanner/template_injection/Dockerfile"}, + 'SESSIONS': {"delete_timeout": 300} } diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 5671110e..a704247d 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -50,7 +50,7 @@ services: ports: - "5000:5000/udp" - "8443:8443" - image: "dtagdevsec/ciscoasa:1903" + image: "dtagdevsec/ciscoasa:2006" read_only: true volumes: - /data/ciscoasa/log:/var/log/ciscoasa @@ -63,7 +63,7 @@ services: - citrixhoneypot_local ports: - "443:443" - image: "dtagdevsec/citrixhoneypot:1903" + image: "dtagdevsec/citrixhoneypot:2006" read_only: true volumes: - /data/citrixhoneypot/logs:/opt/citrixhoneypot/logs @@ -85,7 +85,7 @@ services: ports: - "161:161" - "2404:2404" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -106,7 +106,7 @@ services: - conpot_local_guardian_ast ports: - "10001:10001" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -127,7 +127,7 @@ services: - conpot_local_ipmi ports: - "623:623" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -149,7 +149,7 @@ services: ports: - "1025:1025" - "50100:50100" - image: "dtagdevsec/conpot:1903" + image: "dtagdevsec/conpot:2006" read_only: true volumes: - /data/conpot/log:/var/log/conpot @@ -198,7 +198,7 @@ services: - "5060:5060/udp" - "5061:5061" - "27017:27017" - image: "dtagdevsec/dionaea:1903" + image: "dtagdevsec/dionaea:2006" read_only: true volumes: - /data/dionaea/roots/ftp:/opt/dionaea/var/dionaea/roots/ftp @@ -220,7 +220,7 @@ services: network_mode: "host" cap_add: - NET_ADMIN - image: "dtagdevsec/glutton:1903" + image: "dtagdevsec/glutton:2006" read_only: true volumes: - /data/glutton/log:/var/log/glutton @@ -269,7 +269,7 @@ services: - "2324:2324" - "4096:4096" - "9200:9200" - image: "dtagdevsec/honeypy:1903" + image: "dtagdevsec/honeypy:2006" read_only: true volumes: - /data/honeypy/log:/opt/honeypy/log @@ -301,7 +301,7 @@ services: - medpot_local ports: - "2575:2575" - image: "dtagdevsec/medpot:1903" + image: "dtagdevsec/medpot:2006" read_only: true volumes: - /data/medpot/log/:/var/log/medpot @@ -322,7 +322,7 @@ services: - rdpy_local ports: - "3389:3389" - image: "dtagdevsec/rdpy:1903" + image: "dtagdevsec/rdpy:2006" read_only: true volumes: - /data/rdpy/log:/var/log/rdpy @@ -335,7 +335,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/redis:1903" + image: "dtagdevsec/redis:2006" read_only: true ## PHP Sandbox service @@ -345,7 +345,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/phpox:1903" + image: "dtagdevsec/phpox:2006" read_only: true ## Tanner API Service @@ -357,7 +357,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" read_only: true volumes: - /data/tanner/log:/var/log/tanner @@ -374,7 +374,7 @@ services: # tty: true # networks: # - tanner_local -# image: "dtagdevsec/tanner:1903" +# image: "dtagdevsec/tanner:2006" # command: tannerweb # read_only: true # volumes: @@ -391,7 +391,7 @@ services: tty: true networks: - tanner_local - image: "dtagdevsec/tanner:1903" + image: "dtagdevsec/tanner:2006" command: tanner read_only: true volumes: @@ -411,7 +411,7 @@ services: - tanner_local ports: - "80:80" - image: "dtagdevsec/snare:1903" + image: "dtagdevsec/snare:2006" depends_on: - tanner @@ -429,7 +429,7 @@ services: - NET_ADMIN - SYS_NICE - NET_RAW - image: "dtagdevsec/fatt:1903" + image: "dtagdevsec/fatt:2006" volumes: - /data/fatt/log:/opt/fatt/log @@ -438,7 +438,7 @@ services: container_name: p0f restart: always network_mode: "host" - image: "dtagdevsec/p0f:1903" + image: "dtagdevsec/p0f:2006" read_only: true volumes: - /data/p0f/log:/var/log/p0f @@ -472,7 +472,7 @@ services: - cyberchef_local ports: - "127.0.0.1:64299:8000" - image: "dtagdevsec/cyberchef:1903" + image: "dtagdevsec/cyberchef:2006" read_only: true #### ELK @@ -599,6 +599,6 @@ services: - spiderfoot_local ports: - "127.0.0.1:64303:8080" - image: "dtagdevsec/spiderfoot:1903" + image: "dtagdevsec/spiderfoot:2006" volumes: - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db From 3b8c959c66cdd6a937c73c747685ea970fcf0cb9 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Tue, 3 Mar 2020 12:30:57 +0000 Subject: [PATCH 22/31] tweaking --- docker/tanner/docker-compose.yml | 38 +++---- docker/tanner/phpox/dist/sandbox.py | 10 +- docker/tanner/phpox/dist/sandbox.py.old | 125 ++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 21 deletions(-) create mode 100644 docker/tanner/phpox/dist/sandbox.py.old diff --git a/docker/tanner/docker-compose.yml b/docker/tanner/docker-compose.yml index e50649f9..31a2ae00 100644 --- a/docker/tanner/docker-compose.yml +++ b/docker/tanner/docker-compose.yml @@ -49,23 +49,25 @@ services: - tanner_redis # Tanner WEB Service -# tanner_web: -# build: ./tanner -# container_name: tanner_web -# restart: always -# stop_signal: SIGKILL -# tmpfs: -# - /tmp/tanner:uid=2000,gid=2000 -# tty: true -# networks: -# - tanner_local -# image: "dtagdevsec/tanner:2006" -# command: tannerweb -# read_only: true -# volumes: -# - /data/tanner/log:/var/log/tanner -# depends_on: -# - tanner_redis + tanner_web: + build: ./tanner + container_name: tanner_web + restart: always + stop_signal: SIGKILL + tmpfs: + - /tmp/tanner:uid=2000,gid=2000 + tty: true + networks: + - tanner_local +# ports: +# - "127.0.0.1:8091:8091" + image: "dtagdevsec/tanner:2006" + command: tannerweb + read_only: true + volumes: + - /data/tanner/log:/var/log/tanner + depends_on: + - tanner_redis # Tanner Service tanner: @@ -86,7 +88,7 @@ services: - /data/tanner/files:/opt/tanner/files depends_on: - tanner_api -# - tanner_web + - tanner_web - tanner_phpox # Snare Service diff --git a/docker/tanner/phpox/dist/sandbox.py b/docker/tanner/phpox/dist/sandbox.py index 8b5d363e..9be76b75 100644 --- a/docker/tanner/phpox/dist/sandbox.py +++ b/docker/tanner/phpox/dist/sandbox.py @@ -23,12 +23,16 @@ import json import asyncio import hashlib import argparse +import functools from aiohttp import web from asyncio.subprocess import PIPE from pprint import pprint +_pretty_dumps = functools.partial(json.dumps, sort_keys=True, indent=4) + + class PHPSandbox(object): @classmethod def php_tag_check(cls, script): @@ -51,7 +55,7 @@ class PHPSandbox(object): self.stdout_value += line + b'\n' @asyncio.coroutine - def sandbox(self, script, phpbin="php7.0"): + def sandbox(self, script, phpbin="php7"): if not os.path.isfile(script): raise Exception("Sample not found: {0}".format(script)) @@ -96,12 +100,12 @@ def api(request): except KeyboardInterrupt: pass ret['file_md5'] = file_md5 - return web.Response(body=json.dumps(ret, sort_keys=True, indent=4).encode('utf-8')) + return web.json_response(ret, dumps=_pretty_dumps) if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("--phpbin", help="PHP binary, ex: php7.0", default="php7.0") + parser.add_argument("--phpbin", help="PHP binary, ex: php7", default="php7") args = parser.parse_args() phpbin = args.phpbin diff --git a/docker/tanner/phpox/dist/sandbox.py.old b/docker/tanner/phpox/dist/sandbox.py.old new file mode 100644 index 00000000..8b5d363e --- /dev/null +++ b/docker/tanner/phpox/dist/sandbox.py.old @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Lukas Rist +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +import os +import tempfile +import json +import asyncio +import hashlib +import argparse + +from aiohttp import web +from asyncio.subprocess import PIPE + +from pprint import pprint + +class PHPSandbox(object): + @classmethod + def php_tag_check(cls, script): + with open(script, "r+") as check_file: + file_content = check_file.read() + if "" not in file_content: + file_content += "?>" + check_file.write(file_content) + return script + + @asyncio.coroutine + def read_process(self): + while True: + line = yield from self.proc.stdout.readline() + if not line: + break + else: + self.stdout_value += line + b'\n' + + @asyncio.coroutine + def sandbox(self, script, phpbin="php7.0"): + if not os.path.isfile(script): + raise Exception("Sample not found: {0}".format(script)) + + try: + cmd = [phpbin, "sandbox.php", script] + self.proc = yield from asyncio.create_subprocess_exec(*cmd, stdout=PIPE) + self.stdout_value = b'' + yield from asyncio.wait_for(self.read_process(), timeout=3) + except Exception as e: + try: + self.proc.kill() + except Exception: + pass + print("Error executing the sandbox: {}".format(e)) + # raise e + return {'stdout': self.stdout_value.decode('utf-8')} + + +class EchoServer(asyncio.Protocol): + def connection_made(self, transport): + # peername = transport.get_extra_info('peername') + # print('connection from {}'.format(peername)) + self.transport = transport + + def data_received(self, data): + # print('data received: {}'.format(data.decode())) + self.transport.write(data) + + +@asyncio.coroutine +def api(request): + data = yield from request.read() + file_md5 = hashlib.md5(data).hexdigest() + with tempfile.NamedTemporaryFile(suffix='.php') as f: + f.write(data) + f.seek(0) + sb = PHPSandbox() + try: + server = yield from loop.create_server(EchoServer, '127.0.0.1', 1234) + ret = yield from asyncio.wait_for(sb.sandbox(f.name, phpbin), timeout=10) + server.close() + except KeyboardInterrupt: + pass + ret['file_md5'] = file_md5 + return web.Response(body=json.dumps(ret, sort_keys=True, indent=4).encode('utf-8')) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--phpbin", help="PHP binary, ex: php7.0", default="php7.0") + args = parser.parse_args() + phpbin = args.phpbin + + app = web.Application() + app.router.add_route('POST', '/', api) + + loop = asyncio.get_event_loop() + handler = app.make_handler() + f = loop.create_server(handler, '0.0.0.0', 8088) + srv = loop.run_until_complete(f) + print('serving on', srv.sockets[0].getsockname()) + try: + loop.run_forever() + except KeyboardInterrupt: + pass + finally: + loop.run_until_complete(handler.finish_connections(1.0)) + srv.close() + loop.run_until_complete(srv.wait_closed()) + loop.run_until_complete(app.finish()) + loop.close() From a4227e6a9fd2a0a0544b3688025725597763e141 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Wed, 4 Mar 2020 12:12:12 +0000 Subject: [PATCH 23/31] tweaking --- docker/tanner/tanner/Dockerfile | 2 -- iso/installer/install.sh | 6 +++--- iso/preseed/tpot.seed | 2 +- update.sh | 9 ++++++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docker/tanner/tanner/Dockerfile b/docker/tanner/tanner/Dockerfile index 5c710b82..e7216468 100644 --- a/docker/tanner/tanner/Dockerfile +++ b/docker/tanner/tanner/Dockerfile @@ -19,8 +19,6 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Setup Tanner git clone --depth=1 https://github.com/mushorg/tanner /opt/tanner && \ cd /opt/tanner/ && \ - git fetch origin pull/362/head:test && \ - git checkout test && \ cp /root/dist/config.py /opt/tanner/tanner/ && \ pip3 install --no-cache-dir setuptools && \ pip3 install --no-cache-dir -r requirements.txt && \ diff --git a/iso/installer/install.sh b/iso/installer/install.sh index 48cccd4e..b1a420af 100755 --- a/iso/installer/install.sh +++ b/iso/installer/install.sh @@ -13,8 +13,8 @@ myTPOTCOMPOSE="/opt/tpot/etc/tpot.yml" myLSB_STABLE_SUPPORTED="stretch buster" myLSB_TESTING_SUPPORTED="sid" myREMOTESITES="https://hub.docker.com https://github.com https://pypi.python.org https://debian.org" -myPREINSTALLPACKAGES="aria2 apache2-utils cracklib-check curl dialog figlet fuse grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet" -myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux cracklib-check curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" +myPREINSTALLPACKAGES="aria2 apache2-utils cracklib-runtime curl dialog figlet fuse grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet" +myINSTALLPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux cracklib-runtime curl debconf-utils dialog dnsutils docker.io docker-compose elasticsearch-curator ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" myINFO="\ ######################################## ### T-Pot Installer for Debian (Sid) ### @@ -685,7 +685,7 @@ echo "UseRoaming no" | tee -a /etc/ssh/ssh_config # Installing elasticdump, yq fuBANNER "Installing pkgs" npm install https://github.com/taskrabbit/elasticsearch-dump -g -pip3 install elasticsearch-curator yq +pip3 install yq hash -r # Cloning T-Pot from GitHub diff --git a/iso/preseed/tpot.seed b/iso/preseed/tpot.seed index 12835a5c..11f8b1a8 100755 --- a/iso/preseed/tpot.seed +++ b/iso/preseed/tpot.seed @@ -109,7 +109,7 @@ tasksel tasksel/first multiselect ssh-server ######################## ### Package Installation ######################## -d-i pkgsel/include string apache2-utils cracklib-check curl dialog figlet git grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet +d-i pkgsel/include string apache2-utils cracklib-runtime curl dialog figlet git grc libcrack2 libpq-dev lsb-release netselect-apt net-tools software-properties-common toilet popularity-contest popularity-contest/participate boolean false ################# diff --git a/update.sh b/update.sh index ee4f6a8c..6941c429 100755 --- a/update.sh +++ b/update.sh @@ -63,7 +63,7 @@ function fuSELFUPDATE () { echo "###### $myBLUE""No updates found in repository.""$myWHITE" return fi - myRESULT=$(git diff --name-only origin/master | grep update.sh) + myRESULT=$(git diff --name-only origin/dev | grep update.sh) if [ "$myRESULT" == "update.sh" ]; then echo "###### $myBLUE""Found newer version, will be pulling updates and restart myself.""$myWHITE" @@ -183,7 +183,10 @@ function fuUPDATER () { export DEBIAN_FRONTEND=noninteractive echo "### Installing apt-fast" /bin/bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)" -local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux cracklib-check curl debconf-utils dialog dnsutils docker.io docker-compose ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" +local myPACKAGES="aria2 apache2-utils apparmor apt-transport-https aufs-tools bash-completion build-essential ca-certificates cgroupfs-mount cockpit cockpit-docker console-setup console-setup-linux cracklib-runtime curl debconf-utils dialog dnsutils docker.io docker-compose elasticsearch-curator ethtool fail2ban figlet genisoimage git glances grc haveged html2text htop iptables iw jq kbd libcrack2 libltdl7 man mosh multitail netselect-apt net-tools npm ntp openssh-server openssl pass pigz prips software-properties-common syslinux psmisc pv python3-elasticsearch-curator python3-pip toilet unattended-upgrades unzip vim wget wireless-tools wpasupplicant" +echo "### Removing pip based install of elasticsearch-curator" +pip3 uninstall elasticsearch-curator -y +hash -r echo "### Now upgrading packages ..." dpkg --configure -a apt-fast -y autoclean @@ -197,7 +200,7 @@ echo "debconf debconf/frontend select noninteractive" | debconf-set-selections - apt-fast -y dist-upgrade -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes dpkg --configure -a npm install "https://github.com/taskrabbit/elasticsearch-dump" -g -pip3 install --upgrade elasticsearch-curator yq +pip3 install --upgrade yq hash -r echo "### Removing and holding back problematic packages ..." apt-fast -y purge exim4-base mailutils pcp cockpit-pcp From 07c68c85bb91c91a5db6d793092afd2d491bdb9a Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Wed, 4 Mar 2020 14:36:03 +0000 Subject: [PATCH 24/31] tweaking --- docker/docker-compose.yml | 160 +++++++++++++++++++++++++++++++++++ docker/elk/kibana/Dockerfile | 2 +- 2 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 docker/docker-compose.yml diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..2c2096e5 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,160 @@ +# T-Pot Image Builder (use only for building docker images) +version: '2.3' + +services: + +################## +#### Honeypots +################## + +# Adbhoney service + adbhoney: + build: adbhoney/. + image: "dtagdevsec/adbhoney:2006" + +# Ciscoasa service + ciscoasa: + build: ciscoasa/. + image: "dtagdevsec/ciscoasa:2006" + +# CitrixHoneypot service + citrixhoneypot: + build: citrixhoneypot/. + image: "dtagdevsec/citrixhoneypot:2006" + +# Conpot IEC104 service + conpot_IEC104: + build: conpot/. + image: "dtagdevsec/conpot:2006" + +# Cowrie service + cowrie: + build: cowrie/. + image: "dtagdevsec/cowrie:2006" + +# Dionaea service + dionaea: + build: dionaea/. + image: "dtagdevsec/dionaea:2006" + +# Glutton service + glutton: + build: glutton/. + image: "dtagdevsec/glutton:2006" + +# Heralding service + heralding: + build: heralding/. + image: "dtagdevsec/heralding:2006" + +# HoneyPy service + honeypy: + build: honeypy/. + image: "dtagdevsec/honeypy:2006" + +# Honeytrap service + honeytrap: + build: honeytrap/. + image: "dtagdevsec/honeytrap:1903" + +# Mailoney service + mailoney: + build: mailoney/. + image: "dtagdevsec/mailoney:2006" + +# Medpot service + medpot: + build: medpot/. + image: "dtagdevsec/medpot:2006" + +# Rdpy service + rdpy: + build: rdpy/. + image: "dtagdevsec/rdpy:2006" + +#### Snare / Tanner +## Tanner Redis Service + tanner_redis: + build: tanner/redis/. + image: "dtagdevsec/redis:2006" + +## PHP Sandbox service + tanner_phpox: + build: tanner/phpox/. + image: "dtagdevsec/phpox:2006" + +## Tanner API Service + tanner_api: + build: tanner/tanner/. + image: "dtagdevsec/tanner:2006" + +## Snare Service + snare: + build: tanner/snare/. + image: "dtagdevsec/snare:2006" + + +################## +#### NSM +################## + +# Fatt service + fatt: + build: fatt/. + image: "dtagdevsec/fatt:2006" + +# P0f service + p0f: + build: p0f/. + image: "dtagdevsec/p0f:2006" + +# Suricata service + suricata: + build: suricata/. + image: "dtagdevsec/suricata:2006" + + +################## +#### Tools +################## + +# Cyberchef service + cyberchef: + build: cyberchef/. + image: "dtagdevsec/cyberchef:2006" + +#### ELK +## Elasticsearch service + elasticsearch: + build: elk/elasticsearch/. + image: "dtagdevsec/elasticsearch:2006" + +## Kibana service + kibana: + build: elk/kibana/. + image: "dtagdevsec/kibana:2006" + +## Logstash service + logstash: + build: elk/logstash/. + image: "dtagdevsec/logstash:2006" + +## Elasticsearch-head service + head: + build: elk/head/. + image: "dtagdevsec/head:2006" + +# Ewsposter service + ewsposter: + build: ews/. + image: "dtagdevsec/ewsposter:2006" + +# Nginx service + nginx: + build: heimdall/. + image: "dtagdevsec/nginx:2006" + +# Spiderfoot service + spiderfoot: + build: spiderfoot/. + image: "dtagdevsec/spiderfoot:2006" diff --git a/docker/elk/kibana/Dockerfile b/docker/elk/kibana/Dockerfile index 12804d6c..e96bc471 100644 --- a/docker/elk/kibana/Dockerfile +++ b/docker/elk/kibana/Dockerfile @@ -43,7 +43,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ - echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ +# echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ From df22adb45db5cb4fffeaa095e1ad7c52a20e7cbe Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 5 Mar 2020 21:20:11 +0000 Subject: [PATCH 25/31] bump elk stack to 7.6.1 --- docker/docker-compose.yml | 2 +- docker/elk/elasticsearch/Dockerfile | 2 +- docker/elk/kibana/Dockerfile | 6 +++--- docker/elk/logstash/Dockerfile | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2c2096e5..1a0bb2c4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -55,7 +55,7 @@ services: # Honeytrap service honeytrap: build: honeytrap/. - image: "dtagdevsec/honeytrap:1903" + image: "dtagdevsec/honeytrap:2006" # Mailoney service mailoney: diff --git a/docker/elk/elasticsearch/Dockerfile b/docker/elk/elasticsearch/Dockerfile index 7e0bc58f..88819485 100644 --- a/docker/elk/elasticsearch/Dockerfile +++ b/docker/elk/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ FROM alpine # # VARS -ENV ES_VER=7.6.0 \ +ENV ES_VER=7.6.1 \ JAVA_HOME=/usr/lib/jvm/java-11-openjdk # Include dist ADD dist/ /root/dist/ diff --git a/docker/elk/kibana/Dockerfile b/docker/elk/kibana/Dockerfile index e96bc471..b549f416 100644 --- a/docker/elk/kibana/Dockerfile +++ b/docker/elk/kibana/Dockerfile @@ -1,7 +1,7 @@ -FROM node:10.18.0-alpine +FROM node:10.19.0-alpine # # VARS -ENV KB_VER=7.6.0 +ENV KB_VER=7.6.1 # # Include dist ADD dist/ /root/dist/ @@ -43,7 +43,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ -# echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ diff --git a/docker/elk/logstash/Dockerfile b/docker/elk/logstash/Dockerfile index 2e654923..abf4e0bf 100644 --- a/docker/elk/logstash/Dockerfile +++ b/docker/elk/logstash/Dockerfile @@ -1,7 +1,7 @@ FROM alpine # # VARS -ENV LS_VER=7.6.0 +ENV LS_VER=7.6.1 # Include dist ADD dist/ /root/dist/ # @@ -36,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ chmod u+x /usr/bin/update.sh && \ mkdir -p /etc/logstash/conf.d && \ cp logstash.conf /etc/logstash/conf.d/ && \ - cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.3.0-java/lib/logstash/outputs/elasticsearch/ && \ + cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.3.1-java/lib/logstash/outputs/elasticsearch/ && \ # # Setup user, groups and configs addgroup -g 2000 logstash && \ From 692a21ddb1bea5699027d033d83fe2dfa686d549 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 5 Mar 2020 23:12:49 +0000 Subject: [PATCH 26/31] tanner tweaking and testing include unsecure, fix name bug --- docker/tanner/docker-compose.yml | 2 ++ docker/tanner/tanner/Dockerfile | 2 ++ docker/tanner/tanner/dist/config.py | 9 +++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/tanner/docker-compose.yml b/docker/tanner/docker-compose.yml index 31a2ae00..b70977a3 100644 --- a/docker/tanner/docker-compose.yml +++ b/docker/tanner/docker-compose.yml @@ -23,6 +23,8 @@ services: container_name: tanner_phpox restart: always stop_signal: SIGKILL + tmpfs: + - /tmp:uid=2000,gid=2000 tty: true networks: - tanner_local diff --git a/docker/tanner/tanner/Dockerfile b/docker/tanner/tanner/Dockerfile index e7216468..2a23ec05 100644 --- a/docker/tanner/tanner/Dockerfile +++ b/docker/tanner/tanner/Dockerfile @@ -19,6 +19,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Setup Tanner git clone --depth=1 https://github.com/mushorg/tanner /opt/tanner && \ cd /opt/tanner/ && \ + git fetch origin pull/364/head:test && \ + git checkout test && \ cp /root/dist/config.py /opt/tanner/tanner/ && \ pip3 install --no-cache-dir setuptools && \ pip3 install --no-cache-dir -r requirements.txt && \ diff --git a/docker/tanner/tanner/dist/config.py b/docker/tanner/tanner/dist/config.py index 54a836aa..f1e75e1b 100644 --- a/docker/tanner/tanner/dist/config.py +++ b/docker/tanner/tanner/dist/config.py @@ -13,10 +13,10 @@ config_template = {'DATA': {'db_config': '/opt/tanner/db/db_config.json', 'tornado': '/opt/tanner/data/tornado.py', 'mako': '/opt/tanner/data/mako.py' }, - 'TANNER': {'host': '0.0.0.0', 'port': 8090}, - 'WEB': {'host': '0.0.0.0', 'port': 8091}, - 'API': {'host': '0.0.0.0', 'port': 8092, 'auth': False, 'auth_signature': 'tanner_api_auth'}, - 'PHPOX': {'host': '0.0.0.0', 'port': 8088}, + 'TANNER': {'host': 'tanner', 'port': 8090}, + 'WEB': {'host': 'tanner_web', 'port': 8091}, + 'API': {'host': 'tanner_api', 'port': 8092, 'auth': False, 'auth_signature': 'tanner_api_auth'}, + 'PHPOX': {'host': 'tanner_phpox', 'port': 8088}, 'REDIS': {'host': 'tanner_redis', 'port': 6379, 'poolsize': 80, 'timeout': 1}, 'EMULATORS': {'root_dir': '/opt/tanner'}, 'EMULATOR_ENABLED': {'sqli': True, 'rfi': True, 'lfi': False, 'xss': True, 'cmd_exec': False, @@ -25,6 +25,7 @@ config_template = {'DATA': {'db_config': '/opt/tanner/db/db_config.json', 'SQLI': {'type': 'SQLITE', 'db_name': 'tanner_db', 'host': 'localhost', 'user': 'root', 'password': 'user_pass'}, 'XXE_INJECTION': {'OUT_OF_BAND': False}, + 'RFI': {"allow_insecure": True}, 'DOCKER': {'host_image': 'busybox:latest'}, 'LOGGER': {'log_debug': '/tmp/tanner/tanner.log', 'log_err': '/tmp/tanner/tanner.err'}, 'MONGO': {'enabled': False, 'URI': 'mongodb://localhost'}, From 941a0e1587c269cba80063bf196a7d94d863ba59 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 5 Mar 2020 23:22:03 +0000 Subject: [PATCH 27/31] tweaking --- docker/honeytrap/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/honeytrap/Dockerfile b/docker/honeytrap/Dockerfile index fbd0c110..80df2fdd 100644 --- a/docker/honeytrap/Dockerfile +++ b/docker/honeytrap/Dockerfile @@ -26,8 +26,8 @@ RUN apt-get update -y && \ wget && \ # # Install honeytrap from source -# git clone https://github.com/armedpot/honeytrap && \ - git clone https://github.com/t3chn0m4g3/honeytrap /root/honeytrap && \ + git clone https://github.com/armedpot/honeytrap /root/honeytrap && \ +# git clone https://github.com/t3chn0m4g3/honeytrap /root/honeytrap && \ cd /root/honeytrap/ && \ autoreconf -vfi && \ ./configure \ From 6a703544c6900ad6dfe4e4acf8aaced2633f42fa Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Thu, 5 Mar 2020 23:58:27 +0000 Subject: [PATCH 28/31] tweaking --- docker/tanner/phpox/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tanner/phpox/Dockerfile b/docker/tanner/phpox/Dockerfile index 61fa8c77..26221013 100644 --- a/docker/tanner/phpox/Dockerfile +++ b/docker/tanner/phpox/Dockerfile @@ -32,7 +32,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Install PHP Sandbox git clone --depth=1 https://github.com/mushorg/phpox /opt/phpox && \ cd /opt/phpox && \ - cp /root/dist/sandbox.py . && \ +# cp /root/dist/sandbox.py . && \ pip3 install -r requirements.txt && \ make && \ # From a18304dfdcbc8e28b0baa1c249c618a5f38a7f17 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 9 Mar 2020 09:35:19 +0000 Subject: [PATCH 29/31] tanner, prepare merger w/ master --- docker/tanner/phpox/dist/sandbox.py | 129 ------------------------ docker/tanner/phpox/dist/sandbox.py.old | 125 ----------------------- 2 files changed, 254 deletions(-) delete mode 100644 docker/tanner/phpox/dist/sandbox.py delete mode 100644 docker/tanner/phpox/dist/sandbox.py.old diff --git a/docker/tanner/phpox/dist/sandbox.py b/docker/tanner/phpox/dist/sandbox.py deleted file mode 100644 index 9be76b75..00000000 --- a/docker/tanner/phpox/dist/sandbox.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2016 Lukas Rist -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import os -import tempfile -import json -import asyncio -import hashlib -import argparse -import functools - -from aiohttp import web -from asyncio.subprocess import PIPE - -from pprint import pprint - -_pretty_dumps = functools.partial(json.dumps, sort_keys=True, indent=4) - - -class PHPSandbox(object): - @classmethod - def php_tag_check(cls, script): - with open(script, "r+") as check_file: - file_content = check_file.read() - if "" not in file_content: - file_content += "?>" - check_file.write(file_content) - return script - - @asyncio.coroutine - def read_process(self): - while True: - line = yield from self.proc.stdout.readline() - if not line: - break - else: - self.stdout_value += line + b'\n' - - @asyncio.coroutine - def sandbox(self, script, phpbin="php7"): - if not os.path.isfile(script): - raise Exception("Sample not found: {0}".format(script)) - - try: - cmd = [phpbin, "sandbox.php", script] - self.proc = yield from asyncio.create_subprocess_exec(*cmd, stdout=PIPE) - self.stdout_value = b'' - yield from asyncio.wait_for(self.read_process(), timeout=3) - except Exception as e: - try: - self.proc.kill() - except Exception: - pass - print("Error executing the sandbox: {}".format(e)) - # raise e - return {'stdout': self.stdout_value.decode('utf-8')} - - -class EchoServer(asyncio.Protocol): - def connection_made(self, transport): - # peername = transport.get_extra_info('peername') - # print('connection from {}'.format(peername)) - self.transport = transport - - def data_received(self, data): - # print('data received: {}'.format(data.decode())) - self.transport.write(data) - - -@asyncio.coroutine -def api(request): - data = yield from request.read() - file_md5 = hashlib.md5(data).hexdigest() - with tempfile.NamedTemporaryFile(suffix='.php') as f: - f.write(data) - f.seek(0) - sb = PHPSandbox() - try: - server = yield from loop.create_server(EchoServer, '127.0.0.1', 1234) - ret = yield from asyncio.wait_for(sb.sandbox(f.name, phpbin), timeout=10) - server.close() - except KeyboardInterrupt: - pass - ret['file_md5'] = file_md5 - return web.json_response(ret, dumps=_pretty_dumps) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--phpbin", help="PHP binary, ex: php7", default="php7") - args = parser.parse_args() - phpbin = args.phpbin - - app = web.Application() - app.router.add_route('POST', '/', api) - - loop = asyncio.get_event_loop() - handler = app.make_handler() - f = loop.create_server(handler, '0.0.0.0', 8088) - srv = loop.run_until_complete(f) - print('serving on', srv.sockets[0].getsockname()) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - finally: - loop.run_until_complete(handler.finish_connections(1.0)) - srv.close() - loop.run_until_complete(srv.wait_closed()) - loop.run_until_complete(app.finish()) - loop.close() diff --git a/docker/tanner/phpox/dist/sandbox.py.old b/docker/tanner/phpox/dist/sandbox.py.old deleted file mode 100644 index 8b5d363e..00000000 --- a/docker/tanner/phpox/dist/sandbox.py.old +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2016 Lukas Rist -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import os -import tempfile -import json -import asyncio -import hashlib -import argparse - -from aiohttp import web -from asyncio.subprocess import PIPE - -from pprint import pprint - -class PHPSandbox(object): - @classmethod - def php_tag_check(cls, script): - with open(script, "r+") as check_file: - file_content = check_file.read() - if "" not in file_content: - file_content += "?>" - check_file.write(file_content) - return script - - @asyncio.coroutine - def read_process(self): - while True: - line = yield from self.proc.stdout.readline() - if not line: - break - else: - self.stdout_value += line + b'\n' - - @asyncio.coroutine - def sandbox(self, script, phpbin="php7.0"): - if not os.path.isfile(script): - raise Exception("Sample not found: {0}".format(script)) - - try: - cmd = [phpbin, "sandbox.php", script] - self.proc = yield from asyncio.create_subprocess_exec(*cmd, stdout=PIPE) - self.stdout_value = b'' - yield from asyncio.wait_for(self.read_process(), timeout=3) - except Exception as e: - try: - self.proc.kill() - except Exception: - pass - print("Error executing the sandbox: {}".format(e)) - # raise e - return {'stdout': self.stdout_value.decode('utf-8')} - - -class EchoServer(asyncio.Protocol): - def connection_made(self, transport): - # peername = transport.get_extra_info('peername') - # print('connection from {}'.format(peername)) - self.transport = transport - - def data_received(self, data): - # print('data received: {}'.format(data.decode())) - self.transport.write(data) - - -@asyncio.coroutine -def api(request): - data = yield from request.read() - file_md5 = hashlib.md5(data).hexdigest() - with tempfile.NamedTemporaryFile(suffix='.php') as f: - f.write(data) - f.seek(0) - sb = PHPSandbox() - try: - server = yield from loop.create_server(EchoServer, '127.0.0.1', 1234) - ret = yield from asyncio.wait_for(sb.sandbox(f.name, phpbin), timeout=10) - server.close() - except KeyboardInterrupt: - pass - ret['file_md5'] = file_md5 - return web.Response(body=json.dumps(ret, sort_keys=True, indent=4).encode('utf-8')) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--phpbin", help="PHP binary, ex: php7.0", default="php7.0") - args = parser.parse_args() - phpbin = args.phpbin - - app = web.Application() - app.router.add_route('POST', '/', api) - - loop = asyncio.get_event_loop() - handler = app.make_handler() - f = loop.create_server(handler, '0.0.0.0', 8088) - srv = loop.run_until_complete(f) - print('serving on', srv.sockets[0].getsockname()) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - finally: - loop.run_until_complete(handler.finish_connections(1.0)) - srv.close() - loop.run_until_complete(srv.wait_closed()) - loop.run_until_complete(app.finish()) - loop.close() From fb3777141b3f3949543fb315c15756ef960d0e84 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 9 Mar 2020 09:44:26 +0000 Subject: [PATCH 30/31] tanner, prepare merger w/ master --- docker/tanner/phpox/Dockerfile | 4 ---- docker/tanner/tanner/Dockerfile | 4 ++-- update.sh | 2 +- version | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docker/tanner/phpox/Dockerfile b/docker/tanner/phpox/Dockerfile index 26221013..b5adfc91 100644 --- a/docker/tanner/phpox/Dockerfile +++ b/docker/tanner/phpox/Dockerfile @@ -1,8 +1,5 @@ FROM alpine:3.10 # -# Include dist -ADD dist/ /root/dist/ -# # Install packages RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ apk -U --no-cache add \ @@ -32,7 +29,6 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Install PHP Sandbox git clone --depth=1 https://github.com/mushorg/phpox /opt/phpox && \ cd /opt/phpox && \ -# cp /root/dist/sandbox.py . && \ pip3 install -r requirements.txt && \ make && \ # diff --git a/docker/tanner/tanner/Dockerfile b/docker/tanner/tanner/Dockerfile index 2a23ec05..6512668c 100644 --- a/docker/tanner/tanner/Dockerfile +++ b/docker/tanner/tanner/Dockerfile @@ -19,8 +19,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Setup Tanner git clone --depth=1 https://github.com/mushorg/tanner /opt/tanner && \ cd /opt/tanner/ && \ - git fetch origin pull/364/head:test && \ - git checkout test && \ +# git fetch origin pull/364/head:test && \ +# git checkout test && \ cp /root/dist/config.py /opt/tanner/tanner/ && \ pip3 install --no-cache-dir setuptools && \ pip3 install --no-cache-dir -r requirements.txt && \ diff --git a/update.sh b/update.sh index 6941c429..ee18fad7 100755 --- a/update.sh +++ b/update.sh @@ -63,7 +63,7 @@ function fuSELFUPDATE () { echo "###### $myBLUE""No updates found in repository.""$myWHITE" return fi - myRESULT=$(git diff --name-only origin/dev | grep update.sh) + myRESULT=$(git diff --name-only origin/master | grep update.sh) if [ "$myRESULT" == "update.sh" ]; then echo "###### $myBLUE""Found newer version, will be pulling updates and restart myself.""$myWHITE" diff --git a/version b/version index 2b90a9b3..5298669f 100644 --- a/version +++ b/version @@ -1 +1 @@ -19.03.1 +19.03.2 From f76d8ab161443c3ae769cdc3b3358251c2ed854f Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Mon, 9 Mar 2020 10:43:52 +0000 Subject: [PATCH 31/31] update delivery window --- docker/ews/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/ews/Dockerfile b/docker/ews/Dockerfile index acd4767d..9bbaf4e5 100644 --- a/docker/ews/Dockerfile +++ b/docker/ews/Dockerfile @@ -47,4 +47,4 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Run ewsposter STOPSIGNAL SIGINT USER ews:ews -CMD sleep 10 && exec /usr/bin/python3 -u /opt/ewsposter/ews.py -l $(shuf -i 10-15 -n 1) +CMD sleep 10 && exec /usr/bin/python3 -u /opt/ewsposter/ews.py -l $(shuf -i 10-60 -n 1)