diff --git a/docker/elk/docker-compose.yml b/docker/elk/docker-compose.yml index 4b068b01..10fd8b67 100644 --- a/docker/elk/docker-compose.yml +++ b/docker/elk/docker-compose.yml @@ -24,7 +24,7 @@ services: mem_limit: 4g ports: - "127.0.0.1:64298:9200" - image: "dtagdevsec/elasticsearch:1903" + image: "dtagdevsec/elasticsearch:2006" volumes: - /data:/data @@ -39,7 +39,7 @@ services: condition: service_healthy ports: - "127.0.0.1:64296:5601" - image: "dtagdevsec/kibana:1903" + image: "dtagdevsec/kibana:2006" ## Logstash service logstash: @@ -51,7 +51,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/logstash:1903" + image: "dtagdevsec/logstash:2006" volumes: - /data:/data - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf @@ -66,5 +66,5 @@ services: condition: service_healthy ports: - "127.0.0.1:64302:9100" - image: "dtagdevsec/head:1903" + image: "dtagdevsec/head:2006" read_only: true diff --git a/docker/elk/elasticsearch/Dockerfile b/docker/elk/elasticsearch/Dockerfile index f1eb0183..45108a4d 100644 --- a/docker/elk/elasticsearch/Dockerfile +++ b/docker/elk/elasticsearch/Dockerfile @@ -1,5 +1,8 @@ FROM alpine # +# VARS +ENV ES_VER=7.5.2 \ + JAVA_HOME=/usr/lib/jvm/java-11-openjdk # Include dist ADD dist/ /root/dist/ # @@ -10,13 +13,13 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ bash \ curl \ nss \ - openjdk8-jre && \ + openjdk11-jre && \ # # Get and install packages cd /root/dist/ && \ mkdir -p /usr/share/elasticsearch/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && \ - tar xvfz elasticsearch-6.8.6.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VER-linux-x86_64.tar.gz && \ + tar xvfz elasticsearch-$ES_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ # # Add and move files cd /root/dist/ && \ @@ -40,5 +43,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' # # Start ELK USER elasticsearch:elasticsearch -ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/elasticsearch/Dockerfile.new b/docker/elk/elasticsearch/Dockerfile.new new file mode 100644 index 00000000..45108a4d --- /dev/null +++ b/docker/elk/elasticsearch/Dockerfile.new @@ -0,0 +1,46 @@ +FROM alpine +# +# VARS +ENV ES_VER=7.5.2 \ + JAVA_HOME=/usr/lib/jvm/java-11-openjdk +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + bash \ + curl \ + nss \ + openjdk11-jre && \ +# +# Get and install packages + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VER-linux-x86_64.tar.gz && \ + tar xvfz elasticsearch-$ES_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ +# +# Add and move files + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/config && \ + cp elasticsearch.yml /usr/share/elasticsearch/config/ && \ +# +# Setup user, groups and configs + addgroup -g 2000 elasticsearch && \ + adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \ + chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \ + rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ +# +# Clean up + apk del --purge aria2 && \ + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' +# +# Start ELK +USER elasticsearch:elasticsearch +CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/elasticsearch/Dockerfile.old b/docker/elk/elasticsearch/Dockerfile.old new file mode 100644 index 00000000..f1eb0183 --- /dev/null +++ b/docker/elk/elasticsearch/Dockerfile.old @@ -0,0 +1,44 @@ +FROM alpine +# +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + bash \ + curl \ + nss \ + openjdk8-jre && \ +# +# Get and install packages + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && \ + tar xvfz elasticsearch-6.8.6.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \ +# +# Add and move files + cd /root/dist/ && \ + mkdir -p /usr/share/elasticsearch/config && \ + cp elasticsearch.yml /usr/share/elasticsearch/config/ && \ +# +# Setup user, groups and configs + addgroup -g 2000 elasticsearch && \ + adduser -S -H -s /bin/ash -u 2000 -D -g 2000 elasticsearch && \ + chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \ + rm -rf /usr/share/elasticsearch/modules/x-pack-ml && \ +# +# Clean up + apk del --purge aria2 && \ + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health' +# +# Start ELK +USER elasticsearch:elasticsearch +ENV JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk +CMD ["/usr/share/elasticsearch/bin/elasticsearch"] diff --git a/docker/elk/elasticsearch/dist/elasticsearch.yml b/docker/elk/elasticsearch/dist/elasticsearch.yml index 2466b212..e8b4e479 100644 --- a/docker/elk/elasticsearch/dist/elasticsearch.yml +++ b/docker/elk/elasticsearch/dist/elasticsearch.yml @@ -1,11 +1,14 @@ cluster.name: tpotcluster node.name: "tpotcluster-node-01" xpack.ml.enabled: false +xpack.security.enabled: false path: logs: /data/elk/log data: /data/elk/data http.host: 0.0.0.0 http.cors.enabled: true http.cors.allow-origin: "*" +cluster.initial_master_nodes: +- "tpotcluster-node-01" discovery.zen.ping.unicast.hosts: - - localhost +- localhost diff --git a/docker/elk/elasticsearch/docker-compose.yml b/docker/elk/elasticsearch/docker-compose.yml index a09ec66c..3f51dcb5 100644 --- a/docker/elk/elasticsearch/docker-compose.yml +++ b/docker/elk/elasticsearch/docker-compose.yml @@ -24,6 +24,6 @@ services: mem_limit: 2g ports: - "127.0.0.1:64298:9200" - image: "dtagdevsec/elasticsearch:1903" + image: "dtagdevsec/elasticsearch:2006" volumes: - /data:/data diff --git a/docker/elk/head/docker-compose.yml b/docker/elk/head/docker-compose.yml index 35865584..5cfaafdb 100644 --- a/docker/elk/head/docker-compose.yml +++ b/docker/elk/head/docker-compose.yml @@ -12,5 +12,5 @@ services: # condition: service_healthy ports: - "127.0.0.1:64302:9100" - image: "dtagdevsec/head:1903" + image: "dtagdevsec/head:2006" read_only: true diff --git a/docker/elk/kibana/Dockerfile b/docker/elk/kibana/Dockerfile index 5088f540..6e6fd837 100644 --- a/docker/elk/kibana/Dockerfile +++ b/docker/elk/kibana/Dockerfile @@ -1,5 +1,8 @@ FROM node:10.15.2-alpine # +# VARS +ENV KB_VER=7.5.2 +# # Include dist ADD dist/ /root/dist/ # @@ -12,20 +15,20 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ # Get and install packages cd /root/dist/ && \ mkdir -p /usr/share/kibana/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-6.8.6-linux-x86_64.tar.gz && \ - tar xvfz kibana-6.8.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-$KB_VER-linux-x86_64.tar.gz && \ + tar xvfz kibana-$KB_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \ # # Kibana's bundled node does not work in alpine rm /usr/share/kibana/node/bin/node && \ - ln -s /usr/bin/node /usr/share/kibana/node/bin/node && \ + ln -s /usr/local/bin/node /usr/share/kibana/node/bin/node && \ # # Add and move files cd /root/dist/ && \ - cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \ - cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \ - cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \ +# cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \ +# cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \ +# cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \ +# cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \ +# cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \ # # Setup user, groups and configs sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \ @@ -33,17 +36,19 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \ sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \ sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \ - sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ - sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ - sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ +# sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ +# sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ +# sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.security.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.siem.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ rm -rf /usr/share/kibana/optimize/bundles/* && \ - /usr/share/kibana/bin/kibana --optimize && \ + /usr/share/kibana/bin/kibana --optimize --allow-root && \ addgroup -g 2000 kibana && \ adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \ chown -R kibana:kibana /usr/share/kibana/ && \ diff --git a/docker/elk/kibana/Dockerfile.old b/docker/elk/kibana/Dockerfile.old new file mode 100644 index 00000000..5088f540 --- /dev/null +++ b/docker/elk/kibana/Dockerfile.old @@ -0,0 +1,63 @@ +FROM node:10.15.2-alpine +# +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + curl && \ +# +# Get and install packages + cd /root/dist/ && \ + mkdir -p /usr/share/kibana/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/kibana/kibana-6.8.6-linux-x86_64.tar.gz && \ + tar xvfz kibana-6.8.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/kibana/ && \ +# +# Kibana's bundled node does not work in alpine + rm /usr/share/kibana/node/bin/node && \ + ln -s /usr/bin/node /usr/share/kibana/node/bin/node && \ +# +# Add and move files + cd /root/dist/ && \ + cp kibana.svg /usr/share/kibana/src/ui/public/images/kibana.svg && \ + cp kibana.svg /usr/share/kibana/src/ui/public/icons/kibana.svg && \ + cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon.ico && \ + cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-16x16.png && \ + cp elk.ico /usr/share/kibana/src/ui/public/assets/favicons/favicon-32x32.png && \ +# +# Setup user, groups and configs + sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#elasticsearch.hosts: \["http:\/\/localhost:9200"\]/elasticsearch.hosts: \["http:\/\/elasticsearch:9200"\]/' /usr/share/kibana/config/kibana.yml && \ + sed -i 's/#server.rewriteBasePath: false/server.rewriteBasePath: false/' /usr/share/kibana/config/kibana.yml && \ + sed -i "s/#005571/#e20074/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ + sed -i "s/#007ba4/#9e0051/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ + sed -i "s/#00465d/#4f0028/g" /usr/share/kibana/built_assets/css/plugins/kibana/index.css && \ + echo "xpack.infra.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.logstash.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.canvas.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.spaces.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.apm.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + echo "xpack.uptime.enabled: false" >> /usr/share/kibana/config/kibana.yml && \ + rm -rf /usr/share/kibana/optimize/bundles/* && \ + /usr/share/kibana/bin/kibana --optimize && \ + addgroup -g 2000 kibana && \ + adduser -S -H -s /bin/ash -u 2000 -D -g 2000 kibana && \ + chown -R kibana:kibana /usr/share/kibana/ && \ +# +# Clean up + apk del --purge aria2 && \ + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:5601' +# +# Start kibana +STOPSIGNAL SIGKILL +USER kibana:kibana +CMD ["/usr/share/kibana/bin/kibana"] diff --git a/docker/elk/kibana/docker-compose.yml b/docker/elk/kibana/docker-compose.yml index 3044a84f..2f464089 100644 --- a/docker/elk/kibana/docker-compose.yml +++ b/docker/elk/kibana/docker-compose.yml @@ -12,4 +12,4 @@ services: # condition: service_healthy ports: - "127.0.0.1:64296:5601" - image: "dtagdevsec/kibana:1903" + image: "dtagdevsec/kibana:2006" diff --git a/docker/elk/logstash/Dockerfile b/docker/elk/logstash/Dockerfile index 9d39fedf..1eb89a43 100644 --- a/docker/elk/logstash/Dockerfile +++ b/docker/elk/logstash/Dockerfile @@ -1,5 +1,7 @@ FROM alpine # +# VARS +ENV LS_VER=7.5.2 # Include dist ADD dist/ /root/dist/ # @@ -13,7 +15,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ libc6-compat \ libzmq \ nss \ - openjdk8-jre && \ + openjdk11-jre && \ # # Get and install packages mkdir -p /etc/listbot && \ @@ -23,8 +25,8 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ bunzip2 *.bz2 && \ cd /root/dist/ && \ mkdir -p /usr/share/logstash/ && \ - aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \ - tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER.tar.gz && \ + tar xvfz logstash-$LS_VER.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \ /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \ # @@ -34,7 +36,7 @@ RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ chmod u+x /usr/bin/update.sh && \ mkdir -p /etc/logstash/conf.d && \ cp logstash.conf /etc/logstash/conf.d/ && \ - cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \ + cp elasticsearch-template-es7x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.2.3-java/lib/logstash/outputs/elasticsearch/ && \ # # Setup user, groups and configs addgroup -g 2000 logstash && \ diff --git a/docker/elk/logstash/Dockerfile.old b/docker/elk/logstash/Dockerfile.old new file mode 100644 index 00000000..9d39fedf --- /dev/null +++ b/docker/elk/logstash/Dockerfile.old @@ -0,0 +1,56 @@ +FROM alpine +# +# Include dist +ADD dist/ /root/dist/ +# +# Setup env and apt +RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \ + apk -U --no-cache add \ + aria2 \ + bash \ + bzip2 \ + curl \ + libc6-compat \ + libzmq \ + nss \ + openjdk8-jre && \ +# +# Get and install packages + mkdir -p /etc/listbot && \ + cd /etc/listbot && \ + aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/cve.yaml.bz2 && \ + aria2c -s16 -x 16 https://raw.githubusercontent.com/dtag-dev-sec/listbot/master/iprep.yaml.bz2 && \ + bunzip2 *.bz2 && \ + cd /root/dist/ && \ + mkdir -p /usr/share/logstash/ && \ + aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-6.8.6.tar.gz && \ + tar xvfz logstash-6.8.6.tar.gz --strip-components=1 -C /usr/share/logstash/ && \ + /usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \ + /usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \ +# +# Add and move files + cd /root/dist/ && \ + cp update.sh /usr/bin/ && \ + chmod u+x /usr/bin/update.sh && \ + mkdir -p /etc/logstash/conf.d && \ + cp logstash.conf /etc/logstash/conf.d/ && \ + cp elasticsearch-template-es6x.json /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-9.4.0-java/lib/logstash/outputs/elasticsearch/ && \ +# +# Setup user, groups and configs + addgroup -g 2000 logstash && \ + adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \ + chown -R logstash:logstash /usr/share/logstash && \ + chown -R logstash:logstash /etc/listbot && \ + chmod 755 /usr/bin/update.sh && \ +# +# Clean up + rm -rf /root/* && \ + rm -rf /tmp/* && \ + rm -rf /var/cache/apk/* +# +# Healthcheck +HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600' +# +# Start logstash +#USER logstash:logstash +CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution diff --git a/docker/elk/logstash/dist/elasticsearch-template-es5x.json b/docker/elk/logstash/dist/elasticsearch-template-es5x.json deleted file mode 100644 index f02dfdb8..00000000 --- a/docker/elk/logstash/dist/elasticsearch-template-es5x.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "template" : "logstash-*", - "version" : 50001, - "settings" : { - "index.refresh_interval" : "5s", - "index.number_of_shards" : "1", - "index.number_of_replicas" : "0", - "mapping" : { - "total_fields" : { - "limit" : "2000" - } - } - }, - "mappings" : { - "_default_" : { - "_all" : {"enabled" : true, "norms" : false}, - "dynamic_templates" : [ { - "message_field" : { - "path_match" : "message", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", - "norms" : false - } - } - }, { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", "norms" : false, - "fields" : { - "keyword" : { "type": "keyword", "ignore_above": 256 } - } - } - } - } ], - "properties" : { - "@timestamp": { "type": "date", "include_in_all": false }, - "@version": { "type": "keyword", "include_in_all": false }, - "geoip" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - } - } - } - } -} diff --git a/docker/elk/logstash/dist/elasticsearch-template-es6x.json b/docker/elk/logstash/dist/elasticsearch-template-es6x.json deleted file mode 100644 index aff190b9..00000000 --- a/docker/elk/logstash/dist/elasticsearch-template-es6x.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "template" : "logstash-*", - "version" : 60001, - "settings" : { - "index.refresh_interval" : "5s", - "index.number_of_shards" : "1", - "index.number_of_replicas" : "0", - "index.mapping.total_fields.limit": "2000" - }, - "mappings" : { - "_default_" : { - "dynamic_templates" : [ { - "message_field" : { - "path_match" : "message", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", - "norms" : false - } - } - }, { - "string_fields" : { - "match" : "*", - "match_mapping_type" : "string", - "mapping" : { - "type" : "text", "norms" : false, - "fields" : { - "keyword" : { "type": "keyword", "ignore_above": 256 } - } - } - } - } ], - "properties" : { - "@timestamp": { "type": "date"}, - "@version": { "type": "keyword"}, - "geoip" : { - "dynamic": true, - "properties" : { - "ip": { "type": "ip" }, - "location" : { "type" : "geo_point" }, - "latitude" : { "type" : "half_float" }, - "longitude" : { "type" : "half_float" } - } - } - } - } - } -} diff --git a/docker/elk/logstash/dist/elasticsearch-template-es7x.json b/docker/elk/logstash/dist/elasticsearch-template-es7x.json new file mode 100644 index 00000000..71793ff7 --- /dev/null +++ b/docker/elk/logstash/dist/elasticsearch-template-es7x.json @@ -0,0 +1,46 @@ +{ + "index_patterns" : "logstash-*", + "version" : 60001, + "settings" : { + "index.refresh_interval" : "5s", + "number_of_shards": 1, + "index.number_of_replicas" : "0", + "index.mapping.total_fields.limit": "2000" + }, + "mappings" : { + "dynamic_templates" : [ { + "message_field" : { + "path_match" : "message", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", + "norms" : false + } + } + }, { + "string_fields" : { + "match" : "*", + "match_mapping_type" : "string", + "mapping" : { + "type" : "text", "norms" : false, + "fields" : { + "keyword" : { "type": "keyword", "ignore_above": 256 } + } + } + } + } ], + "properties" : { + "@timestamp": { "type": "date"}, + "@version": { "type": "keyword"}, + "geoip" : { + "dynamic": true, + "properties" : { + "ip": { "type": "ip" }, + "location" : { "type" : "geo_point" }, + "latitude" : { "type" : "half_float" }, + "longitude" : { "type" : "half_float" } + } + } + } + } +} diff --git a/docker/elk/logstash/dist/logstash.conf b/docker/elk/logstash/dist/logstash.conf index 3d118364..0beb0c10 100644 --- a/docker/elk/logstash/dist/logstash.conf +++ b/docker/elk/logstash/dist/logstash.conf @@ -413,12 +413,12 @@ if "_grokparsefailure" in [tags] { drop {} } geoip { cache_size => 10000 source => "src_ip" - database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-City.mmdb" } geoip { cache_size => 10000 source => "src_ip" - database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-ASN.mmdb" + database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-ASN.mmdb" } translate { refresh_interval => 86400 diff --git a/docker/elk/logstash/docker-compose.yml b/docker/elk/logstash/docker-compose.yml index c213a098..5b74feb3 100644 --- a/docker/elk/logstash/docker-compose.yml +++ b/docker/elk/logstash/docker-compose.yml @@ -12,7 +12,7 @@ services: # condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/logstash:1903" + image: "dtagdevsec/logstash:2006" volumes: - /data:/data - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf diff --git a/etc/compose/nextgen.yml b/etc/compose/nextgen.yml index 0bb76907..509d97e6 100644 --- a/etc/compose/nextgen.yml +++ b/etc/compose/nextgen.yml @@ -496,7 +496,7 @@ services: mem_limit: 4g ports: - "127.0.0.1:64298:9200" - image: "dtagdevsec/elasticsearch:1903" + image: "dtagdevsec/elasticsearch:2006" volumes: - /data:/data @@ -509,7 +509,7 @@ services: condition: service_healthy ports: - "127.0.0.1:64296:5601" - image: "dtagdevsec/kibana:1903" + image: "dtagdevsec/kibana:2006" ## Logstash service logstash: @@ -520,7 +520,7 @@ services: condition: service_healthy env_file: - /opt/tpot/etc/compose/elk_environment - image: "dtagdevsec/logstash:1903" + image: "dtagdevsec/logstash:2006" volumes: - /data:/data @@ -533,7 +533,7 @@ services: condition: service_healthy ports: - "127.0.0.1:64302:9100" - image: "dtagdevsec/head:1903" + image: "dtagdevsec/head:2006" read_only: true # Ewsposter service