mirror of
https://github.com/telekom-security/tpotce.git
synced 2025-07-01 20:42:11 +00:00
elk 6.x
start adjusting helper scripts for elk 6.x migrate patterns, dashboards, viz, search, etc. tweaking
This commit is contained in:
Marco Ochse
parent
6e072980a0
commit
fd40fc96a6
12 changed files with 286 additions and 3011 deletions
|
|
@ -1,7 +1,8 @@
|
|||
#!/bin/bash
|
||||
# Export all Kibana objects
|
||||
# Export all Kibana objects through Kibana Saved Objects API
|
||||
# Make sure ES is available
|
||||
myES="http://127.0.0.1:64298/"
|
||||
myKIBANA="http://127.0.0.1:64296/"
|
||||
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
|
||||
if ! [ "$myESSTATUS" = "1" ]
|
||||
then
|
||||
|
|
@ -14,10 +15,10 @@ fi
|
|||
|
||||
# Set vars
|
||||
myDATE=$(date +%Y%m%d%H%M)
|
||||
myINDEXCOUNT=$(curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*' | tr '\\' '\n' | grep "scripted" | wc -w)
|
||||
myDASHBOARDS=$(curl -s -XGET ''$myES'.kibana/dashboard/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
|
||||
myVISUALIZATIONS=$(curl -s -XGET ''$myES'.kibana/visualization/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
|
||||
mySEARCHES=$(curl -s -XGET ''$myES'.kibana/search/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
|
||||
myINDEXCOUNT=$(curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern' | jq '.saved_objects[].attributes' | tr '\\' '\n' | grep "scripted" | wc -w)
|
||||
myDASHBOARDS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
|
||||
myVISUALIZATIONS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/visualization?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
|
||||
mySEARCHES=$(curl -s -XGET ''$myKIBANA'api/saved_objects/search?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
|
||||
myCOL1="[0;34m"
|
||||
myCOL0="[0;0m"
|
||||
|
||||
|
|
@ -30,7 +31,7 @@ trap fuCLEANUP EXIT
|
|||
# Export index patterns
|
||||
mkdir -p patterns
|
||||
echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*?' | jq '._source' > patterns/index-patterns.json
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern' | jq '.saved_objects[] | {attributes}' > patterns/index-patterns.json
|
||||
echo
|
||||
|
||||
# Export dashboards
|
||||
|
|
@ -39,7 +40,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"das
|
|||
for i in $myDASHBOARDS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/dashboard/'$i'' | jq '._source' > dashboards/$i.json
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard/'$i'' | jq '. | {attributes}' > dashboards/$i.json
|
||||
done;
|
||||
echo
|
||||
|
||||
|
|
@ -49,7 +50,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1
|
|||
for i in $myVISUALIZATIONS;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/visualization/'$i'' | jq '._source' > visualizations/$i.json
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/visualization/'$i'' | jq '. | {attributes}' > visualizations/$i.json
|
||||
done;
|
||||
echo
|
||||
|
||||
|
|
@ -59,7 +60,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searc
|
|||
for i in $mySEARCHES;
|
||||
do
|
||||
echo $myCOL1"###### "$i $myCOL0
|
||||
curl -s -XGET ''$myES'.kibana/search/'$i'' | jq '._source' > searches/$i.json
|
||||
curl -s -XGET ''$myKIBANA'api/saved_objects/search/'$i'' | jq '. | {attributes}' > searches/$i.json
|
||||
done;
|
||||
echo
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ services:
|
|||
# ELK services
|
||||
## Elasticsearch service
|
||||
elasticsearch:
|
||||
build: .
|
||||
container_name: elasticsearch
|
||||
restart: always
|
||||
environment:
|
||||
|
|
@ -30,8 +31,10 @@ services:
|
|||
|
||||
## Kibana service
|
||||
kibana:
|
||||
build: .
|
||||
container_name: kibana
|
||||
restart: always
|
||||
stop_signal: SIGKILL
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
|
|
@ -41,6 +44,7 @@ services:
|
|||
|
||||
## Logstash service
|
||||
logstash:
|
||||
build: .
|
||||
container_name: logstash
|
||||
restart: always
|
||||
depends_on:
|
||||
|
|
@ -52,9 +56,11 @@ services:
|
|||
volumes:
|
||||
- /data:/data
|
||||
- /var/log:/data/host/log
|
||||
- /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
|
||||
|
||||
## Elasticsearch-head service
|
||||
head:
|
||||
build: .
|
||||
container_name: head
|
||||
restart: always
|
||||
depends_on:
|
||||
|
|
@ -63,3 +69,4 @@ services:
|
|||
ports:
|
||||
- "127.0.0.1:64302:9100"
|
||||
image: "dtagdevsec/head:1804"
|
||||
read_only: true
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ RUN apk -U upgrade && \
|
|||
# Get and install packages
|
||||
cd /root/dist/ && \
|
||||
mkdir -p /usr/share/elasticsearch/ && \
|
||||
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.tar.gz && \
|
||||
tar xvfz elasticsearch-5.6.8.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
|
||||
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.4.tar.gz && \
|
||||
tar xvfz elasticsearch-6.2.4.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
|
||||
|
||||
# Add and move files
|
||||
cd /root/dist/ && \
|
||||
|
|
@ -36,4 +36,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
|
|||
|
||||
# Start ELK
|
||||
USER elasticsearch:elasticsearch
|
||||
CMD ["/usr/share/elasticsearch/bin/elasticsearch"]
|
||||
CMD export ES_TMPDIR=/tmp && exec /usr/share/elasticsearch/bin/elasticsearch
|
||||
|
|
|
|||
39
docker/elk/elasticsearch/Dockerfile.5x
Normal file
39
docker/elk/elasticsearch/Dockerfile.5x
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
FROM alpine
|
||||
|
||||
# Include dist
|
||||
ADD dist/ /root/dist/
|
||||
|
||||
# Setup env and apt
|
||||
RUN apk -U upgrade && \
|
||||
apk add bash \
|
||||
curl \
|
||||
openjdk8-jre \
|
||||
procps \
|
||||
wget && \
|
||||
|
||||
# Get and install packages
|
||||
cd /root/dist/ && \
|
||||
mkdir -p /usr/share/elasticsearch/ && \
|
||||
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.tar.gz && \
|
||||
tar xvfz elasticsearch-5.6.8.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
|
||||
|
||||
# Add and move files
|
||||
cd /root/dist/ && \
|
||||
mkdir -p /usr/share/elasticsearch/config && \
|
||||
cp elasticsearch.yml /usr/share/elasticsearch/config/ && \
|
||||
|
||||
# Setup user, groups and configs
|
||||
addgroup -g 2000 elasticsearch && \
|
||||
adduser -S -H -s /bin/bash -u 2000 -D -g 2000 elasticsearch && \
|
||||
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \
|
||||
|
||||
# Clean up
|
||||
apk del --purge wget && \
|
||||
rm -rf /root/*
|
||||
|
||||
# Healthcheck
|
||||
HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
|
||||
|
||||
# Start ELK
|
||||
USER elasticsearch:elasticsearch
|
||||
CMD ["/usr/share/elasticsearch/bin/elasticsearch"]
|
||||
30
docker/elk/elasticsearch/docker-compose.yml.5x
Normal file
30
docker/elk/elasticsearch/docker-compose.yml.5x
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# T-Pot (Standard)
|
||||
# For docker-compose ...
|
||||
version: '2.2'
|
||||
|
||||
services:
|
||||
|
||||
# ELK services
|
||||
## Elasticsearch service
|
||||
elasticsearch:
|
||||
build: .
|
||||
container_name: elasticsearch
|
||||
restart: always
|
||||
environment:
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
|
||||
cap_add:
|
||||
- IPC_LOCK
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
nofile:
|
||||
soft: 65536
|
||||
hard: 65536
|
||||
mem_limit: 2g
|
||||
ports:
|
||||
- "127.0.0.1:64298:9200"
|
||||
image: "dtagdevsec/elasticsearch:1804"
|
||||
volumes:
|
||||
- /data:/data
|
||||
|
|
@ -44,7 +44,7 @@ RUN apk -U upgrade && \
|
|||
|
||||
# Setup user, groups and configs
|
||||
sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \
|
||||
sed -i 's/#kibana.defaultAppId: "discover"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
|
||||
sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
|
||||
sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \
|
||||
sed -i 's/#elasticsearch.url: "http:\/\/localhost:9200"/elasticsearch.url: "http:\/\/elasticsearch:9200"/' /usr/share/kibana/config/kibana.yml && \
|
||||
/usr/share/kibana/bin/kibana 2>&1 | grep -m 1 "Optimization of bundles" && \
|
||||
|
|
|
|||
|
|
@ -49,4 +49,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
|
|||
|
||||
# Start logstash
|
||||
#USER logstash:logstash
|
||||
CMD update.sh && /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf
|
||||
CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Binary file not shown.
Binary file not shown.
194
etc/objects/kibana.esdump.json
Normal file
194
etc/objects/kibana.esdump.json
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -327,7 +327,7 @@ apt-get autoremove -y 2>&1 | dialog --title "[ Pulling updates ]" $myPROGRESSBOX
|
|||
pip install --upgrade pip 2>&1 | dialog --title "[ Installing pip ]" $myPROGRESSBOXCONF
|
||||
pip install elasticsearch-curator==5.4.1 2>&1 | dialog --title "[ Installing elasticsearch-curator ]" $myPROGRESSBOXCONF
|
||||
pip install yq==2.4.1 2>&1 | dialog --title "[ Installing yq ]" $myPROGRESSBOXCONF
|
||||
npm install https://github.com/taskrabbit/elasticsearch-dump#ac9f62a -g 2>&1 | dialog --title "[ Installing elasticsearch-dump ]" $myPROGRESSBOXCONF
|
||||
npm install https://github.com/taskrabbit/elasticsearch-dump#9fcc8cc -g 2>&1 | dialog --title "[ Installing elasticsearch-dump ]" $myPROGRESSBOXCONF
|
||||
wget https://github.com/bcicen/ctop/releases/download/v0.7/ctop-0.7-linux-amd64 -O ctop 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF
|
||||
mv ctop /usr/bin/ 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF
|
||||
chmod +x /usr/bin/ctop 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF
|
||||
|
|
|
|||
Loading…
Reference in a new issue