Alex-Devera/tpotce
1
0
Fork 0
mirror of https://github.com/telekom-security/tpotce.git synced 2025-07-01 20:42:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

elk 6.x

Browse source 
start adjusting helper scripts for elk 6.x
migrate patterns, dashboards, viz, search, etc.
tweaking
This commit is contained in:
Marco Ochse 2018-04-19 22:38:45 +00:00
parent 6e072980a0
commit fd40fc96a6
12 changed files with 286 additions and 3011 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
bin/export_kibana-objects.sh
View file

@ -1,7 +1,8 @@
#!/bin/bash
# Export all Kibana objects
# Export all Kibana objects through Kibana Saved Objects API
# Make sure ES is available
myES="http://127.0.0.1:64298/"
myKIBANA="http://127.0.0.1:64296/"
myESSTATUS=$(curl -s -XGET ''$myES'_cluster/health' | jq '.' | grep -c green)
if ! [ "$myESSTATUS" = "1" ]
then
@ -14,10 +15,10 @@ fi
# Set vars
myDATE=$(date +%Y%m%d%H%M)
myINDEXCOUNT=$(curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*' | tr '\\' '\n' | grep "scripted" | wc -w)
myDASHBOARDS=$(curl -s -XGET ''$myES'.kibana/dashboard/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
myVISUALIZATIONS=$(curl -s -XGET ''$myES'.kibana/visualization/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
mySEARCHES=$(curl -s -XGET ''$myES'.kibana/search/_search?filter_path=hits.hits._id&pretty&size=10000' | jq '.hits.hits[] | {_id}' | jq -r '._id')
myINDEXCOUNT=$(curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern' | jq '.saved_objects[].attributes' | tr '\\' '\n' | grep "scripted" | wc -w)
myDASHBOARDS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
myVISUALIZATIONS=$(curl -s -XGET ''$myKIBANA'api/saved_objects/visualization?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
mySEARCHES=$(curl -s -XGET ''$myKIBANA'api/saved_objects/search?per_page=200' | jq '.saved_objects[].id' | tr -d '"')
myCOL1=""
myCOL0=""
@ -30,7 +31,7 @@ trap fuCLEANUP EXIT
# Export index patterns
mkdir -p patterns
echo $myCOL1"### Now exporting"$myCOL0 $myINDEXCOUNT $myCOL1"index patterns." $myCOL0
curl -s -XGET ''$myES'.kibana/index-pattern/logstash-*?' | jq '._source' > patterns/index-patterns.json
curl -s -XGET ''$myKIBANA'api/saved_objects/index-pattern' | jq '.saved_objects[] | {attributes}' > patterns/index-patterns.json
echo
# Export dashboards
@ -39,7 +40,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myDASHBOARDS | wc -w) $myCOL1"das
for i in $myDASHBOARDS;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XGET ''$myES'.kibana/dashboard/'$i'' | jq '._source' > dashboards/$i.json
curl -s -XGET ''$myKIBANA'api/saved_objects/dashboard/'$i'' | jq '. | {attributes}' > dashboards/$i.json
done;
echo
@ -49,7 +50,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $myVISUALIZATIONS | wc -w) $myCOL1
for i in $myVISUALIZATIONS;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XGET ''$myES'.kibana/visualization/'$i'' | jq '._source' > visualizations/$i.json
curl -s -XGET ''$myKIBANA'api/saved_objects/visualization/'$i'' | jq '. | {attributes}' > visualizations/$i.json
done;
echo
@ -59,7 +60,7 @@ echo $myCOL1"### Now exporting"$myCOL0 $(echo $mySEARCHES | wc -w) $myCOL1"searc
for i in $mySEARCHES;
do
echo $myCOL1"###### "$i $myCOL0
curl -s -XGET ''$myES'.kibana/search/'$i'' | jq '._source' > searches/$i.json
curl -s -XGET ''$myKIBANA'api/saved_objects/search/'$i'' | jq '. | {attributes}' > searches/$i.json
done;
echo

7
docker/elk/docker-compose.yml
View file

@ -7,6 +7,7 @@ services:
# ELK services
## Elasticsearch service
elasticsearch:
build: .
container_name: elasticsearch
restart: always
environment:
@ -30,8 +31,10 @@ services:
## Kibana service
kibana:
build: .
container_name: kibana
restart: always
stop_signal: SIGKILL
depends_on:
elasticsearch:
condition: service_healthy
@ -41,6 +44,7 @@ services:
## Logstash service
logstash:
build: .
container_name: logstash
restart: always
depends_on:
@ -52,9 +56,11 @@ services:
volumes:
- /data:/data
- /var/log:/data/host/log
- /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
## Elasticsearch-head service
head:
build: .
container_name: head
restart: always
depends_on:
@ -63,3 +69,4 @@ services:
ports:
- "127.0.0.1:64302:9100"
image: "dtagdevsec/head:1804"
read_only: true

6
docker/elk/elasticsearch/Dockerfile
View file

@ -14,8 +14,8 @@ RUN apk -U upgrade && \
# Get and install packages
cd /root/dist/ && \
mkdir -p /usr/share/elasticsearch/ && \
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.tar.gz && \
tar xvfz elasticsearch-5.6.8.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.4.tar.gz && \
tar xvfz elasticsearch-6.2.4.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
# Add and move files
cd /root/dist/ && \
@ -36,4 +36,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
# Start ELK
USER elasticsearch:elasticsearch
CMD ["/usr/share/elasticsearch/bin/elasticsearch"]
CMD export ES_TMPDIR=/tmp && exec /usr/share/elasticsearch/bin/elasticsearch

39
docker/elk/elasticsearch/Dockerfile.5x Normal file
View file

@ -0,0 +1,39 @@
FROM alpine
# Include dist
ADD dist/ /root/dist/
# Setup env and apt
RUN apk -U upgrade && \
apk add bash \
curl \
openjdk8-jre \
procps \
wget && \
# Get and install packages
cd /root/dist/ && \
mkdir -p /usr/share/elasticsearch/ && \
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.tar.gz && \
tar xvfz elasticsearch-5.6.8.tar.gz --strip-components=1 -C /usr/share/elasticsearch/ && \
# Add and move files
cd /root/dist/ && \
mkdir -p /usr/share/elasticsearch/config && \
cp elasticsearch.yml /usr/share/elasticsearch/config/ && \
# Setup user, groups and configs
addgroup -g 2000 elasticsearch && \
adduser -S -H -s /bin/bash -u 2000 -D -g 2000 elasticsearch && \
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/ && \
# Clean up
apk del --purge wget && \
rm -rf /root/*
# Healthcheck
HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9200/_cat/health'
# Start ELK
USER elasticsearch:elasticsearch
CMD ["/usr/share/elasticsearch/bin/elasticsearch"]

30
docker/elk/elasticsearch/docker-compose.yml.5x Normal file
View file

@ -0,0 +1,30 @@
# T-Pot (Standard)
# For docker-compose ...
version: '2.2'
services:
# ELK services
## Elasticsearch service
elasticsearch:
build: .
container_name: elasticsearch
restart: always
environment:
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
cap_add:
- IPC_LOCK
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
mem_limit: 2g
ports:
- "127.0.0.1:64298:9200"
image: "dtagdevsec/elasticsearch:1804"
volumes:
- /data:/data

2
docker/elk/kibana/Dockerfile
View file

@ -44,7 +44,7 @@ RUN apk -U upgrade && \
# Setup user, groups and configs
sed -i 's/#server.basePath: ""/server.basePath: "\/kibana"/' /usr/share/kibana/config/kibana.yml && \
sed -i 's/#kibana.defaultAppId: "discover"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
sed -i 's/#kibana.defaultAppId: "home"/kibana.defaultAppId: "dashboards"/' /usr/share/kibana/config/kibana.yml && \
sed -i 's/#server.host: "localhost"/server.host: "0.0.0.0"/' /usr/share/kibana/config/kibana.yml && \
sed -i 's/#elasticsearch.url: "http:\/\/localhost:9200"/elasticsearch.url: "http:\/\/elasticsearch:9200"/' /usr/share/kibana/config/kibana.yml && \
/usr/share/kibana/bin/kibana 2>&1 | grep -m 1 "Optimization of bundles" && \

2
docker/elk/logstash/Dockerfile
View file

@ -49,4 +49,4 @@ HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
# Start logstash
#USER logstash:logstash
CMD update.sh && /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf
CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf

2996
etc/objects/dashboards.json
View file

File diff suppressed because it is too large Load diff

BIN
etc/objects/elkbase.tgz
View file

Binary file not shown.

BIN
etc/objects/kibana-objects.tgz
View file

Binary file not shown.

194
etc/objects/kibana.esdump.json Normal file
View file

File diff suppressed because one or more lines are too long

2
iso/installer/install.sh
View file

@ -327,7 +327,7 @@ apt-get autoremove -y 2>&1 | dialog --title "[ Pulling updates ]" $myPROGRESSBOX
pip install --upgrade pip 2>&1 | dialog --title "[ Installing pip ]" $myPROGRESSBOXCONF
pip install elasticsearch-curator==5.4.1 2>&1 | dialog --title "[ Installing elasticsearch-curator ]" $myPROGRESSBOXCONF
pip install yq==2.4.1 2>&1 | dialog --title "[ Installing yq ]" $myPROGRESSBOXCONF
npm install https://github.com/taskrabbit/elasticsearch-dump#ac9f62a -g 2>&1 | dialog --title "[ Installing elasticsearch-dump ]" $myPROGRESSBOXCONF
npm install https://github.com/taskrabbit/elasticsearch-dump#9fcc8cc -g 2>&1 | dialog --title "[ Installing elasticsearch-dump ]" $myPROGRESSBOXCONF
wget https://github.com/bcicen/ctop/releases/download/v0.7/ctop-0.7-linux-amd64 -O ctop 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF
mv ctop /usr/bin/ 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF
chmod +x /usr/bin/ctop 2>&1 | dialog --title "[ Installing ctop ]" $myPROGRESSBOXCONF