mirror of
https://github.com/telekom-security/tpotce.git
synced 2025-04-20 06:02:24 +00:00
logstash cleanup, prep for multiarch, move to ubuntu
log4pot tweaking
This commit is contained in:
t3chn0m4g3
parent
ae18cb592e
commit
a507bc5f39
6 changed files with 33 additions and 105 deletions
|
|
@ -1,4 +1,4 @@
|
|||
FROM alpine:3.15
|
||||
FROM ubuntu:20.04
|
||||
#
|
||||
# VARS
|
||||
ENV LS_VER=7.16.3
|
||||
|
|
@ -6,65 +6,59 @@ ENV LS_VER=7.16.3
|
|||
ADD dist/ /root/dist/
|
||||
#
|
||||
# Setup env and apt
|
||||
#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
|
||||
RUN apk -U --no-cache add \
|
||||
RUN apt-get update -y && \
|
||||
apt-get dist-upgrade -y && \
|
||||
apt-get install -y \
|
||||
aria2 \
|
||||
autossh \
|
||||
bash \
|
||||
bzip2 \
|
||||
curl \
|
||||
libc6-compat \
|
||||
libzmq \
|
||||
nss \
|
||||
openssh && \
|
||||
apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community openjdk16-jre && \
|
||||
openssh-client && \
|
||||
#
|
||||
# Get and install packages
|
||||
# Determine arch, get and install packages
|
||||
ARCH=$(arch) && \
|
||||
if [ "$ARCH" = "x86_64" ]; then LS_ARCH="amd64"; fi && \
|
||||
if [ "$ARCH" = "aarch64" ]; then LS_ARCH="arm64"; fi && \
|
||||
echo "$ARCH" && \
|
||||
mkdir -p /etc/listbot && \
|
||||
cd /etc/listbot && \
|
||||
aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \
|
||||
aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \
|
||||
bunzip2 *.bz2 && \
|
||||
cd /root/dist/ && \
|
||||
mkdir -p /usr/share/logstash/ && \
|
||||
aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-linux-x86_64.tar.gz && \
|
||||
tar xvfz logstash-$LS_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
|
||||
rm -rf /usr/share/logstash/jdk && \
|
||||
# For some reason Alpine 3.14 does not report the -x flag correctly and thus elasticsearch does not find java
|
||||
sed -i 's/! -x/! -e/g' /usr/share/logstash/bin/logstash.lib.sh && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-input-http && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-output-gelf && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-output-http && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
|
||||
aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-$LS_ARCH.deb && \
|
||||
dpkg -i logstash-$LS_VER-$LS_ARCH.deb && \
|
||||
# /usr/share/logstash/bin/logstash-plugin install logstash-output-gelf logstash-output-syslog && \
|
||||
#
|
||||
# Add and move files
|
||||
cd /root/dist/ && \
|
||||
cp update.sh /usr/bin/ && \
|
||||
chmod u+x /usr/bin/update.sh && \
|
||||
mkdir -p /etc/logstash/conf.d && \
|
||||
cp entrypoint.sh /usr/bin/ && \
|
||||
chmod u+x /usr/bin/entrypoint.sh && \
|
||||
mkdir -p /etc/logstash/conf.d /usr/share/logstash/config && \
|
||||
cp logstash.conf /etc/logstash/conf.d/ && \
|
||||
cp http_input.conf /etc/logstash/conf.d/ && \
|
||||
cp http_output.conf /etc/logstash/conf.d/ && \
|
||||
cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \
|
||||
cp pipelines_pot.yml /usr/share/logstash/config/pipelines_pot.yml && \
|
||||
cp pipelines_sensor.yml /usr/share/logstash/config/pipelines_sensor.yml && \
|
||||
cp tpot_es_template.json /etc/logstash/ && \
|
||||
#
|
||||
# Setup user, groups and configs
|
||||
addgroup -g 2000 logstash && \
|
||||
adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \
|
||||
chown -R logstash:logstash /usr/share/logstash && \
|
||||
chown -R logstash:logstash /etc/listbot && \
|
||||
chmod 755 /usr/bin/update.sh && \
|
||||
groupmod -g 2000 logstash && \
|
||||
usermod -u 2000 logstash && \
|
||||
chown -R logstash:logstash /etc/listbot \
|
||||
/var/log/logstash/ \
|
||||
/var/lib/logstash \
|
||||
/usr/share/logstash/data && \
|
||||
chmod 755 /usr/bin/entrypoint.sh && \
|
||||
#
|
||||
# Clean up
|
||||
rm -rf /root/* && \
|
||||
rm -rf /tmp/* && \
|
||||
rm -rf /var/cache/apk/*
|
||||
apt-get autoremove -y --purge && \
|
||||
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/dist
|
||||
#
|
||||
# Healthcheck
|
||||
HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
|
||||
#
|
||||
# Start logstash
|
||||
#USER logstash:logstash
|
||||
CMD update.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution
|
||||
USER logstash:logstash
|
||||
CMD entrypoint.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
FROM alpine:3.14
|
||||
#
|
||||
# VARS
|
||||
ENV LS_VER=7.15.1
|
||||
# Include dist
|
||||
ADD dist/ /root/dist/
|
||||
#
|
||||
# Setup env and apt
|
||||
#RUN sed -i 's/dl-cdn/dl-2/g' /etc/apk/repositories && \
|
||||
RUN apk -U --no-cache add \
|
||||
aria2 \
|
||||
bash \
|
||||
bzip2 \
|
||||
curl \
|
||||
libc6-compat \
|
||||
libzmq \
|
||||
nss && \
|
||||
apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community openjdk16-jre && \
|
||||
#
|
||||
# Get and install packages
|
||||
mkdir -p /etc/listbot && \
|
||||
cd /etc/listbot && \
|
||||
aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/cve.yaml.bz2 && \
|
||||
aria2c -s16 -x 16 https://listbot.sicherheitstacho.eu/iprep.yaml.bz2 && \
|
||||
bunzip2 *.bz2 && \
|
||||
cd /root/dist/ && \
|
||||
mkdir -p /usr/share/logstash/ && \
|
||||
aria2c -s 16 -x 16 https://artifacts.elastic.co/downloads/logstash/logstash-$LS_VER-linux-x86_64.tar.gz && \
|
||||
tar xvfz logstash-$LS_VER-linux-x86_64.tar.gz --strip-components=1 -C /usr/share/logstash/ && \
|
||||
rm -rf /usr/share/logstash/jdk && \
|
||||
# For some reason Alpine 3.14 does not report the -x flag correctly and thus elasticsearch does not find java
|
||||
sed -i 's/! -x/! -e/g' /usr/share/logstash/bin/logstash.lib.sh && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-filter-translate && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-input-http && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-output-gelf && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-output-http && \
|
||||
/usr/share/logstash/bin/logstash-plugin install logstash-output-syslog && \
|
||||
#
|
||||
# Add and move files
|
||||
cd /root/dist/ && \
|
||||
cp update.sh /usr/bin/ && \
|
||||
chmod u+x /usr/bin/update.sh && \
|
||||
mkdir -p /etc/logstash/conf.d && \
|
||||
cp logstash.conf /etc/logstash/conf.d/ && \
|
||||
cp http.conf /etc/logstash/conf.d/ && \
|
||||
cp pipelines.yml /usr/share/logstash/config/pipelines.yml && \
|
||||
cp tpot_es_template.json /etc/logstash/ && \
|
||||
#
|
||||
# Setup user, groups and configs
|
||||
addgroup -g 2000 logstash && \
|
||||
adduser -S -H -s /bin/bash -u 2000 -D -g 2000 logstash && \
|
||||
chown -R logstash:logstash /usr/share/logstash && \
|
||||
chown -R logstash:logstash /etc/listbot && \
|
||||
chmod 755 /usr/bin/update.sh && \
|
||||
#
|
||||
# Clean up
|
||||
rm -rf /root/* && \
|
||||
rm -rf /tmp/* && \
|
||||
rm -rf /var/cache/apk/*
|
||||
#
|
||||
# Healthcheck
|
||||
HEALTHCHECK --retries=10 CMD curl -s -XGET 'http://127.0.0.1:9600'
|
||||
#
|
||||
# Start logstash
|
||||
#USER logstash:logstash
|
||||
#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution --log.level debug
|
||||
#CMD update.sh && exec /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf --config.reload.automatic --java-execution
|
||||
CMD update.sh && exec /usr/share/logstash/bin/logstash --config.reload.automatic --java-execution
|
||||
|
|
@ -46,7 +46,7 @@ if [ "$MY_TPOT_TYPE" == "SENSOR" ];
|
|||
echo "Hive username: $MY_HIVE_USERNAME"
|
||||
echo "Hive IP: $MY_HIVE_IP"
|
||||
echo
|
||||
cp /usr/share/logstash/config/pipelines_pot.yml /usr/share/logstash/config/pipelines.yml
|
||||
cp /usr/share/logstash/config/pipelines_sensor.yml /usr/share/logstash/config/pipelines.yml
|
||||
autossh -f -M 0 -4 -l $MY_HIVE_USERNAME -i $MY_SENSOR_PRIVATEKEYFILE -p 64295 -N -L64305:127.0.0.1:64305 $MY_HIVE_IP -o "ServerAliveInterval 30" -o "ServerAliveCountMax 3" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null"
|
||||
exit 0
|
||||
fi
|
||||
2
docker/elk/logstash/dist/pipelines.yml
vendored
2
docker/elk/logstash/dist/pipelines.yml
vendored
|
|
@ -1,4 +1,6 @@
|
|||
- pipeline.id: logstash
|
||||
path.config: "/etc/logstash/conf.d/logstash.conf"
|
||||
pipeline.ecs_compatibility: disabled
|
||||
- pipeline.id: http_input
|
||||
path.config: "/etc/logstash/conf.d/http_input.conf"
|
||||
pipeline.ecs_compatibility: disabled
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
- pipeline.id: http_output
|
||||
path.config: "/etc/logstash/conf.d/http_output.conf"
|
||||
pipeline.ecs_compatibility: disabled
|
||||
|
|
@ -2,8 +2,7 @@ FROM ubuntu:20.04
|
|||
ENV DEBIAN_FRONTEND noninteractive
|
||||
#
|
||||
# Install packages
|
||||
RUN apt-get update && \
|
||||
apt-get update -y && \
|
||||
RUN apt-get update -y && \
|
||||
apt-get dist-upgrade -y && \
|
||||
apt-get install -y \
|
||||
build-essential \
|
||||
|
|
|
|||
Loading…
Reference in a new issue