From 3bcf85a92f1fd3e58d155dbcae26b5a9b22af9a0 Mon Sep 17 00:00:00 2001 From: t3chn0m4g3 Date: Sun, 13 Mar 2022 10:19:08 +0000 Subject: [PATCH] tweaking, reduce build times --- docker/elasticpot/Dockerfile | 1 + docker/ews/Dockerfile | 3 ++- docker/heralding/Dockerfile | 1 + docker/honeytrap/docker-compose.yml | 4 ++-- docker/ipphoney/Dockerfile | 13 ++++++++++- docker/ipphoney/dist/requirements.txt | 4 ++++ docker/ipphoney/docker-compose.yml | 4 ++-- docker/log4pot/docker-compose.yml | 4 ++-- docker/mailoney/docker-compose.yml | 4 ++-- docker/medpot/docker-compose.yml | 4 ++-- docker/nginx/docker-compose.yml | 4 ++-- docker/p0f/docker-compose.yml | 4 ++-- docker/redishoneypot/docker-compose.yml | 4 ++-- docker/sentrypeer/docker-compose.yml | 4 ++-- docker/spiderfoot/Dockerfile | 29 ++++++++++++++++++++++++- docker/spiderfoot/dist/requirements.txt | 11 ++++++++++ docker/spiderfoot/docker-compose.yml | 4 ++-- docker/tanner/docker-compose.yml | 20 ++++++++--------- 18 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 docker/ipphoney/dist/requirements.txt create mode 100644 docker/spiderfoot/dist/requirements.txt diff --git a/docker/elasticpot/Dockerfile b/docker/elasticpot/Dockerfile index 29a166cd..6be72abc 100644 --- a/docker/elasticpot/Dockerfile +++ b/docker/elasticpot/Dockerfile @@ -25,6 +25,7 @@ RUN apk -U --no-cache add \ py3-setuptools \ py3-pip \ py3-twisted \ + py3-wheel \ python3 \ python3-dev && \ mkdir -p /opt && \ diff --git a/docker/ews/Dockerfile b/docker/ews/Dockerfile index 5994bd09..c34da45c 100644 --- a/docker/ews/Dockerfile +++ b/docker/ews/Dockerfile @@ -20,7 +20,8 @@ RUN apk -U --no-cache add \ py3-openssl \ py3-requests \ py3-pip \ - py3-setuptools && \ + py3-setuptools \ + py3-wheel && \ pip3 install --upgrade pip && \ pip3 install --no-cache-dir configparser hpfeeds3 influxdb influxdb-client xmljson && \ # diff --git a/docker/heralding/Dockerfile b/docker/heralding/Dockerfile index 1450159a..9972ff75 100644 --- a/docker/heralding/Dockerfile +++ b/docker/heralding/Dockerfile @@ -23,6 +23,7 @@ RUN apk -U --no-cache add \ py3-requests \ py3-rsa \ py3-typing-extensions \ + py3-wheel \ py3-yaml \ python3 \ python3-dev && \ diff --git a/docker/honeytrap/docker-compose.yml b/docker/honeytrap/docker-compose.yml index ac04c431..11204f28 100644 --- a/docker/honeytrap/docker-compose.yml +++ b/docker/honeytrap/docker-compose.yml @@ -9,8 +9,8 @@ services: restart: always tmpfs: - /tmp/honeytrap:uid=2000,gid=2000 - cpu_count: 1 - cpus: 0.75 +# cpu_count: 1 +# cpus: 0.75 network_mode: "host" cap_add: - NET_ADMIN diff --git a/docker/ipphoney/Dockerfile b/docker/ipphoney/Dockerfile index 11fa9985..489364c3 100644 --- a/docker/ipphoney/Dockerfile +++ b/docker/ipphoney/Dockerfile @@ -14,9 +14,20 @@ RUN apk -U --no-cache add \ openssl-dev \ postgresql-dev \ py3-cryptography \ + py3-elasticsearch \ + py3-geoip2 \ + py3-maxminddb \ py3-mysqlclient \ py3-requests \ + py3-packaging \ py3-pip \ + py3-psycopg2 \ + py3-redis \ + py3-requests \ + py3-service_identity \ + py3-setuptools \ + py3-twisted \ + py3-wheel \ python3 \ python3-dev && \ mkdir -p /opt && \ @@ -24,7 +35,7 @@ RUN apk -U --no-cache add \ git clone https://gitlab.com/bontchev/ipphoney.git/ && \ cd ipphoney && \ git checkout 7ab1cac437baba17cb2cd25d5bb1400327e1bb79 && \ - pip3 install --ignore-installed packaging && \ + cp /root/dist/requirements.txt . && \ pip3 install -r requirements.txt && \ setcap cap_net_bind_service=+ep /usr/bin/python3.9 && \ # diff --git a/docker/ipphoney/dist/requirements.txt b/docker/ipphoney/dist/requirements.txt new file mode 100644 index 00000000..882d8f6f --- /dev/null +++ b/docker/ipphoney/dist/requirements.txt @@ -0,0 +1,4 @@ +configparser>=3.5.0 +couchdb +hpfeeds>=3.0.0 +pymongo diff --git a/docker/ipphoney/docker-compose.yml b/docker/ipphoney/docker-compose.yml index 9b1b9f40..26b86fb4 100644 --- a/docker/ipphoney/docker-compose.yml +++ b/docker/ipphoney/docker-compose.yml @@ -10,8 +10,8 @@ services: build: . container_name: ipphoney restart: always - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - ipphoney_local ports: diff --git a/docker/log4pot/docker-compose.yml b/docker/log4pot/docker-compose.yml index 4afe0944..9ebe1623 100644 --- a/docker/log4pot/docker-compose.yml +++ b/docker/log4pot/docker-compose.yml @@ -12,8 +12,8 @@ services: restart: always tmpfs: - /tmp:uid=2000,gid=2000 - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - log4pot_local ports: diff --git a/docker/mailoney/docker-compose.yml b/docker/mailoney/docker-compose.yml index 44be4d26..b36405c4 100644 --- a/docker/mailoney/docker-compose.yml +++ b/docker/mailoney/docker-compose.yml @@ -16,8 +16,8 @@ services: - HPFEEDS_SECRET=pass - HPFEEDS_PORT=20000 - HPFEEDS_CHANNELPREFIX=prefix - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - mailoney_local ports: diff --git a/docker/medpot/docker-compose.yml b/docker/medpot/docker-compose.yml index fc12746e..63b152dc 100644 --- a/docker/medpot/docker-compose.yml +++ b/docker/medpot/docker-compose.yml @@ -10,8 +10,8 @@ services: build: . container_name: medpot restart: always - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - medpot_local ports: diff --git a/docker/nginx/docker-compose.yml b/docker/nginx/docker-compose.yml index 8f8be8ae..ee4387f0 100644 --- a/docker/nginx/docker-compose.yml +++ b/docker/nginx/docker-compose.yml @@ -15,8 +15,8 @@ services: - /var/tmp/nginx/scgi - /run - /var/lib/nginx/tmp:uid=100,gid=82 - cpu_count: 1 - cpus: 0.75 +# cpu_count: 1 +# cpus: 0.75 network_mode: "host" ports: - "64297:64297" diff --git a/docker/p0f/docker-compose.yml b/docker/p0f/docker-compose.yml index 3dca0c17..ed7984e1 100644 --- a/docker/p0f/docker-compose.yml +++ b/docker/p0f/docker-compose.yml @@ -7,8 +7,8 @@ services: build: . container_name: p0f restart: always - cpu_count: 1 - cpus: 0.75 +# cpu_count: 1 +# cpus: 0.75 network_mode: "host" image: "dtagdevsec/p0f:2203" read_only: true diff --git a/docker/redishoneypot/docker-compose.yml b/docker/redishoneypot/docker-compose.yml index 74e5c934..709a738d 100644 --- a/docker/redishoneypot/docker-compose.yml +++ b/docker/redishoneypot/docker-compose.yml @@ -10,8 +10,8 @@ services: build: . container_name: redishoneypot restart: always - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - redishoneypot_local ports: diff --git a/docker/sentrypeer/docker-compose.yml b/docker/sentrypeer/docker-compose.yml index 69cfcc3d..c23f1ecc 100644 --- a/docker/sentrypeer/docker-compose.yml +++ b/docker/sentrypeer/docker-compose.yml @@ -10,8 +10,8 @@ services: build: . container_name: sentrypeer restart: always - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - sentrypeer_local ports: diff --git a/docker/spiderfoot/Dockerfile b/docker/spiderfoot/Dockerfile index 11ddda36..9b2f845c 100644 --- a/docker/spiderfoot/Dockerfile +++ b/docker/spiderfoot/Dockerfile @@ -1,5 +1,8 @@ FROM alpine:3.15 # +# Include dist +COPY dist/ /root/dist/ +# # Get and install dependencies & packages RUN apk -U --no-cache add \ build-base \ @@ -18,7 +21,31 @@ RUN apk -U --no-cache add \ openssl-dev \ python3 \ python3-dev \ + py3-cryptography \ + py3-ipaddr \ + py3-beautifulsoup4 \ + py3-dnspython \ + py3-exifread \ + py3-future \ + py3-jaraco.classes \ + py3-jaraco.context \ + py3-jaraco.functools \ + py3-lxml \ + py3-mako \ + py3-more-itertools \ + py3-netaddr \ + py3-networkx \ + py3-openssl \ + py3-pillow \ + py3-portend \ + py3-pypdf2 \ + py3-phonenumbers \ py3-pip \ + py3-pysocks \ + py3-requests \ + py3-tempora \ + py3-wheel \ + py3-xlsxwriter \ swig \ tinyxml \ tinyxml-dev \ @@ -32,7 +59,7 @@ RUN apk -U --no-cache add \ git clone --depth=1 -b v3.5 https://github.com/smicallef/spiderfoot /home/spiderfoot && \ cd /home/spiderfoot && \ pip3 install --upgrade pip && \ - pip3 install --no-cache-dir wheel && \ + cp /root/dist/requirements.txt . && \ pip3 install --no-cache-dir -r requirements.txt && \ mkdir -p /home/spiderfoot/.spiderfoot/logs && \ chown -R spiderfoot:spiderfoot /home/spiderfoot && \ diff --git a/docker/spiderfoot/dist/requirements.txt b/docker/spiderfoot/dist/requirements.txt new file mode 100644 index 00000000..3da7bd68 --- /dev/null +++ b/docker/spiderfoot/dist/requirements.txt @@ -0,0 +1,11 @@ +adblockparser>=0.7,<1 +CherryPy>=18.6.1,<19 +cherrypy-cors>=1.6,<2 +ipwhois>=1.1.0,<1.2.0 +pygexf>=0.2.2,<0.3 +python-whois>=0.7.3,<0.8 +secure>=0.3.0,<0.4.0 +python-docx>=0.8.11,<0.9 +python-pptx>=0.6.21,<0.7 +publicsuffixlist>=0.7.9,<0.8 +openpyxl>=3.0.9,<4 diff --git a/docker/spiderfoot/docker-compose.yml b/docker/spiderfoot/docker-compose.yml index dae417c0..ec33b57f 100644 --- a/docker/spiderfoot/docker-compose.yml +++ b/docker/spiderfoot/docker-compose.yml @@ -10,8 +10,8 @@ services: build: . container_name: spiderfoot restart: always - cpu_count: 1 - cpus: 0.75 +# cpu_count: 1 +# cpus: 0.75 networks: - spiderfoot_local ports: diff --git a/docker/tanner/docker-compose.yml b/docker/tanner/docker-compose.yml index 0dde5fee..18c5bc1e 100644 --- a/docker/tanner/docker-compose.yml +++ b/docker/tanner/docker-compose.yml @@ -12,8 +12,8 @@ services: restart: always stop_signal: SIGKILL tty: true - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - tanner_local image: "dtagdevsec/redis:2203" @@ -28,8 +28,8 @@ services: tmpfs: - /tmp:uid=2000,gid=2000 tty: true - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - tanner_local image: "dtagdevsec/phpox:2203" @@ -44,8 +44,8 @@ services: tmpfs: - /tmp/tanner:uid=2000,gid=2000 tty: true - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - tanner_local image: "dtagdevsec/tanner:2203" @@ -86,8 +86,8 @@ services: tmpfs: - /tmp/tanner:uid=2000,gid=2000 tty: true - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - tanner_local image: "dtagdevsec/tanner:2203" @@ -108,8 +108,8 @@ services: restart: always stop_signal: SIGKILL tty: true - cpu_count: 1 - cpus: 0.25 +# cpu_count: 1 +# cpus: 0.25 networks: - tanner_local ports: