tpotce/docker/elk/logstash/dist/http_output.conf

# Input section
input {

# Fatt
  file {
    path => ["/data/fatt/log/fatt.log"]
    codec => json
    type => "Fatt"
  }

# Suricata
  file {
    path => ["/data/suricata/log/eve.json"]
    codec => json
    type => "Suricata"
  }

# P0f
  file {
    path => ["/data/p0f/log/p0f.json"]
    codec => json
    type => "P0f"
  }

# Adbhoney
  file {
    path => ["/data/adbhoney/log/adbhoney.json"]
    codec => json
    type => "Adbhoney"
  }

# Beelzebub
  file {
    path => ["/data/beelzebub/log/beelzebub.json"]
    codec => json
    type => "Beelzebub"
  }

# Ciscoasa
  file {
    path => ["/data/ciscoasa/log/ciscoasa.log"]
    codec => plain
    type => "Ciscoasa"
  }

# CitrixHoneypot
  file {
    path => ["/data/citrixhoneypot/log/server.log"]
    codec => json
    type => "CitrixHoneypot"
  }

# Conpot
  file {
    path => ["/data/conpot/log/*.json"]
    codec => json
    type => "ConPot"
  }

# Cowrie
  file {
    path => ["/data/cowrie/log/cowrie.json"]
    codec => json
    type => "Cowrie"
  }

# Dionaea
  file {
    path => ["/data/dionaea/log/dionaea.json"]
    codec => json
    type => "Dionaea"
  }

# Dicompot
  file {
    path => ["/data/dicompot/log/dicompot.log"]
    codec => json
    type => "Dicompot"
  }

# Ddospot
  file {
    path => ["/data/ddospot/log/*.log"]
    codec => json
    type => "Ddospot"
  }

# ElasticPot
  file {
    path => ["/data/elasticpot/log/elasticpot.json"]
    codec => json
    type => "ElasticPot"
  }

# Endlessh
  file {
    path => ["/data/endlessh/log/endlessh.log"]
    codec => plain
    type => "Endlessh"
  }

# Galah
  file {
    path => ["/data/galah/log/galah.json"]
    codec => json
    type => "Galah"
  }

# Glutton
  file {
    path => ["/data/glutton/log/glutton.log"]
    codec => json
    type => "Glutton"
  }

# Go-pot
  file {
    path => ["/data/go-pot/log/go-pot.json"]
    codec => json
    type => "Go-pot"
  }

# H0neytr4p
  file {
    path => ["/data/h0neytr4p/log/log.json"]
    codec => json
    type => "H0neytr4p"
  }

# Hellpot
  file {
    path => ["/data/hellpot/log/hellpot.log"]
    codec => json
    type => "Hellpot"
  }

# Heralding
  file {
    path => ["/data/heralding/log/auth.csv"]
    type => "Heralding"
  }

# Honeyaml
  file {
    path => ["/data/honeyaml/log/honeyaml.log"]
    codec => json
    type => "Honeyaml"
  }

# Honeypots
  file {
    path => ["/data/honeypots/log/*.log"]
    codec => json
    type => "Honeypots"
  }

# Honeytrap
  file {
    path => ["/data/honeytrap/log/attackers.json"]
    codec => json
    type => "Honeytrap"
  }

# Ipphoney
  file {
    path => ["/data/ipphoney/log/ipphoney.json"]
    codec => json
    type => "Ipphoney"
  }

# Log4pot
  file {
    path => ["/data/log4pot/log/log4pot.log"]
    codec => json
    type => "Log4pot"
  }

# Miniprint
  file {
    path => ["/data/miniprint/log/miniprint.json"]
    codec => json
    type => "Miniprint"
  }

# Mailoney
  file {
    path => ["/data/mailoney/log/commands.log"]
    codec => json
    type => "Mailoney"
  }

# Medpot
  file {
    path => ["/data/medpot/log/medpot.log"]
    codec => json
    type => "Medpot"
  }

# Redishoneypot
  file {
    path => ["/data/redishoneypot/log/redishoneypot.log"]
    codec => json
    type => "Redishoneypot"
  }

# Host NGINX
  file {
    path => ["/data/nginx/log/access.log"]
    codec => json
    type => "NGINX"
  }

# Sentrypeer
  file {
    path => ["/data/sentrypeer/log/sentrypeer.json"]
    codec => json
    type => "Sentrypeer"
  }

# Tanner
  file {
    path => ["/data/tanner/log/tanner_report.json"]
    codec => json
    type => "Tanner"
  }

# Wordpot
  file {
    path => ["/data/wordpot/log/wordpot.log"]
    codec => json
    type => "Wordpot"
  }

}

# Filter Section
filter {


# Fatt
  if [type] == "Fatt" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "sourceIp" => "src_ip"
	      "destinationIp" => "dest_ip"
        "sourcePort" => "src_port"
        "destinationPort" => "dest_port"
        "gquic" => "fatt_gquic"
        "http" => "fatt_http"
        "rdp" => "fatt_rdp"
        "ssh" => "fatt_ssh"
        "tls" => "fatt_tls"
      }
    }
  }

# Suricata
  if [type] == "Suricata" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    translate {
      refresh_interval => 86400
      source => "[alert][signature_id]"
      target => "[alert][cve_id]"
      dictionary_path => "/etc/listbot/cve.yaml"
#      fallback => "-"
    }
  }

# P0f
  if [type] == "P0f" {
    date {
      match => [ "timestamp", "yyyy'/'MM'/'dd HH:mm:ss" ]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "server_port" => "dest_port"
        "server_ip" => "dest_ip"
        "client_port" => "src_port"
        "client_ip" => "src_ip"
      }
    }
  }

# Adbhoney
  if [type] == "Adbhoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["unixtime"]
    }
  }

# Beelzebub
  if [type] == "Beelzebub" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Ciscoasa
  if [type] == "Ciscoasa" {
    kv {
      remove_char_key => " '{}"
      remove_char_value => "'{}"
      value_split => ":"
      field_split => ","
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      add_field => {
        "dest_ip" => "${MY_EXTIP}"
      }
    }
  }

# CitrixHoneypot
  if [type] == "CitrixHoneypot" {
    grok {
      match => {
        "message" => [ "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{JAVAMETHOD:http.http_method:string}%{SPACE}%{CISCO_REASON:fileinfo.state:string}: %{UNIXPATH:fileinfo.filename:string}",
	               "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{JAVAMETHOD:http.http_method:string}%{SPACE}%{CISCO_REASON:fileinfo.state:string}: %{GREEDYDATA:payload:string}",
		       "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{S3_REQUEST_LINE:msg:string} %{CISCO_REASON:fileinfo.state:string}: %{GREEDYDATA:payload:string:string}",
		       "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{GREEDYDATA:msg:string}" ]
      }
    }
    date {
      match => [ "asctime", "ISO8601" ]
      remove_field => ["asctime"]
      remove_field => ["message"]
    }
    mutate {
      add_field => {
        "dest_port" => "443"
      }
      rename => {
        "levelname" => "level"
      }
    }
  }

# Conpot
  if [type] == "ConPot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
      }
    }
  }

# Cowrie
  if [type] == "Cowrie" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
      }
    }
  }

# Ddospot
  if [type] == "Ddospot" {
    date {
      match => [ "time", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
      remove_field => ["time"]
    }
    if [path] == "/data/ddospot/log/chargenpot.log" {
      mutate {
        add_field => {
	  "dest_port" => "19"
          "dest_ip" => "${MY_EXTIP}"
	}
      }
    }
    if [path] == "/data/ddospot/log/dnspot.log" {
      mutate {
        add_field => {
          "dest_port" => "53"
          "dest_ip" => "${MY_EXTIP}"
        }
      }
    }
    if [path] == "/data/ddospot/log/ntpot.log" {
      mutate {
        add_field => {
          "dest_port" => "123"
          "dest_ip" => "${MY_EXTIP}"
        }
      }
    }
    if [path] == "/data/ddospot/log/ssdpot.log" {
      mutate {
        add_field => {
          "dest_port" => "1900"
          "dest_ip" => "${MY_EXTIP}"
        }
      }
    }
  }

# Dionaea
  if [type] == "Dionaea" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
      }
      gsub => [
        "src_ip", "::ffff:", "",
        "dest_ip", "::ffff:", ""
      ]
    }
    if [credentials] {
      mutate {
        add_field => {
          "username" => "%{[credentials][username]}"
          "password" => "%{[credentials][password]}"
        }
        remove_field => "[credentials]"
      }
    }
  }

# Dicompot
  if [type] == "Dicompot" {
    date {
      match => [ "time", "yyyy-MM-dd HH:mm:ss" ]
      remove_field => ["time"]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "ID" => "id"
        "IP" => "src_ip"
        "Port" => "src_port"
        "AETitle" => "aetitle"
        "Command" => "input"
        "Files" => "files"
        "Identifier" => "identifier"
        "Matches" => "matches"
        "Status" => "session"
        "Version" => "version"
      }
    }
  }

# ElasticPot
  if [type] == "ElasticPot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "content_type" => "http.http_content_type"
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
        "message" => "event_type"
        "request" => "request_method"
        "user_agent" => "http_user_agent"
	"url" => "http.url"
      }
    }
  }

# Endlessh
# Example: 2021-10-29T21:08:31.026Z CLOSE host=1.2.3.4 port=12345 fd=4 time=20.015 bytes=24
# Example: 2021-10-29T21:08:11.011Z ACCEPT host=1.2.3.4 port=12346 fd=4 n=1/4096
  if [type] == "Endlessh" {
    grok { match => { "message" => [ "\A%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:reason}%{SPACE}host=%{IPV4:src_ip}%{SPACE}port=%{INT:src_port}%{SPACE}fd=%{INT}%{SPACE}time=%{SECOND:duration}%{SPACE}bytes=%{NUMBER:bytes}", "\A%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:reason}%{SPACE}host=%{IPV4:src_ip}%{SPACE}port=%{INT:src_port}%{SPACE}fd=%{INT}%{SPACE}n=%{INT}/%{INT}" ] } }
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["timestamp"]
    }
    mutate {
      add_field => {
        "dest_port" => "22"
        "dest_ip" => "${MY_EXTIP}"
      }
    }
  }

# Galah
  if [type] == "Galah" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Glutton
  if [type] == "Glutton" {
    date {
      match => [ "ts", "UNIX" ]
      remove_field => ["ts"]
    }
  }

# Go-pot
  if [type] == "Go-pot" {
    if ! [dest_port] {
      drop {}
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      remove_field => ["ts"]
      rename => {
        "duration" => "session_duration"
      }
    }
  }

# Hellpot
  if [type] == "Hellpot" {
    date {
      match => [ "time", "ISO8601" ]
      remove_field => ["time"]
      remove_field => ["timestamp"]
    }
    mutate {
      add_field => {
        "dest_port" => "80"
        "dest_ip" => "${MY_EXTIP}"
      }
      rename => {
        "BYTES" => "bytes"
        "DURATION" => "duration"
        "REMOTE_ADDR" => "src_ip"
        "URL" => "url"
        "USERAGENT" => "http_user_agent"
        "message" => "reason"
      }
    }
  }

# H0neytr4p
  if [type] == "H0neytr4p" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Heralding
  if [type] == "Heralding" {
    csv {
      columns => ["timestamp","auth_id","session_id","src_ip","src_port","dest_ip","dest_port","proto","username","password"] separator => ","
    }
    date {
      match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
      remove_field => ["timestamp"]
    }
  }

# Honeyaml
  if [type] == "Honeyaml" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Honeypots
  if [type] == "Honeypots" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Honeytrap
  if [type] == "Honeytrap" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "[attack_connection][local_port]" => "dest_port"
        "[attack_connection][local_ip]" => "dest_ip"
        "[attack_connection][remote_port]" => "src_port"
        "[attack_connection][remote_ip]" => "src_ip"
      }
    }
  }

# Ipphoney
  if [type] == "Ipphoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
	"query" => "ipp_query"
        "content_type" => "http.http_content_type"
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
        "request" => "request_method"
        "operation" => "data"
        "user_agent" => "http_user_agent"
        "url" => "http.url"
      }
    }
  }

# Log4pot
  if [type] == "Log4pot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "request" => "request_uri"
        "server_port" => "dest_port"
        "port" => "src_port"
        "client" => "src_ip"
      }
    }
  }

# Mailoney
  if [type] == "Mailoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      add_field => { "dest_port" => "25" }
    }
  }

# Medpot
  if [type] == "Medpot" {
    mutate {
      add_field => {
        "dest_port" => "2575"
        "dest_ip" => "${MY_EXTIP}"
      }
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Miniprint
  if [type] == "Miniprint" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Redishoneypot
  if [type] == "Redishoneypot" {
    date {
      match => [ "time", "yyyy-MM-dd HH:mm:ss" ]
      remove_field => ["time"]
      remove_field => ["timestamp"]
    }
    mutate {
      split => { "addr" => ":" }
      add_field => {
        "src_ip" => "%{[addr][0]}"
        "src_port" => "%{[addr][1]}"
        "dest_port" => "6379"
        "dest_ip" => "${MY_EXTIP}"
      }
      remove_field => ["addr"]
    }
  }

# NGINX
  if [type] == "NGINX" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "request" => "request_data"
      }
    }
  }

# Sentrypeer
  if [type] == "Sentrypeer" {
    date {
      match => [ "event_timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSSSSS" ]
      remove_field => ["event_timestamp"]
    }
    mutate {
      rename => {
        "source_ip" => "src_ip"
        "destination_ip" => "dest_ip"
      }
      add_field => { "dest_port" => "5060" }
    }
  }

# Tanner
  if [type] == "Tanner" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "[peer][ip]" => "src_ip"
        "[peer][port]" => "src_port"
      }
      add_field => { "dest_port" => "80" }
    }
  }

# Wordpot
  if [type] == "Wordpot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Drop if parse fails
if "_grokparsefailure" in [tags] { drop {} }
if "_jsonparsefailure" in [tags] { drop {} }

# Add T-Pot hostname and external IP
  mutate {
    add_field => {
      "t-pot_ip_ext" => "${MY_EXTIP}"
      "t-pot_ip_int" => "${MY_INTIP}"
      "t-pot_hostname" => "${MY_HOSTNAME}"
    }
  }

# Add geo coordinates / ASN info / IP rep.
  if [src_ip] {
    geoip {
      cache_size => 10000
      source => "src_ip"
      default_database_type => "City"
#      database => "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-filter-geoip-7.2.13-java/vendor/GeoLite2-City.mmdb"
    }
    geoip {
      cache_size => 10000
      source => "src_ip"
      default_database_type => "ASN"
#      database => "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-filter-geoip-7.2.13-java/vendor/GeoLite2-ASN.mmdb"
    }
    translate {
      refresh_interval => 86400
      source => "src_ip"
      target => "ip_rep"
      dictionary_path => "/etc/listbot/iprep.yaml"
    }
  }
  if [t-pot_ip_ext]  {
    geoip {
      cache_size => 10000
      source => "t-pot_ip_ext"
      target => "geoip_ext"
      default_database_type => "City"
#      database => "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-filter-geoip-7.2.13-java/vendor/GeoLite2-City.mmdb"
    }
    geoip {
      cache_size => 10000
      source => "t-pot_ip_ext"
      target => "geoip_ext"
      default_database_type => "ASN"
#      database => "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-filter-geoip-7.2.13-java/vendor/GeoLite2-ASN.mmdb"
    }
  }

# In some rare conditions dest_port, src_port, status are indexed as string, forcing integer for now
  if [dest_port] {
    mutate {
        convert => { "dest_port" => "integer" }
    }
  }
  if [src_port] {
    mutate {
        convert => { "src_port" => "integer" }
    }
  }
  if [status] {
    mutate {
        convert => { "status" => "integer" }
    }
  }
  if [id] {
    mutate {
        convert => { "id" => "string" }
    }
  }
  if [request] {
    mutate {
        convert => { "request" => "string" }
    }
  }

}

# Output section
output {
    http {
      http_method => "post"
      http_compression => true
      id => "${MY_HOSTNAME}"
      codec => "json"
      format => "json_batch"
      url => "https://${TPOT_HIVE_IP}:64294"
#      cacert => "/data/hive.crt"
      ssl_verification_mode => "${LS_SSL_VERIFICATION}"
      ssl_certificate_authorities => "/data/hive.crt"
      headers => {
        "Authorization" => "Basic ${TPOT_HIVE_USER}"
      }
    }
}