tpotce/docker/elk/logstash/dist/http_output.conf

# Input section
input {

# Fatt                                 
  file {                                   
    path => ["/data/fatt/log/fatt.log"]
    codec => json     
    type => "Fatt"
  }  

# Suricata
  file {
    path => ["/data/suricata/log/eve.json"]
    codec => json
    type => "Suricata"
  }

# P0f
  file {
    path => ["/data/p0f/log/p0f.json"]
    codec => json
    type => "P0f"
  }

# Adbhoney
  file {
    path => ["/data/adbhoney/log/adbhoney.json"]
    codec => json
    type => "Adbhoney"
  }

# Ciscoasa
  file {
    path => ["/data/ciscoasa/log/ciscoasa.log"]
    codec => plain
    type => "Ciscoasa"
  }

# CitrixHoneypot
  file {
    path => ["/data/citrixhoneypot/logs/server.log"]
    codec => json
    type => "CitrixHoneypot"
  }

# Conpot
  file {
    path => ["/data/conpot/log/*.json"]
    codec => json
    type => "ConPot"
  }

# Cowrie
  file {
    path => ["/data/cowrie/log/cowrie.json"]
    codec => json
    type => "Cowrie"
  }

# Dionaea
  file {
    path => ["/data/dionaea/log/dionaea.json"]
    codec => json
    type => "Dionaea"
  }

# Dicompot
  file {
    path => ["/data/dicompot/log/dicompot.log"]
    codec => json
    type => "Dicompot"
  }

# Ddospot
  file {
    path => ["/data/ddospot/log/*.log"]
    codec => json
    type => "Ddospot"
  }

# ElasticPot
  file {
    path => ["/data/elasticpot/log/elasticpot.json"]
    codec => json
    type => "ElasticPot"
  }

# Endlessh
  file {
    path => ["/data/endlessh/log/endlessh.log"]
    codec => plain
    type => "Endlessh"
  }

# Glutton
  file {
    path => ["/data/glutton/log/glutton.log"]
    codec => json
    type => "Glutton"
  }

# Hellpot
  file {
    path => ["/data/hellpot/log/hellpot.log"]
    codec => json
    type => "Hellpot"
  }

# Heralding
  file {
    path => ["/data/heralding/log/auth.csv"]
    type => "Heralding"
  }

# Honeypots
  file {
    path => ["/data/honeypots/log/*.log"]
    codec => json
    type => "Honeypots"
  }

# Honeypy
  file {
    path => ["/data/honeypy/log/json.log"]
    codec => json
    type => "Honeypy"
  }

# Honeysap
  file {
    path => ["/data/honeysap/log/honeysap-external.log"]
    codec => json
    type => "Honeysap"
  }

# Honeytrap
  file {
    path => ["/data/honeytrap/log/attackers.json"]
    codec => json
    type => "Honeytrap"
  }

# Ipphoney
  file {
    path => ["/data/ipphoney/log/ipphoney.json"]
    codec => json
    type => "Ipphoney"
  }

# Mailoney
  file {
    path => ["/data/mailoney/log/commands.log"]
    codec => json
    type => "Mailoney"
  }

# Medpot
  file {
    path => ["/data/medpot/log/medpot.log"]
    codec => json
    type => "Medpot"
  }

# Rdpy
  file {
    path => ["/data/rdpy/log/rdpy.log"]
    type => "Rdpy"
  }

# Redishoneypot
  file {
    path => ["/data/redishoneypot/log/redishoneypot.log"]
    codec => json
    type => "Redishoneypot"
  }

# Host NGINX
  file {
    path => ["/data/nginx/log/access.log"]
    codec => json
    type => "NGINX"
  }

# Tanner
  file {
    path => ["/data/tanner/log/tanner_report.json"]
    codec => json
    type => "Tanner"
  }

}

# Filter Section
filter {


# Fatt
  if [type] == "Fatt" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "sourceIp" => "src_ip"
	"destinationIp" => "dest_ip"
	"sourcePort" => "src_port"
	"destinationPort" => "dest_port"
        "gquic" => "fatt_gquic"
        "http" => "fatt_http"
        "rdp" => "fatt_rdp"
        "ssh" => "fatt_ssh"
        "tls" => "fatt_tls"
      }
    }
  }

# Suricata
  if [type] == "Suricata" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    translate {
      refresh_interval => 86400
      field => "[alert][signature_id]"
      destination => "[alert][cve_id]"
      dictionary_path => "/etc/listbot/cve.yaml"
#      fallback => "-"
    }
  }

# P0f
  if [type] == "P0f" {
    date {
      match => [ "timestamp", "yyyy'/'MM'/'dd HH:mm:ss" ]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "server_port" => "dest_port"
        "server_ip" => "dest_ip"
        "client_port" => "src_port"
        "client_ip" => "src_ip"
      }
    }
  }

# Adbhoney
  if [type] == "Adbhoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["unixtime"]
    }
  }

# Ciscoasa
  if [type] == "Ciscoasa" {
    kv {
      remove_char_key => " '{}"
      remove_char_value => "'{}"
      value_split => ":"
      field_split => ","
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      add_field => {
        "dest_ip" => "${MY_EXTIP}"
      }
    }
  }

# CitrixHoneypot
  if [type] == "CitrixHoneypot" {
    grok { 
      match => { 
        "message" => [ "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{JAVAMETHOD:http.http_method:string}%{SPACE}%{CISCO_REASON:fileinfo.state:string}: %{UNIXPATH:fileinfo.filename:string}", 
	               "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{JAVAMETHOD:http.http_method:string}%{SPACE}%{CISCO_REASON:fileinfo.state:string}: %{GREEDYDATA:payload:string}", 
		       "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{S3_REQUEST_LINE:msg:string} %{CISCO_REASON:fileinfo.state:string}: %{GREEDYDATA:payload:string:string}",
		       "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{GREEDYDATA:msg:string}" ] 
      } 
    }
    date {
      match => [ "asctime", "ISO8601" ]
      remove_field => ["asctime"]
      remove_field => ["message"]
    }
    mutate {
      add_field => {
        "dest_port" => "443"
      }
      rename => {
        "levelname" => "level"
      }
    }
  }
  
# Conpot
  if [type] == "ConPot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate { 
      rename => { 
        "dst_port" => "dest_port" 
        "dst_ip" => "dest_ip" 
      } 
    } 
  }

# Cowrie
  if [type] == "Cowrie" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
      }
    }
  }

# Ddospot
  if [type] == "Ddospot" {
    date {
      match => [ "time", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
      remove_field => ["time"]
    }
    if [path] == "/data/ddospot/log/chargenpot.log" {
      mutate {
        add_field => {
	  "dest_port" => "19"
          "dest_ip" => "${MY_EXTIP}"
	}
      }
    }
    if [path] == "/data/ddospot/log/dnspot.log" {
      mutate {
        add_field => {
          "dest_port" => "53"
          "dest_ip" => "${MY_EXTIP}"
        }
      }
    }
    if [path] == "/data/ddospot/log/ntpot.log" {
      mutate {
        add_field => {
          "dest_port" => "123"
          "dest_ip" => "${MY_EXTIP}"
        }
      }
    }
    if [path] == "/data/ddospot/log/ssdpot.log" {
      mutate {
        add_field => {
          "dest_port" => "1900"
          "dest_ip" => "${MY_EXTIP}"
        }
      }
    }
  }

# Dionaea
  if [type] == "Dionaea" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
      }
      gsub => [
        "src_ip", "::ffff:", "",
        "dest_ip", "::ffff:", ""
      ]
    }
    if [credentials] {
      mutate {
        add_field => {
          "username" => "%{[credentials][username]}"
          "password" => "%{[credentials][password]}"
        }
        remove_field => "[credentials]"
      }
    }
  }

# Dicompot
  if [type] == "Dicompot" {
    date {
      match => [ "time", "yyyy-MM-dd HH:mm:ss" ]
      remove_field => ["time"]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "ID" => "id"
        "IP" => "src_ip"
        "Port" => "src_port"
        "AETitle" => "aetitle"
        "Command" => "input"
        "Files" => "files"
        "Identifier" => "identifier"
        "Matches" => "matches"
        "Status" => "session"
        "Version" => "version"
      }
    }
  }

# ElasticPot
  if [type] == "ElasticPot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "content_type" => "http.http_content_type"
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
        "message" => "event_type"
        "request" => "request_method"
        "user_agent" => "http_user_agent"
	"url" => "http.url"
      }
    }
  }

# Endlessh
# Example: 2021-10-29T21:08:31.026Z CLOSE host=1.2.3.4 port=12345 fd=4 time=20.015 bytes=24
# Example: 2021-10-29T21:08:11.011Z ACCEPT host=1.2.3.4 port=12346 fd=4 n=1/4096
  if [type] == "Endlessh" {
    grok { match => { "message" => [ "\A%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:reason}%{SPACE}host=%{IPV4:src_ip}%{SPACE}port=%{INT:src_port}%{SPACE}fd=%{INT}%{SPACE}time=%{SECOND:duration}%{SPACE}bytes=%{NUMBER:bytes}", "\A%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:reason}%{SPACE}host=%{IPV4:src_ip}%{SPACE}port=%{INT:src_port}%{SPACE}fd=%{INT}%{SPACE}n=%{INT}/%{INT}" ] } }    
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["timestamp"]
    }
    mutate {
      add_field => {
        "dest_port" => "22"
        "dest_ip" => "${MY_EXTIP}"
      }
    }
  }

# Glutton
  if [type] == "Glutton" {
    date {
      match => [ "ts", "UNIX" ]
      remove_field => ["ts"]
    }
  }

# Hellpot
  if [type] == "Hellpot" {
    date {
      match => [ "time", "ISO8601" ]
      remove_field => ["time"]
      remove_field => ["timestamp"]
    }
    mutate {
      add_field => {
        "dest_port" => "80"
        "dest_ip" => "${MY_EXTIP}"
      }
      rename => {
        "BYTES" => "bytes"
        "DURATION" => "duration"
        "REMOTE_ADDR" => "src_ip"
        "URL" => "url"
        "USERAGENT" => "http_user_agent"
        "message" => "reason"
      }
    }
  }

# Heralding
  if [type] == "Heralding" {
    csv {
      columns => ["timestamp","auth_id","session_id","src_ip","src_port","dest_ip","dest_port","proto","username","password"] separator => ","
    }
    date {
      match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
      remove_field => ["timestamp"]
    }
  }

# Honeypy
  if [type] == "Honeypy" {
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["timestamp"]
      remove_field => ["date"]
      remove_field => ["time"]
      remove_field => ["millisecond"]
    }
  }

# Honeypots
  if [type] == "Honeypots" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Honeysap
  if [type] == "Honeysap" {
    date {
      match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "[data][error_msg]" => "event_type"
        "service" => "sensor"
        "source_port" => "src_port"
        "source_ip" => "src_ip"
        "target_port" => "dest_port"
        "target_ip" => "dest_ip"
      }
      remove_field => "event"
      remove_field => "return_code"
    }
    if [data] {
      mutate {
	remove_field => "[data]"
      }
    }    
  }

# Honeytrap
  if [type] == "Honeytrap" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "[attack_connection][local_port]" => "dest_port"
        "[attack_connection][local_ip]" => "dest_ip"
        "[attack_connection][remote_port]" => "src_port"
        "[attack_connection][remote_ip]" => "src_ip"
      }
    }
  }

# Ipphoney
  if [type] == "Ipphoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
	"query" => "ipp_query"
        "content_type" => "http.http_content_type"
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
        "request" => "request_method"
        "operation" => "data"
        "user_agent" => "http_user_agent"
        "url" => "http.url"
      }
    }
  }

# Mailoney
  if [type] == "Mailoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      add_field => { "dest_port" => "25" }
    }
  }

# Medpot
  if [type] == "Medpot" {
    mutate {
      add_field => {
        "dest_port" => "2575"
        "dest_ip" => "${MY_EXTIP}"
      }
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Rdpy
  if [type] == "Rdpy" {
    grok { match => { "message" => [ "\A%{TIMESTAMP_ISO8601:timestamp},domain:%{CISCO_REASON:domain},username:%{CISCO_REASON:username},password:%{CISCO_REASON:password},hostname:%{GREEDYDATA:hostname}", "\A%{TIMESTAMP_ISO8601:timestamp},Connection from %{IPV4:src_ip}:%{INT:src_port:integer}" ] } }
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["timestamp"]
    }
    mutate {
      add_field => { "dest_port" => "3389" }
    }
  }

# Redishoneypot
  if [type] == "Redishoneypot" {
    date {
      match => [ "time", "yyyy-MM-dd HH:mm:ss" ]
      remove_field => ["time"]
      remove_field => ["timestamp"]
    }
    mutate {
      split => { "addr" => ":" }
      add_field => { 
        "src_ip" => "%{[addr][0]}" 
        "src_port" => "%{[addr][1]}"
        "dest_port" => "6379"
        "dest_ip" => "${MY_EXTIP}"
      }
      remove_field => ["addr"]
    }
  }

# NGINX
  if [type] == "NGINX" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Tanner
  if [type] == "Tanner" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "[peer][ip]" => "src_ip"
        "[peer][port]" => "src_port"
      }
      add_field => { "dest_port" => "80" }
    }
  }

# Drop if parse fails
if "_grokparsefailure" in [tags] { drop {} }
if "_jsonparsefailure" in [tags] { drop {} }


# Add geo coordinates / ASN info / IP rep.
  if [src_ip]  {
    geoip {
      cache_size => 10000
      source => "src_ip"
      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.3-java/vendor/GeoLite2-City.mmdb"
    }
    geoip {
      cache_size => 10000
      source => "src_ip"
      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-7.2.3-java/vendor/GeoLite2-ASN.mmdb"
    }
    translate {
      refresh_interval => 86400
      field => "src_ip"
      destination => "ip_rep"
      dictionary_path => "/etc/listbot/iprep.yaml"
    }
  }

# In some rare conditions dest_port, src_port, status are indexed as string, forcing integer for now
  if [dest_port] {
    mutate {
        convert => { "dest_port" => "integer" }
    }
  }
  if [src_port] {
    mutate {
        convert => { "src_port" => "integer" }
    }
  }
  if [status] {
    mutate {
        convert => { "status" => "integer" }
    }
  }
  if [id] {
    mutate {
        convert => { "id" => "string" }
    }
  }
  if [request] {
    mutate {
        convert => { "request" => "string" }
    }
  }

# Add T-Pot hostname and external IP
  mutate {
    add_field => {
      "t-pot_ip_ext" => "${MY_EXTIP}"
      "t-pot_ip_int" => "${MY_INTIP}"
      "t-pot_hostname" => "${MY_HOSTNAME}"
    }
  }

}

# Output section
output {
    http {
      http_method => "post"
      http_compression => true
      id => "${MY_HOSTNAME}"
      codec => "json"
      url => "http://127.0.0.1:64305"
    }

}