tpotce/docker/elk/logstash/dist/logstash.conf

# Input section
input {

# Fatt                                 
  file {                                   
    path => ["/data/fatt/log/fatt.log"]
    codec => json     
    type => "Fatt"
  }  

# Suricata
  file {
    path => ["/data/suricata/log/eve.json"]
    codec => json
    type => "Suricata"
  }

# P0f
  file {
    path => ["/data/p0f/log/p0f.json"]
    codec => json
    type => "P0f"
  }

# Adbhoney
  file {
    path => ["/data/adbhoney/log/adbhoney.json"]
    codec => json
    type => "Adbhoney"
  }

# Ciscoasa
  file {
    path => ["/data/ciscoasa/log/ciscoasa.log"]
    codec => plain
    type => "Ciscoasa"
  }

# CitrixHoneypot
  file {
    path => ["/data/citrixhoneypot/logs/server.log"]
    codec => json
    type => "CitrixHoneypot"
  }

# Conpot
  file {
    path => ["/data/conpot/log/*.json"]
    codec => json
    type => "ConPot"
  }

# Cowrie
  file {
    path => ["/data/cowrie/log/cowrie.json"]
    codec => json
    type => "Cowrie"
  }

# Dionaea
  file {
    path => ["/data/dionaea/log/dionaea.json"]
    codec => json
    type => "Dionaea"
  }

# Dicompot
  file {
    path => ["/data/dicompot/log/dicompot.log"]
    codec => json
    type => "Dicompot"
  }

# ElasticPot
  file {
    path => ["/data/elasticpot/log/elasticpot.json"]
    codec => json
    type => "ElasticPot"
  }

# Glutton
  file {
    path => ["/data/glutton/log/glutton.log"]
    codec => json
    type => "Glutton"
  }

# Heralding
  file {
    path => ["/data/heralding/log/auth.csv"]
    type => "Heralding"
  }

# Honeypy
  file {
    path => ["/data/honeypy/log/json.log"]
    codec => json
    type => "Honeypy"
  }

# Honeysap
  file {
    path => ["/data/honeysap/log/honeysap-external.log"]
    codec => json
    type => "Honeysap"
  }

# Honeytrap
  file {
    path => ["/data/honeytrap/log/attackers.json"]
    codec => json
    type => "Honeytrap"
  }

# Mailoney
  file {
    path => ["/data/mailoney/log/commands.log"]
    codec => json
    type => "Mailoney"
  }

# Medpot
  file {
    path => ["/data/medpot/log/medpot.log"]
    codec => json
    type => "Medpot"
  }

# Rdpy
  file {
    path => ["/data/rdpy/log/rdpy.log"]
    type => "Rdpy"
  }

# Host NGINX
  file {
    path => ["/data/nginx/log/access.log"]
    codec => json
    type => "NGINX"
  }

# Tanner
  file {
    path => ["/data/tanner/log/tanner_report.json"]
    codec => json
    type => "Tanner"
  }

}

# Filter Section
filter {


# Fatt
  if [type] == "Fatt" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "sourceIp" => "src_ip"
	"destinationIp" => "dest_ip"
	"sourcePort" => "src_port"
	"destinationPort" => "dest_port"
        "gquic" => "fatt_gquic"
        "http" => "fatt_http"
        "rdp" => "fatt_rdp"
        "ssh" => "fatt_ssh"
        "tls" => "fatt_tls"
      }
    }
  }

# Suricata
  if [type] == "Suricata" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    translate {
      refresh_interval => 86400
      field => "[alert][signature_id]"
      destination => "[alert][cve_id]"
      dictionary_path => "/etc/listbot/cve.yaml"
#      fallback => "-"
    }
  }

# P0f
  if [type] == "P0f" {
    date {
      match => [ "timestamp", "yyyy'/'MM'/'dd HH:mm:ss" ]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "server_port" => "dest_port"
        "server_ip" => "dest_ip"
        "client_port" => "src_port"
        "client_ip" => "src_ip"
      }
    }
  }

# Adbhoney
  if [type] == "Adbhoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["unixtime"]
    }
  }

# Ciscoasa
  if [type] == "Ciscoasa" {
    kv {
      remove_char_key => " '{}"
      remove_char_value => "'{}"
      value_split => ":"
      field_split => ","
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      add_field => {
        "dest_ip" => "${MY_EXTIP}"
      }
    }
  }

# CitrixHoneypot
  if [type] == "CitrixHoneypot" {
    grok { 
      match => { 
        "message" => [ "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{JAVAMETHOD:http.http_method:string}%{SPACE}%{CISCO_REASON:fileinfo.state:string}: %{UNIXPATH:fileinfo.filename:string}", 
	               "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{JAVAMETHOD:http.http_method:string}%{SPACE}%{CISCO_REASON:fileinfo.state:string}: %{GREEDYDATA:payload:string}", 
		       "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{S3_REQUEST_LINE:msg:string} %{CISCO_REASON:fileinfo.state:string}: %{GREEDYDATA:payload:string:string}",
		       "\A\(%{IPV4:src_ip:string}:%{INT:src_port:integer}\): %{GREEDYDATA:msg:string}" ] 
      } 
    }
    date {
      match => [ "asctime", "ISO8601" ]
      remove_field => ["asctime"]
      remove_field => ["message"]
    }
    mutate {
      add_field => {
        "dest_port" => "443"
      }
      rename => {
        "levelname" => "level"
      }
    }
  }
  
# Conpot
  if [type] == "ConPot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate { 
      rename => { 
        "dst_port" => "dest_port" 
        "dst_ip" => "dest_ip" 
      } 
    } 
  }

# Cowrie
  if [type] == "Cowrie" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
      }
    }
  }

# Dionaea
  if [type] == "Dionaea" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
      }
      gsub => [
        "src_ip", "::ffff:", "",
        "dest_ip", "::ffff:", ""
      ]
    }
    if [credentials] {
      mutate {
        add_field => {
          "username" => "%{[credentials][username]}"
          "password" => "%{[credentials][password]}"
        }
        remove_field => "[credentials]"
      }
    }
  }

# Dicompot
  if [type] == "Dicompot" {
    date {
      match => [ "time", "yyyy-MM-dd HH:mm:ss" ]
      remove_field => ["time"]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "[Address][IP]" => "src_ip"
        "[Address][Port]" => "src_port"
        "[Address][Zone]" => "zone"
        "AETitle" => "aetitle"
        "Command" => "input"
        "Files" => "files"
        "Identifier" => "identifier"
        "Matches" => "matches"
        "Status" => "session"
        "Version" => "version"
      }
    }
    if [Address] {
      mutate {
        remove_field => "[Address]"
      }
    }
  }

# ElasticPot
  if [type] == "ElasticPot" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "content_type" => "http.http_content_type"
        "dst_port" => "dest_port"
        "dst_ip" => "dest_ip"
        "message" => "event_type"
        "request" => "request_method"
        "user_agent" => "http_user_agent"
	"url" => "http.url"
      }
    }
  }

# Glutton
  if [type] == "Glutton" {
    date {
      match => [ "ts", "UNIX" ]
      remove_field => ["ts"]
    }
  }

# Heralding
  if [type] == "Heralding" {
    csv {
      columns => ["timestamp","auth_id","session_id","src_ip","src_port","dest_ip","dest_port","proto","username","password"] separator => ","
    }
    date {
      match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
      remove_field => ["timestamp"]
    }
  }

# Honeypy
  if [type] == "Honeypy" {
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["timestamp"]
      remove_field => ["date"]
      remove_field => ["time"]
      remove_field => ["millisecond"]
    }
  }

# Honeysap
  if [type] == "Honeysap" {
    date {
      match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
      remove_field => ["timestamp"]
    }
    mutate {
      rename => {
        "[data][error_msg]" => "event_type"
        "service" => "sensor"
        "source_port" => "src_port"
        "source_ip" => "src_ip"
        "target_port" => "dest_port"
        "target_ip" => "dest_ip"
      }
      remove_field => "event"
      remove_field => "return_code"
    }
    if [data] {
      mutate {
	remove_field => "[data]"
      }
    }    
  }

# Honeytrap
  if [type] == "Honeytrap" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "[attack_connection][local_port]" => "dest_port"
        "[attack_connection][local_ip]" => "dest_ip"
        "[attack_connection][remote_port]" => "src_port"
        "[attack_connection][remote_ip]" => "src_ip"
      }
    }
  }

# Mailoney
  if [type] == "Mailoney" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      add_field => {
        "dest_port" => "25"
      }
    }
  }

# Medpot
  if [type] == "Medpot" {
    mutate {
      add_field => {
        "dest_port" => "2575"
        "dest_ip" => "${MY_EXTIP}"
      }
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Rdpy
  if [type] == "Rdpy" {
    grok { match => { "message" => [ "\A%{TIMESTAMP_ISO8601:timestamp},domain:%{CISCO_REASON:domain},username:%{CISCO_REASON:username},password:%{CISCO_REASON:password},hostname:%{GREEDYDATA:hostname}", "\A%{TIMESTAMP_ISO8601:timestamp},Connection from %{IPV4:src_ip}:%{INT:src_port:integer}" ] } }
    date {
      match => [ "timestamp", "ISO8601" ]
      remove_field => ["timestamp"]
    }
    mutate {
      add_field => {
        "dest_port" => "3389"
      }
    }
  }

# NGINX
  if [type] == "NGINX" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }

# Tanner
  if [type] == "Tanner" {
    date {
      match => [ "timestamp", "ISO8601" ]
    }
    mutate {
      rename => {
        "[peer][ip]" => "src_ip"
        "[peer][port]" => "src_port"
      }
      add_field => {
        "dest_port" => "80"
      }
    }
  }

# Drop if parse fails
if "_grokparsefailure" in [tags] { drop {} }

# Add geo coordinates / ASN info / IP rep.
  if [src_ip]  {
    geoip {
      cache_size => 10000
      source => "src_ip"
      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-City.mmdb"
    }
    geoip {
      cache_size => 10000
      source => "src_ip"
      database => "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-filter-geoip-6.0.3-java/vendor/GeoLite2-ASN.mmdb"
    }
    translate {
      refresh_interval => 86400
      field => "src_ip"
      destination => "ip_rep"
      dictionary_path => "/etc/listbot/iprep.yaml"
    }
  }

# In some rare conditions dest_port, src_port, status are indexed as string, forcing integer for now
  if [dest_port] {
    mutate {
        convert => { "dest_port" => "integer" }
    }
  }
  if [src_port] {
    mutate {
        convert => { "src_port" => "integer" }
    }
  }
  if [status] {
    mutate {
        convert => { "status" => "integer" }
    }
  }

# Add T-Pot hostname and external IP
  if [type] == "Adbhoney" or [type] == "Ciscoasa" or [type] == "CitrixHoneypot" or [type] == "ConPot" or [type] == "Cowrie" or [type] == "Dicompot" or [type] == "Dionaea" or [type] == "ElasticPot" or [type] == "Fatt" or [type] == "Glutton" or [type] == "Honeysap" or [type] == "Honeytrap" or [type] == "Heralding" or [type] == "Honeypy" or [type] == "Mailoney" or [type] == "Medpot" or [type] == "P0f" or [type] == "Rdpy" or [type] == "Suricata" or [type] == "Tanner" {
    mutate {
      add_field => {
        "t-pot_ip_ext" => "${MY_EXTIP}"
        "t-pot_ip_int" => "${MY_INTIP}"
        "t-pot_hostname" => "${MY_HOSTNAME}"
      }
    }
  }

}

# Output section
output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    # With ILM in place we need to set the daily index manually, if not => FUBAR
    index => "logstash-%{+YYYY.MM.dd}"
#    document_type => "doc"
  }

  #if [type] == "Suricata" {
  #    file {
  #      file_mode => 0770
  #      path => "/data/suricata/log/suricata_ews.log"
  #    }
  #}
  # Debug output
  #if [type] == "CitrixHoneypot" {
  #  stdout {
  #    codec => rubydebug
  #  }
  #}
  # Debug output
  #stdout {
  #  codec => rubydebug
  #}

}