NGINX logs are in /data/nginx/log/

Syslog should be viewed in Cockpit now, storing events of Syslog in ELK
is ineffective

docker/elk/docker-compose.yml

@@ -54,7 +54,6 @@ services:
     image: "dtagdevsec/logstash:1804"
     volumes:
      - /data:/data
-     - /var/log:/data/host/log
      - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf
 
 ## Elasticsearch-head service

docker/elk/logstash/dist/logstash.conf

@@ -22,7 +22,7 @@ input {
     type => "Ciscoasa"
   }
 
-# Conpot 
+# Conpot
   file {
     path => ["/data/conpot/log/*.json"]
     codec => json
@@ -82,7 +82,7 @@ input {
     type => "Honeytrap"
   }
 
-# Mailoney 
+# Mailoney
   file {
     path => ["/data/mailoney/log/commands.log"]
     type => "Mailoney"
@@ -94,21 +94,14 @@ input {
     type => "Rdpy"
   }
 
-# Host Syslog
-  file {
-    path => ["/data/host/log/auth.log"]
-    codec => plain
-    type => "Syslog"
-  }
-
 # Host NGINX
   file {
-    path => ["/data/host/log/nginx/access.log"]
+    path => ["/data/nginx/log/access.log"]
     codec => json
     type => "NGINX"
   }
 
-# Tanner 
+# Tanner
   file {
     path => ["/data/tanner/log/tanner_report.json"]
     codec => json
@@ -256,8 +249,8 @@ filter {
 
 # Heralding
   if [type] == "Heralding" {
-    csv { 
+    csv {
-      columns => ["timestamp","auth_id","session_id","src_ip","src_port","dest_ip","dest_port","proto","username","password"] separator => "," 
+      columns => ["timestamp","auth_id","session_id","src_ip","src_port","dest_ip","dest_port","proto","username","password"] separator => ","
     }
     date {
       match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSSSSS" ]
@@ -310,79 +303,6 @@ filter {
     }
   }
 
-# Syslog
-  if [type] == "Syslog" {
-      grok {
-        match => {
-          "message" => ["%{SYSLOGPAMSESSION}", "%{CRONLOG}", "%{SYSLOGLINE}"]
-        }
-        overwrite => "message"
-      }
-      date {
-        match => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
-        remove_field => ["timestamp"]
-      }
-      date {
-        match => ["timestamp8601", "ISO8601"]
-        remove_field => ["timestamp8601"]
-      }
-      grok {
-        match => { "message" => "Connection closed by %{IP:src_ip}" }
-        add_tag => [ "ssh_connection_closed" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "Received disconnect from %{IP:src_ip}" }
-        add_tag => [ "ssh_connection_disconnect" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "Failed password for invalid user %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2" }
-        add_tag => [ "ssh_failed_password" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "Did not receive identification string from %{IP:src_ip}" }
-        add_tag => [ "ssh_no_id" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "User %{USERNAME:username} from %{IP:src_ip} not allowed because not listed in AllowUsers" }
-        add_tag => [ "ssh_user_not_allowed" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "authentication failure; logname=%{USERNAME:logname} uid=%{BASE10NUM:uid} euid=%{BASE10NUM:euid} tty=%{TTY:tty} ruser=%{USERNAME:ruser} rhost=(?:%{HOSTNAME:remote_host}|\s*) user=%{USERNAME:user}"}
-        add_tag => [ "ssh_auth_failure" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "pam_unix\(sshd:auth\): authentication failure; logname= uid=0 euid=0 tty=%{NOTSPACE:tty} ruser= rhost=(?:%{HOSTNAME:remote_host}|\s*)  user=%{USERNAME:user}"}
-        add_tag => [ "ssh_auth_failure" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "Failed password for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"}
-        add_tag => [ "ssh_failed_password" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "Accepted password for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"}
-        add_tag => [ "ssh_accepted_password" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "Accepted publickey for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"}
-        add_tag => [ "ssh_accepted_pubkey" ]
-        tag_on_failure => []
-      }
-      grok {
-        match => { "message" => "Accepted keyboard-interactive/pam for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2"}
-        add_tag => [ "ssh_accepted_interactive" ]
-        tag_on_failure => []
-      }
-  }
-
 # NGINX
   if [type] == "NGINX" {
     date {
@@ -390,7 +310,7 @@ filter {
     }
   }
 
-# Tanner 
+# Tanner
   if [type] == "Tanner" {
     date {
       match => [ "timestamp", "ISO8601" ]
@@ -406,7 +326,7 @@ filter {
     }
   }
 
-# Vnclowpot 
+# Vnclowpot
   if [type] == "Vnclowpot" {
     grok {
       match => [ "message", "\A%{NOTSPACE}%{SPACE}%{TIME}%{SPACE}%{IPV4:src_ip}:%{INT:src_port}%{SPACE}%{NOTSPACE:vnc_handshake}" ]
@@ -431,19 +351,19 @@ if "_grokparsefailure" in [tags] { drop {} }
       cache_size => 10000
       source => "src_ip"
       database => "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb"
-    } 
+    }
     geoip {
       cache_size => 10000
       source => "src_ip"
       database => "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-ASN.mmdb"
-    } 
+    }
     translate {
       refresh_interval => 86400
       field => "src_ip"
       destination => "ip_rep"
       dictionary_path => "/etc/listbot/iprep.yaml"
     }
-  } 
+  }
 
 # In some rare conditions dest_port, src_port is indexed as string, forcing integer for now
   if [dest_port] {

docker/elk/logstash/docker-compose.yml

@@ -15,5 +15,4 @@ services:
     image: "dtagdevsec/logstash:1804"
     volumes:
      - /data:/data
-     - /var/log:/data/host/log
      - /root/tpotce/docker/elk/logstash/dist/logstash.conf:/etc/logstash/conf.d/logstash.conf

etc/compose/collector.yml

@@ -143,7 +143,6 @@ services:
     image: "dtagdevsec/logstash:1804"
     volumes:
      - /data:/data
-     - /var/log:/data/host/log
 
 ## Elasticsearch-head service
   head:
@@ -203,4 +202,3 @@ services:
     image: "dtagdevsec/spiderfoot:1804"
     volumes:
      - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db
-

etc/compose/experimental.yml

@@ -477,7 +477,6 @@ services:
     image: "dtagdevsec/logstash:1804"
     volumes:
      - /data:/data
-     - /var/log:/data/host/log
 
 ## Elasticsearch-head service
   head:
@@ -537,4 +536,3 @@ services:
     image: "dtagdevsec/spiderfoot:1804"
     volumes:
      - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db
-

etc/compose/industrial.yml

@@ -296,7 +296,6 @@ services:
     image: "dtagdevsec/logstash:1804"
     volumes:
      - /data:/data
-     - /var/log:/data/host/log
 
 ## Elasticsearch-head service
   head:

etc/compose/legacy.yml

@@ -262,7 +262,6 @@ services:
     image: "dtagdevsec/logstash:1804"
     volumes:
      - /data:/data
-     - /var/log:/data/host/log
 
 ## Elasticsearch-head service
   head:
@@ -322,4 +321,3 @@ services:
     image: "dtagdevsec/spiderfoot:1804"
     volumes:
      - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db
-

etc/compose/standard.yml

@@ -476,7 +476,6 @@ services:
     image: "dtagdevsec/logstash:1804"
     volumes:
      - /data:/data
-     - /var/log:/data/host/log
 
 ## Elasticsearch-head service
   head:
@@ -536,4 +535,3 @@ services:
     image: "dtagdevsec/spiderfoot:1804"
     volumes:
      - /data/spiderfoot/spiderfoot.db:/home/spiderfoot/spiderfoot.db
-