More complete sylog-pri filter example

recipes/syslog-pri/syslog.conf

@@ -1,41 +1,100 @@
 input {
+  # Syslog server
   tcp {
     port => 5000
-    type => syslog
+    type => syslog_relay
   }
   udp {
     port => 5000
-    type => syslog
+    type => syslog_relay
   }
+
+# Default Syslog server port require root permissions due to port < 1024
+#  tcp {
+#    port => 514
+#    type => syslog_relay
+#  }
+#  udp {
+#    port => 514
+#    type => syslog_relay
+#  }
 }
 
 filter {
+  # strip the syslog PRI part and create facility and severity fields.
+  # the original syslog message is saved in field %{syslog_raw_message}.
+  # the extracted PRI is available in the %{syslog_pri} field.
+  #
+  # You get %{syslog_facility_code} and %{syslog_severity_code} fields.
+  # You also get %{syslog_facility} and %{syslog_severity} fields if the
+  # use_labels option is set True (the default) on syslog_pri filter.
   grok {
-      type => "syslog"
-      pattern => [ "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" ]
-      add_field => [ "received_at", "%{@timestamp}" ]
-      add_field => [ "received_from", "%{@source_host}" ]
+    type => "syslog_relay"
+    pattern => [ "<%{POSINT:syslog_pri}>%{SPACE}%{GREEDYDATA:message_remainder}" ]
+    add_tag => "got_syslog_pri"
+    add_field => [ "syslog_raw_message", "%{@message}" ]
   }
   syslog_pri {
-      type => "syslog"
+    type => "syslog_relay"
+    tags => [ "got_syslog_pri" ]
   }
-  date {
-      type => "syslog"
-      syslog_timestamp => [ "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+  mutate {
+    type => "syslog_relay"
+    tags => [ "got_syslog_pri" ]
+    replace => [ "@message", "%{message_remainder}" ]
+    remove => [ "message_remainder" ]
+    remove_tag => "got_syslog_pri"
+  }
+
+  # strip the syslog timestamp and force event timestamp to be the same.
+  # the original string is saved in field %{syslog_timestamp}.
+  # the original logstash input timestamp is saved in field %{received_at}.
+  grok {
+    type => "syslog_relay"
+    pattern => [ "%{SYSLOGTIMESTAMP:syslog_timestamp}%{SPACE}%{GREEDYDATA:message_remainder}" ]
+    add_tag => "got_syslog_timestamp"
+    add_field => [ "received_at", "%{@timestamp}" ]
   }
   mutate {
-      type => "syslog"
-      exclude_tags => "_grokparsefailure"
-      replace => [ "@source_host", "%{syslog_hostname}" ]
-      replace => [ "@message", "%{syslog_message}" ]
+    type => "syslog_relay"
+    tags => [ "got_syslog_timestamp" ]
+    replace => [ "@message", "%{message_remainder}" ]
+    remove => [ "message_remainder" ]
+    remove_tag => "got_syslog_timestamp"
+  }
+  date {
+    type => "syslog_relay"
+    tags => [ "got_syslog_timestamp" ]
+    # season to taste for your own syslog format(s)
+    syslog_timestamp => [ "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
+  }
+
+  # strip the host field from the syslog line.
+  # the extracted host field becomes the logstash %{@source_host} metadata
+  # and is also available in the filed %{syslog_hostname}.
+  # the original logstash source_host is saved in field %{logstash_source}.
+  grok {
+    type => "syslog_relay"
+    pattern => [ "%{SYSLOGHOST:syslog_hostname}%{SPACE}%{GREEDYDATA:message_remainder}" ]
+    add_tag => "got_syslog_host"
+    add_field => [ "logstash_source", "%{@source_host}" ]
   }
   mutate {
-      type => "syslog"
-      remove => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
+    type => "syslog_relay"
+    tags => [ "got_syslog_host" ]
+    replace => [ "@source_host", "%{syslog_hostname}" ]
+    replace => [ "@message", "%{message_remainder}" ]
+    remove => [ "message_remainder" ]
+    remove_tag => "got_syslog_host"
   }
 }
 
 output {
-  # Example just to output to elasticsearch
+  # If your elasticsearch server is discoverable with multicast, use this:
   elasticsearch { }
+
+  # If you can't discover using multicast, set the address explicitly
+  #elasticsearch {
+  #  host => "myelasticsearchserver"
+  #}
 }