Filterlog field extractions for log analytics products

[Azzir]

Hi All,

I've just finished onboarding firewall logs into Sumo Logic and have created the following (huge) parsing statement that extracts out the fields according to the Pfsense docs.

| parse regex  "\w+\s+\d+\s+\d{2}:\d{2}:\d{2} (?<log_type>[^:[]+)" // There *MAY* be a host in here according to the docs
| parse regex "filterlog: (?<rule>\d+),(?<subrule>\d+),(?<anchor>[^,]*),(?<tracker>\d+),(?<interface>\w+),(?<reason>\w+),(?<action>\w+),(?<direction>\w+),(?<ip_specific_data>.*)" nodrop
| parse regex field=ip_specific_data "^4,(?<tos>[^,]*),(?<ecn>[^,]*),(?<ttl>\d+),(?<id>\d+),(?<offset>\d+),(?<flags>\w+),(?<protocol_id>\d+),(?<protocol_text>[^,]+)" nodrop
| parse regex field=ip_specific_data "^6,(?<class>\w+),(?<flow_label>[^,]*),(?<hop_limit>\d+),(?<protocol_text>[^,]+),(?<protocol_id>\d+)" nodrop
| parse regex "tcp,(?:\d+,)?(?<length>\d+),(?<source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<source_port>\d+),(?<destination_port>\d+),(?<data_length>\d+),(?<tcp_flags>\w+),(?<sequence_number>[\d:]*),(?<ack_number>\d*),(?<tcp_window>\d*),(?<urg>[^,]*),(?<tcp_options>.*)" nodrop 
| parse regex "udp,(?:\d+,)?(?<length>\d+),(?<source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<source_port>\d+),(?<destination_port>\d+),(?<data_length>\d+)" nodrop 
| parse regex "icmp,(?:\d+,)?(?<length>\d+),(?<source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<icmp_data>.*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>request|reply),(?<echo_id>\d+),(?<echo_sequence>\d+)" nodrop
| parse regex field=icmp_data "(?<icmp_type>unreach|timexceed|paramprob|redirect|maskreply),(?<icmp_description>.*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>unreachproto),(?<icmp_destination_ip_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachable_protocol_id>.*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>unreachport),(?<icmp_destination_ip_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachable_protocol_id>[^,]+),(?<unreachable_port_number>\d+)" nodrop
| parse regex field=icmp_data "(?<icmp_type>needfrag),(?<icmp_destination_ip_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<icmp_mtu>\d+)" nodrop
| parse regex field=icmp_data "(?<icmp_type>tstamp),(?<icmp_id>[^,]*),(?<icmp_sequence>[^,]*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>tstampreply),(?<icmp_id>[^,]*),(?<icmp_sequence>[^,]*),(?<icmp_otime>\d*),(?<icmp_rtime>\d*),(?<icmp_ttime>\d*)" nodrop
| fields - ip_specific_data, icmp_data</icmp_ttime></icmp_rtime></icmp_otime></icmp_sequence></icmp_id></icmp_type></icmp_sequence></icmp_id></icmp_type></icmp_mtu></icmp_destination_ip_address></icmp_type></unreachable_port_number></unreachable_protocol_id></icmp_destination_ip_address></icmp_type></unreachable_protocol_id></icmp_destination_ip_address></icmp_type></icmp_description></icmp_type></echo_sequence></echo_id></icmp_type></icmp_data></destination_address></source_address></length></data_length></destination_port></source_port></destination_address></source_address></length></tcp_options></urg></tcp_window></ack_number></sequence_number></tcp_flags></data_length></destination_port></source_port></destination_address></source_address></length></protocol_id></protocol_text></hop_limit></flow_label></class></protocol_text></protocol_id></flags></offset></id></ttl></ecn></tos></ip_specific_data></direction></action></reason></interface></tracker></anchor></subrule></rule></log_type> 

These are specific to Sumo Logic, but should be easy enough to convert into something usable for Elastic or Splunk if you know your Regular Expressions.

I hope this is useful to someone :-)

[pranav]

Thank you @azzir
This was very helpful. I was trying to compile similar query for Splunk.
After spending some time, I could come up with following.

host="pfSense.HOME.COM" filterlog

| rex "(?P<Month>\w+)\s\s(?<Day>\d{1,2})\s(?<Hour>\d{1,2}):(?<Minutes>\d{1,2}):(?<Seconds>\d{1,2})\s(?<RouterName>[^\.]+)\.(?<Suffix>[\S]+)\s\w+\s\s\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}\s(?P<LogType>\w+):\s(?<RuleNumber>\d+),,,(?<Tracker>\d+),(?P<RealInterface>\w+),(?P<ReasonForLogEntry>\w+),(?P<Action>\w+),(?P<Direction>\w+),(?P<IPVersion>\w+),(?<tos>[^,]*),(?<ecn>[^,]*),(?<ttl>\d+),(?<id>\d+),(?<offset>\d+),(?<flags>\w+),(?<ProtocolId>\d+),(?<Protocol>[^,]+)"

| rex "^6,(?<class>\w+),(?<flowLabel>[^,]*),(?<hopLimit>\d+),(?<protocolText>[^,]+),(?<protocolId>\d+)"

| rex "tcp,(?:\d+,)?(?<Length>\d+),(?<SourceAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<DestinationAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<SourcePort>\d+),(?<DestinationPort>\d+),(?<DataLength>\d+),(?<TCPFlags>\w+),(?<SequenceNumber>[\d:]*),(?<AckNumber>\d*),(?<TCPWindow>\d*),(?<urg>[^,]*),(?<TCPOptions>.*)"

| rex "udp,(?:\d+,)?(?<Length>\d+),(?<SourceAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<DestinationAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<SourcePort>\d+),(?<DestinationPort>\d+),(?<DataLength>\d+)"

| rex "icmp,(?:\d+,)?(?<length>\d+),(?<SourceAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<DestinationAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<ICMPData>.*)"

| rex "(?<icmpType>request|reply),(?<EchoId>\d+),(?<EchoSequence>\d+)"

| rex "(?<icmpType>unreach|timexceed|paramprob|redirect|maskreply),(?<icmpDescription>.*)"

| rex "(?<icmpType>unreachproto),(?<icmpDestinationIpAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachableProtocolId>.*)"

| rex "(?<icmpType>unreachport),(?<icmpDestinationIpAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachableProtocolId>[^,]+),(?<unreachablePortNumber>\d+)"

| rex "(?<icmpType>needfrag),(?<icmpDestinationIpAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<icmpMTU>\d+)"

| rex "(?<icmpType>tstamp),(?<icmp_id>[^,]*),(?<icmpSequence>[^,]*)"

| rex "(?<icmpType>tstampreply),(?<icmpId>[^,]*),(?<icmpSequence>[^,]*),(?<icmpOTime>\d*),(?<icmpRTime>\d*),(?<icmpTtime>\d*)"

| table Month,Day,Hour,Minutes,Seconds,RouterName,Suffix,LogType,RuleNumber,Tracker,RealInterface,ReasonForLogEntry,Action,Direction,IPVersion,tos,ecn,ttl,id,offset,flags,ProtocolId,Protocol,class,flowLabel,hopLimit,protocolText,protocolId,Length,SourceAddress,DestinationAddress,SourcePort,DestinationPort,DataLength,DataLength,TCPFlags,SequenceNumber,AckNumber,TCPWindow,urg,TCPOptions,ICMPData,icmpType,EchoId,EchoSequence,icmpDescription,icmpDestinationIpAddress,unreachableProtocolId,unreachablePortNumber,icmpMTU,icmpId,icmpSequence,icmpOTime,icmpRTime,icmpTtime
code

Tools used:
To validate regex aginst data: https://regex101.com/
Official Documentation About Log: https://doc.pfsense.org/index.php/Filter_Log_Format_for_pfSense_2.2