Filterlog field extractions for log analytics products

Azzir

Hi All,

I’ve just finished onboarding firewall logs into Sumo Logic and have created the following (huge) parsing statement that extracts out the fields according to the Pfsense docs.

| parse regex  "\w+\s+\d+\s+\d{2}:\d{2}:\d{2} (?<log_type>[^:[]+)" // There *MAY* be a host in here according to the docs
| parse regex "filterlog: (?<rule>\d+),(?<subrule>\d+),(?<anchor>[^,]*),(?<tracker>\d+),(?<interface>\w+),(?<reason>\w+),(?<action>\w+),(?<direction>\w+),(?<ip_specific_data>.*)" nodrop
| parse regex field=ip_specific_data "^4,(?<tos>[^,]*),(?<ecn>[^,]*),(?<ttl>\d+),(?<id>\d+),(?<offset>\d+),(?<flags>\w+),(?<protocol_id>\d+),(?<protocol_text>[^,]+)" nodrop
| parse regex field=ip_specific_data "^6,(?<class>\w+),(?<flow_label>[^,]*),(?<hop_limit>\d+),(?<protocol_text>[^,]+),(?<protocol_id>\d+)" nodrop
| parse regex "tcp,(?:\d+,)?(?<length>\d+),(?<source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<source_port>\d+),(?<destination_port>\d+),(?<data_length>\d+),(?<tcp_flags>\w+),(?<sequence_number>[\d:]*),(?<ack_number>\d*),(?<tcp_window>\d*),(?<urg>[^,]*),(?<tcp_options>.*)" nodrop 
| parse regex "udp,(?:\d+,)?(?<length>\d+),(?<source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<source_port>\d+),(?<destination_port>\d+),(?<data_length>\d+)" nodrop 
| parse regex "icmp,(?:\d+,)?(?<length>\d+),(?<source_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<destination_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<icmp_data>.*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>request|reply),(?<echo_id>\d+),(?<echo_sequence>\d+)" nodrop
| parse regex field=icmp_data "(?<icmp_type>unreach|timexceed|paramprob|redirect|maskreply),(?<icmp_description>.*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>unreachproto),(?<icmp_destination_ip_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachable_protocol_id>.*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>unreachport),(?<icmp_destination_ip_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachable_protocol_id>[^,]+),(?<unreachable_port_number>\d+)" nodrop
| parse regex field=icmp_data "(?<icmp_type>needfrag),(?<icmp_destination_ip_address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<icmp_mtu>\d+)" nodrop
| parse regex field=icmp_data "(?<icmp_type>tstamp),(?<icmp_id>[^,]*),(?<icmp_sequence>[^,]*)" nodrop
| parse regex field=icmp_data "(?<icmp_type>tstampreply),(?<icmp_id>[^,]*),(?<icmp_sequence>[^,]*),(?<icmp_otime>\d*),(?<icmp_rtime>\d*),(?<icmp_ttime>\d*)" nodrop
| fields - ip_specific_data, icmp_data</icmp_ttime></icmp_rtime></icmp_otime></icmp_sequence></icmp_id></icmp_type></icmp_sequence></icmp_id></icmp_type></icmp_mtu></icmp_destination_ip_address></icmp_type></unreachable_port_number></unreachable_protocol_id></icmp_destination_ip_address></icmp_type></unreachable_protocol_id></icmp_destination_ip_address></icmp_type></icmp_description></icmp_type></echo_sequence></echo_id></icmp_type></icmp_data></destination_address></source_address></length></data_length></destination_port></source_port></destination_address></source_address></length></tcp_options></urg></tcp_window></ack_number></sequence_number></tcp_flags></data_length></destination_port></source_port></destination_address></source_address></length></protocol_id></protocol_text></hop_limit></flow_label></class></protocol_text></protocol_id></flags></offset></id></ttl></ecn></tos></ip_specific_data></direction></action></reason></interface></tracker></anchor></subrule></rule></log_type>

These are specific to Sumo Logic, but should be easy enough to convert into something usable for Elastic or Splunk if you know your Regular Expressions.

I hope this is useful to someone

pranav

Thank you @azzir
This was very helpful. I was trying to compile similar query for Splunk.
After spending some time, I could come up with following.

host="pfSense.HOME.COM" filterlog

| rex "(?P<Month>\w+)\s\s(?<Day>\d{1,2})\s(?<Hour>\d{1,2}):(?<Minutes>\d{1,2}):(?<Seconds>\d{1,2})\s(?<RouterName>[^\.]+)\.(?<Suffix>[\S]+)\s\w+\s\s\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}\s(?P<LogType>\w+):\s(?<RuleNumber>\d+),,,(?<Tracker>\d+),(?P<RealInterface>\w+),(?P<ReasonForLogEntry>\w+),(?P<Action>\w+),(?P<Direction>\w+),(?P<IPVersion>\w+),(?<tos>[^,]*),(?<ecn>[^,]*),(?<ttl>\d+),(?<id>\d+),(?<offset>\d+),(?<flags>\w+),(?<ProtocolId>\d+),(?<Protocol>[^,]+)"

| rex "^6,(?<class>\w+),(?<flowLabel>[^,]*),(?<hopLimit>\d+),(?<protocolText>[^,]+),(?<protocolId>\d+)"

| rex "tcp,(?:\d+,)?(?<Length>\d+),(?<SourceAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<DestinationAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<SourcePort>\d+),(?<DestinationPort>\d+),(?<DataLength>\d+),(?<TCPFlags>\w+),(?<SequenceNumber>[\d:]*),(?<AckNumber>\d*),(?<TCPWindow>\d*),(?<urg>[^,]*),(?<TCPOptions>.*)"

| rex "udp,(?:\d+,)?(?<Length>\d+),(?<SourceAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<DestinationAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<SourcePort>\d+),(?<DestinationPort>\d+),(?<DataLength>\d+)"

| rex "icmp,(?:\d+,)?(?<length>\d+),(?<SourceAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<DestinationAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<ICMPData>.*)"

| rex "(?<icmpType>request|reply),(?<EchoId>\d+),(?<EchoSequence>\d+)"

| rex "(?<icmpType>unreach|timexceed|paramprob|redirect|maskreply),(?<icmpDescription>.*)"

| rex "(?<icmpType>unreachproto),(?<icmpDestinationIpAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachableProtocolId>.*)"

| rex "(?<icmpType>unreachport),(?<icmpDestinationIpAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<unreachableProtocolId>[^,]+),(?<unreachablePortNumber>\d+)"

| rex "(?<icmpType>needfrag),(?<icmpDestinationIpAddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}),(?<icmpMTU>\d+)"

| rex "(?<icmpType>tstamp),(?<icmp_id>[^,]*),(?<icmpSequence>[^,]*)"

| rex "(?<icmpType>tstampreply),(?<icmpId>[^,]*),(?<icmpSequence>[^,]*),(?<icmpOTime>\d*),(?<icmpRTime>\d*),(?<icmpTtime>\d*)"

| table Month,Day,Hour,Minutes,Seconds,RouterName,Suffix,LogType,RuleNumber,Tracker,RealInterface,ReasonForLogEntry,Action,Direction,IPVersion,tos,ecn,ttl,id,offset,flags,ProtocolId,Protocol,class,flowLabel,hopLimit,protocolText,protocolId,Length,SourceAddress,DestinationAddress,SourcePort,DestinationPort,DataLength,DataLength,TCPFlags,SequenceNumber,AckNumber,TCPWindow,urg,TCPOptions,ICMPData,icmpType,EchoId,EchoSequence,icmpDescription,icmpDestinationIpAddress,unreachableProtocolId,unreachablePortNumber,icmpMTU,icmpId,icmpSequence,icmpOTime,icmpRTime,icmpTtime
code

Tools used:
To validate regex aginst data: https://regex101.com/
Official Documentation About Log: https://doc.pfsense.org/index.php/Filter_Log_Format_for_pfSense_2.2