Crash

[TommyTheKid]

We have had one of our two HA firewalls that keeps crashing. I dont know if its hardware related or if its because of some package. Are there any clues in this output?

Crash report begins. Anonymous machine information:

amd64
8.1-RELEASE-p13
FreeBSD 8.1-RELEASE-p13 #1: Fri Dec 7 23:07:32 EST 2012 root@snapshots-8_1-amd64.builders.pfsense.org:/usr/obj./usr/pfSensesrc/src/sys/pfSense_SMP.8

Crash report details:

Filename: /var/crash/bounds
2

Filename: /var/crash/info.0
Dump header from device /dev/ad4s1b
Architecture: amd64
Architecture Version: 1
Dump Length: 73216B (0 MB)
Blocksize: 512
Dumptime: Tue Aug 27 11:33:30 2013
Hostname: twfw-1.transwest.com
Magic: FreeBSD Text Dump
Version String: FreeBSD 8.1-RELEASE-p13 #1: Fri Dec 7 23:07:32 EST 2012
root@snapshots-8_1-amd64.builders.pfsense.org:/usr/obj./usr/pfSensesrc/src/sys/pfSense_SMP.8
Panic String:
Dump Parity: 2441754477
Bounds: 0
Dump Status: good

Filename: /var/crash/info.1
Dump header from device /dev/ad4s1b
Architecture: amd64
Architecture Version: 1
Dump Length: 89600B (0 MB)
Blocksize: 512
Dumptime: Tue Sep 3 13:45:16 2013
Hostname: twfw-1.transwest.com
Magic: FreeBSD Text Dump
Version String: FreeBSD 8.1-RELEASE-p13 #1: Fri Dec 7 23:07:32 EST 2012
root@snapshots-8_1-amd64.builders.pfsense.org:/usr/obj./usr/pfSensesrc/src/sys/pfSense_SMP.8
Panic String:
Dump Parity: 924061037
Bounds: 1
Dump Status: good

Filename: /var/crash/textdump.tar.0
ddb.txt06000014000012207161352 7071 ustarrootwheeldb:0:kdb.enter.default> run lockinfo
db:1:lockinfo> show locks
No such command
db:1:locks> show alllocks
No such command
db:1:alllocks> show lockedvnods
Locked vnodes
db:0:kdb.enter.default> show pcpu
cpuid = 2
dynamic pcpu = 0xffffff807ef05080
curthread = 0xffffff000254f7c0: pid 0 "em1 que"
curpcb = 0xffffff8000140d40
fpcurthread = none
idlethread = 0xffffff00022ea3e0: pid 11 "idle: cpu2"
curpmap = 0
tssp = 0xffffffff811dde50
commontssp = 0xffffffff811dde50
rsp0 = 0xffffff8000140d40
gs32p = 0xffffffff811dcc88
ldt = 0xffffffff811dccc8
tss = 0xffffffff811dccb8
db:0:kdb.enter.default> bt
Tracing pid 0 tid 64052 td 0xffffff000254f7c0
rn_match() at rn_match+0x25
pfr_match_addr() at pfr_match_addr+0xcb
pf_test_tcp() at pf_test_tcp+0xaa1
pf_test() at pf_test+0x20fe
pf_check_in() at pf_check_in+0x39
pfil_run_hooks() at pfil_run_hooks+0xa2
ip_input() at ip_input+0x34e
netisr_dispatch_src() at netisr_dispatch_src+0x7b
ether_demux() at ether_demux+0x169
ether_input() at ether_input+0x174
em_rxeof() at em_rxeof+0x1d0
em_handle_que() at em_handle_que+0x4d
taskqueue_run() at taskqueue_run+0x93
taskqueue_thread_loop() at taskqueue_thread_loop+0x46
fork_exit() at fork_exit+0x118
fork_trampoline() at fork_trampoline+0xe
–- trap 0, rip = 0, rsp = 0xffffff8000140d30, rbp = 0 ---
db:0:kdb.enter.default> ps
pid ppid pgrp uid state wmesg wchan cmd
22685 21829 24 0 S piperd 0xffffff0002aeb888 awk
22643 21829 24 0 S piperd 0xffffff0031d6bb60 wc
22638 21829 24 0 S piperd 0xffffff0002b26888 awk
22511 21829 24 0 S select 0xffffff001447ecc0 nc
22289 21999 24 0 S nanslp 0xffffffff8115cc28 sleep
21999 21829 24 0 S wait 0xffffff00319ba460 sh
21829 49943 24 0 S wait 0xffffff00148c0460 sh
4808 17301 17274 0 S accept 0xffffff0031ec5066 initial thread
23787 17301 17274 0 S accept 0xffffff001419630e initial thread
21900 1 21900 0 Ss select 0xffffff0014160dc0 racoon
59471 61306 61306 0 S piperd 0xffffff001433b888 rrdtool
5253 1705 5253 0 S+ ttyin 0xffffff00028168a8 sh
5233 1987 5233 0 S+ ttyin 0xffffff00025a88a8 sh
1987 1420 1987 0 S+ wait 0xffffff00029b88c0 sh
1705 1356 1705 0 S+ wait 0xffffff0002baa460 sh
1568 53664 1568 0 Ss (threaded) sshlockout_pf
64213 S nanslp 0xffffffff8115cc28 sshlockout_pf
64108 S piperd 0xffffff0002aeb5b0 initial thread
1420 1 1420 0 Ss+ wait 0xffffff00319c4000 login
1356 1 1356 0 Ss+ wait 0xffffff00319b98c0 login
60689 60207 60207 0 S nanslp 0xffffffff8115cc28 minicron
60207 1 60207 0 Ss wait 0xffffff001424b460 minicron
59981 59633 59633 0 S nanslp 0xffffffff8115cc28 minicron
59633 1 59633 0 Ss wait 0xffffff0002aac460 minicron
59430 59116 59116 0 S nanslp 0xffffffff8115cc28 minicron
59116 1 59116 0 Ss wait 0xffffff00319c58c0 minicron
11428 1 11428 0 Ss select 0xffffff0031849640 ntpd
55011 1 55011 0 Ss nanslp 0xffffffff8115cc28 cron
49943 1 24 0 S+ piperd 0xffffff0002b242d8 sh
30909 1 30722 65534 S select 0xffffff00319d89c0 dnsmasq
27545 1 27545 1002 Ss select 0xffffff00028f7bc0 dhcpd
17301 1 17274 0 S kqread 0xffffff000294d600 lighttpd
61306 1 61306 0 Ss select 0xffffff0031849a40 apinger
55048 1 55048 0 Ss select 0xffffff0002972240 inetd
54426 1 54426 0 Ss select 0xffffff0002a899c0 openvpn
53986 1 24 0 S+ piperd 0xffffff0002966b60 logger
53846 1 24 0 S+ bpf 0xffffff00319be000 tcpdump
53664 1 53664 0 Ss select 0xffffff00319d8040 syslogd
7603 1 7603 0 Ss select 0xffffff00028f7040 sshd
260 1 260 0 Ss select 0xffffff00028aeb40 devd
250 248 248 0 S kqread 0xffffff0002905600 check_reload_status
248 1 248 0 Ss kqread 0xffffff00028b7100 check_reload_status
39 0 0 0 SL mdwait 0xffffff0002a46800 [md0]
23 0 0 0 SL sdflush 0xffffffff811a3738 [softdepflush]
22 0 0 0 SL vlruwt 0xffffff00028b8000 [vnlru]
21 0 0 0 SL syncer 0xffffffff81180e40 [syncer]
20 0 0 0 SL psleep 0xffffffff81180968 [bufdaemon]
19 0 0 0 SL pollid 0xffffffff8115ba48 [idlepoll]
18 0 0 0 SL pgzero 0xffffffff811a51cc [pagezero]
17 0 0 0 SL psleep 0xffffffff811a4568 [vmdaemon]
16 0 0 0 SL psleep 0xffffffff811a452c [pagedaemon]
9 0 0 0 SL ccb_scan 0xffffffff811229e0 [xpt_thrd]
8 0 0 0 SL pftm 0xffffffff80204dd0 [pfpurge]
7 0 0 0 SL waiting_ 0xffffffff8118ce00 [sctp_iterator]
15 0 0 0 SL (threaded) usb
64079 D - 0xffffff8000484dd0 [usbus6]
64078 D - 0xffffff8000484d78 [usbus6]
64077 D - 0xffffff8000484d20 [usbus6]
64076 D - 0xffffff8000484cc8 [usbus6]
64075 D - 0xffffff800047bef0 [usbus5]
64074 D - 0xffffff800047be98 [usbus5]
64073 D - 0xffffff800047be40 [usbus5]
64072 D - 0xffffff800047bde8 [usbus5]
64071 D - 0xffffff8000472ef0 [usbus4]
64070 D - 0xffffff8000472e98 [usbus4]
64069 D - 0xffffff8000472e40 [usbus4]
64068 D - 0xffffff8000472de8 [usbus4]
64066 D - 0xffffff8000469ef0 [usbus3]
64065 D - 0xffffff8000469e98 [usbus3]
64064 D - 0xffffff8000469e40 [usbus3]
64063 D - 0xffffff8000469de8 [usbus3]
64049 D - 0xffffff80003c4dd0 [usbus2]
64048 D - 0xffffff80003c4d78 [usbus2]
64047 D - 0xffffff80003c4d20 [usbus2]
64046 D - 0xffffff80003c4cc8 [usbus2]
64044 D - 0xffffff80003bbef0 [usbus1]
64043 D - 0xffffff80003bbe98 [usbus1]
64042 D - 0xffffff80003bbe40 [usbus1]
64041 D - 0xffffff80003bbde8 [usbus1]
64039 D - 0xffffff80003b2ef0 [usbus0]
64038 D - 0xffffff80003b2e98 [usbus0]
64037 D - 0xffffff80003b2e40 [usbus0]
64036 D - 0xffffff80003b2de8 [usbus0]
14 0 0 0 SL - 0xffffffff8115c904 [yarrow]
6 0 0 0 SL crypto_r 0xffffffff811a2660 [crypto returns]
5 0 0 0 SL crypto_w 0xffffffff811a2620 [crypto]
4 0 0 0 SL - 0xffffffff811586e8 [g_down]
3 0 0 0 SL - 0xffffffff811586e0 [g_up]
2 0 0 0 SL - 0xffffffff811586d0 [g_event]
13 0 0 0 SL (threaded) ng_queue
64017 D sleep 0xffffffff810d19d0 [ng_queue3]
64016 D sleep 0xffffffff810d19d0 [ng_queue2]
64015 D sleep 0xffffffff810d19d0 [ng_queue1]
64014 D sleep 0xffffffff810d19d0 [ng_queue0]
12 0 0 0 RL (threaded) intr
64096 I [swi1: netisr 1]
64095 I [swi1: netisr 2]
64094 I [swi1: netisr 3]
64082 I [irq1: atkbd0]
64081 I [swi0: uart uart]
64080 I [irq14: ata0]
64067 I [irq19: uhci3]
64062 I [irq23: uhci2 ehci1]
64045 I [irq18: ehci0 uhci4+]
64040 I [irq21: uhci1]
64035 I [irq16: uhci0]
64034 I [irq9: acpi0]
64032 I [swi5: +]
64030 I [swi2: cambio]
64026 I [swi6: task queue]
64025 I [swi6: Giant taskq]
64013 I [swi3: vm]
64012 RunQ [swi4: clock]
64011 I [swi4: clock]
64010 RunQ [swi4: clock]
64009 RunQ [swi4: clock]
64008 I [swi1: netisr 0]
11 0 0 0 RL (threaded) idle
64007 CanRun [idle: cpu0]
64006 CanRun [idle: cpu1]
64005 CanRun [idle: cpu2]
64004 CanRun [idle: cpu3]
1 0 1 0 SLs wait 0xffffff00022e78c0 [init]
10 0 0 0 SL audit_wo 0xffffffff811a2b90 [audit]
0 0 0 0 RLs (threaded) kernel
64061 D - 0xffffff000259d180 [em5 txq]
64060 D - 0xffffff000259d200 [em5 que]
64059 D - 0xffffff0002596300 [em4 txq]
64058 D - 0xffffff000257f580 [em4 que]
64057 D - 0xffffff000257e980 [em3 txq]
64056 Run CPU 1 [em3 que]
64055 D - 0xffffff000256d380 [em2 txq]
64054 Run CPU 3 [em2 que]
64053 D - 0xffffff0002560a80 [em1 txq]
64052 Run CPU 2 [em1 que]
64051 D - 0xffffff0002555080 [em0 txq]
64050 Run CPU 0 [em0 que]
64033 D - 0xffffff000242f880