Soekris 5501 panic'ing
-
Hoping someone can help me identify what is causing this system to panic. Seems to do it once or twice a day
panic: vm_fault: fault on nofault entry, addr: c404b000
cpuid = 0
KDB: enter: panic
[ thread pid 43182 tid 100085 ]
Stopped at kdb_enter+0x3d: movl $0,kdb_why
db:0:kdb.enter.default> textdump set
textdump set
db:0:kdb.enter.default> capture on
db:0:kdb.enter.default> run lockinfo
db:1:lockinfo> show locks
No such command
db:1:locks> show alllocks
No such command
db:1:alllocks> show lockedvnods
Locked vnodes
db:0:kdb.enter.default> show pcpu
cpuid = 0
dynamic pcpu = 0x5d9500
curthread = 0xc5ee5c40: pid 43182 "netstat"
curpcb = 0xebd23d60
fpcurthread = 0xc5ee5c40: pid 43182 "netstat"
idlethread = 0xc52e8000: tid 100003 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
db:0:kdb.enter.default> bt
Tracing pid 43182 tid 100085 td 0xc5ee5c40
kdb_enter(c143df33,c143df33,c14764a9,ebd23724,1,…) at kdb_enter+0x3d/frame 0xebd236d8
panic(c14764a9,c404b000,2,ebd2383c,ebd2382c,...) at panic+0x144/frame 0xebd23718
vm_fault_hold(c27e1000,c404b000,2,0,0,...) at vm_fault_hold+0x2074/frame 0xebd23878
vm_fault(c27e1000,c404b000,2,0,ebd23a54,...) at vm_fault+0x81/frame 0xebd238a0
trap_pfault(c404b004,560aab55,c5da592d,c5ee52e0,c5ee5c40,...) at trap_pfault+0x1bb/frame 0xebd23920
trap(ebd23a68) at trap+0x677/frame 0xebd23a5c
calltrap() at calltrap+0x6/frame 0xebd23a5c
--- trap 0xc, eip = 0xc12a9b00, esp = 0xebd23aa8, ebp = 0xebd23ab4 ---
free_pv_entry(1d889000,0,806b000,c25901ac,807c000,...) at free_pv_entry+0xb0/frame 0xebd23ab4
pmap_remove_pte(806b000,ebd23b00,c5eecb94,0,1000,...) at pmap_remove_pte+0xb5/frame 0xebd23ad0
pmap_remove(c27adbf4,806a000,807c000,0,c34a1eb8,...) at pmap_remove+0x1fd/frame 0xebd23b18
vm_map_delete(c27adb40,1000,bfc00000,0,c27adbf4,...) at vm_map_delete+0x27b/frame 0xebd23b78
vm_map_remove(c27adb40,1000,bfc00000,c27adbf4,c67818d0,...) at vm_map_remove+0xc2/frame 0xebd23ba0
vmspace_exit(c5ee5c40,c67818d0,1,0,0,...) at vmspace_exit+0xd7/frame 0xebd23bc8
exit1(c5ee5c40,0,ebd23cfc,c12b252b,c5ee5c40,...) at exit1+0x88d/frame 0xebd23c30
sys_sys_exit(c5ee5c40,ebd23cc8,c5ee5000,c1f03c00,3,...) at sys_sys_exit+0x1d/frame 0xebd23c40
syscall(ebd23d08) at syscall+0x48b/frame 0xebd23cfc
Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebd23cfc
--- syscall (1, FreeBSD ELF32, sys_sys_exit), eip = 0x28195aa3, esp = 0xbfbfe598, ebp = 0xbfbfe5a8 ---
db:0:kdb.enter.default> ps
pid ppid pgrp uid state wmesg wchan cmd
43221 43032 21 0 R awk
43182 43032 21 0 RE CPU 0 netstat
43032 35652 21 0 S wait 0xc5b5e8d0 sh
49290 1 49290 0 Ss+ ttyin 0xc52c4270 getty
49208 1 49208 0 Ss+ ttyin 0xc52c4670 getty
46835 46499 46499 0 S nanslp 0xc1ef79e8 minicron
46499 1 46499 0 Ss wait 0xc5ed4000 minicron
46376 45930 45930 0 S nanslp 0xc1ef79e8 minicron
45930 1 45930 0 Ss wait 0xc5ed35e0 minicron
45620 45299 45299 0 S nanslp 0xc1ef79e8 minicron
45299 1 45299 0 Ss wait 0xc5ed3000 minicron
44639 1 44639 0 Ss nanslp 0xc1ef79e8 cron
41803 1 41803 0 Ss select 0xc59edae4 syslogd
35652 1 21 0 S+ piperd 0xc5a99198 sh
32805 1 32805 0 Ss (threaded) ntpd
100075 S select 0xc549e9a4 ntpd
23830 1 23830 1002 Ss select 0xc59ec164 dhcpd
19188 1 19183 65534 S select 0xc59ebc24 dnsmasq
16245 1 16006 0 S kqread 0xc5a70780 lighttpd
10767 10538 10538 0 S piperd 0xc5a9a000 rrdtool
10538 1 10538 0 Ss select 0xc549d824 apinger
10046 1 10046 0 Ss select 0xc549d6a4 sshd
6917 1 6917 0 Ss select 0xc59ec624 inetd
6511 1 6511 0 Ss bpf 0xc5969400 filterlog
315 1 315 0 Ss select 0xc59ebda4 devd
301 299 299 0 S kqread 0xc5a6dc80 check_reload_status
299 1 299 0 Ss kqread 0xc5a6dd80 check_reload_status
285 282 282 0 S accept 0xc5b8f536 php-fpm
284 282 282 0 S accept 0xc5b8f536 php-fpm
283 282 282 0 S accept 0xc5b8f536 php-fpm
282 1 282 0 Ss kqread 0xc5a6f900 php-fpm
52 0 0 0 DL mdwait 0xc5ab1000 [md1]
47 0 0 0 DL mdwait 0xc584c800 [md0]
20 0 0 0 DL syncer 0xc1f16f44 [syncer]
19 0 0 0 DL vlruwt 0xc59f7000 [vnlru]
18 0 0 0 DL psleep 0xc1f16684 [bufdaemon]
17 0 0 0 DL pollid 0xc1ef63a8 [idlepoll]
9 0 0 0 DL pgzero 0xc20421a0 [pagezero]
8 0 0 0 DL psleep 0xc2041ec4 [vmdaemon]
7 0 0 0 DL psleep 0xc20a0484 [pagedaemon]
6 0 0 0 DL waiting_ 0xc209ad8c [sctp_iterator]
5 0 0 0 DL pftm 0xc0f794d0 [pf purge]
16 0 0 0 DL (threaded) [usb]
100035 D - 0xc5432d34 [usbus1]
100034 D - 0xc5432d04 [usbus1]
100033 D - 0xc5432cd4 [usbus1]
100032 D - 0xc5432ca4 [usbus1]
100031 D - 0xc5437b5c [usbus0]
100030 D - 0xc5437b2c [usbus0]
100029 D - 0xc5437afc [usbus0]
100028 D - 0xc5437acc [usbus0]
4 0 0 0 DL (threaded) [cam]
100043 D - 0xc1e4c9a8 [scanner]
100018 D - 0xc1e4cb00 [doneq0]
3 0 0 0 DL crypto_r 0xc2040df0 [crypto returns]
2 0 0 0 DL crypto_w 0xc2040d30 [crypto]
15 0 0 0 DL - 0xc1e66c38 [rand_harvestq]
14 0 0 0 DL (threaded) [geom]
100010 D - 0xc2096df4 [g_down]
100009 D - 0xc2096df0 [g_up]
100008 D - 0xc2096dec [g_event]
13 0 0 0 DL sleep 0xc1e23658 [ng_queue0]
12 0 0 0 WL (threaded) [intr]
100041 I [swi1: pfsync]
100039 I [swi1: pf send]
100037 I [swi0: uart uart]
100036 I [irq1: atkbd0]
100027 I [irq7: ohci0 ehci0]
100026 I [irq14: ata0]
100025 I [irq15: hifn0 ata1]
100023 I [swi5: fast taskq]
100021 I [swi6: Giant taskq]
100019 I [swi6: task queue]
100006 I [swi1: netisr 0]
100005 I [swi3: vm]
100004 I [swi4: clock]
11 0 0 0 RL [idle: cpu0]
1 0 1 0 SLs wait 0xc52e12f0 [init]
10 0 0 0 DL audit_wo 0xc209ed88 [audit]
0 0 0 0 DLs (threaded) [kernel]
100042 D - 0xc52e4800 [CAM taskq]
100024 D - 0xc52e4180 [kqueue taskq]
100022 D - 0xc52e4300 -
Is this a single system or part of an HA cluster with CARP?
What features are you using?
Which version/architecture of pfSense?At first glance I'm inclined to say it's hardware since it's crashing in what appears to be some memory management, but not allocation or I'd say it's run out of some resource. If you can capture more crash reports, attach them here (attaching as a .txt file would be better than pasting inline)
