Pfsense crashing after upgrade to 2.3.1
-
Need a llittle help. I heard my pfsense box restart earlier, was curious what caused it, logged in and saw a crash report:
Crash report begins. Anonymous machine information: i386 10.3-RELEASE-p3 FreeBSD 10.3-RELEASE-p3 #1 3ef16fb(RELENG_2_3_1): Tue May 17 19:34:28 CDT 2016 root@ce23-i386-builder:/builder/pfsense-231/tmp/obj/builder/pfsense-231/tmp/FreeBSD-src/sys/pfSense Crash report details: Filename: /var/crash/bounds 1 Filename: /var/crash/info.0 Dump header from device /dev/label/swap0 Architecture: i386 Architecture Version: 1 Dump Length: 68608B (0 MB) Blocksize: 512 Dumptime: Tue May 24 15:26:23 2016 Hostname: pfSense.Synergy Magic: FreeBSD Text Dump Version String: FreeBSD 10.3-RELEASE-p3 #1 3ef16fb(RELENG_2_3_1): Tue May 17 19:34:28 CDT 2016 root@ce23-i386-builder:/builder/pfsense-231/tmp/obj/builder/pfsense-231/tmp/FreeBSD-src/sys/pfSense Panic String: Dump Parity: 2296393849 Bounds: 0 Dump Status: good Filename: /var/crash/info.last Dump header from device /dev/label/swap0 Architecture: i386 Architecture Version: 1 Dump Length: 68608B (0 MB) Blocksize: 512 Dumptime: Tue May 24 15:26:23 2016 Hostname: pfSense.Synergy Magic: FreeBSD Text Dump Version String: FreeBSD 10.3-RELEASE-p3 #1 3ef16fb(RELENG_2_3_1): Tue May 17 19:34:28 CDT 2016 root@ce23-i386-builder:/builder/pfsense-231/tmp/obj/builder/pfsense-231/tmp/FreeBSD-src/sys/pfSense Panic String: Dump Parity: 2296393849 Bounds: 0 Dump Status: good Filename: /var/crash/textdump.tar.0 ddb.txt06000014000012721125337 7075 ustarrootwheeldb:0:kdb.enter.default> run lockinfo db:1:lockinfo> show locks No such command db:1:locks> show alllocks No such command db:1:alllocks> show lockedvnods Locked vnodes db:0:kdb.enter.default> show pcpu cpuid = 0 dynamic pcpu = 0x56f080 curthread = 0xc7345320: pid 30208 "ifconfig" curpcb = 0xef381d40 fpcurthread = none idlethread = 0xc6716000: tid 100003 "idle: cpu0" APIC ID = 0 currentldt = 0x50 db:0:kdb.enter.default> bt Tracing pid 30208 tid 100215 td 0xc7345320 sctp_find_ifa_by_addr(c75bec98,0,1,3,c75bb300,...) at sctp_find_ifa_by_addr+0x93/frame 0xef381784 sctp_del_addr_from_vrf(0,c75bec98,1,c6894014,0,0,0,0,c6ebbe00,0,0) at sctp_del_addr_from_vrf+0x91/frame 0xef3817b0 sctp_addr_change(c75bec00,2,0,ef381870,ffffffff,...) at sctp_addr_change+0x184/frame 0xef3817f0 rt_newaddrmsg_fib(2,c75bec00,0,ef381870,ffffffff) at rt_newaddrmsg_fib+0x9b/frame 0xef381818 rt_newaddrmsg(2,c75bec00,0,ef381870,0,...) at rt_newaddrmsg+0x2f/frame 0xef381838 in6_purgeaddr(c75bec00,19e,0,c272a1b4,1,...) at in6_purgeaddr+0x202/frame 0xef381970 in6_control(c7220848,81206919,c75bea00,c6894000,c7345320,...) at in6_control+0x73b/frame 0xef381a38 ifioctl(c7220848,81206919,c75bea00,c7345320,10000000,...) at ifioctl+0x11ed/frame 0xef381ae4 soo_ioctl(c6d45620,81206919,c75bea00,c6d25a00,c7345320,...) at soo_ioctl+0x2e8/frame 0xef381b10 kern_ioctl(c7345320,3,81206919,c75bea00,c12eb311,...) at kern_ioctl+0x258/frame 0xef381b58 sys_ioctl(c7345320,ef381ca8,c7360000,ef381c64,c0d1bba2,...) at sys_ioctl+0xfc/frame 0xef381c18 syscall(ef381ce8) at syscall+0x5c9/frame 0xef381cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef381cdc --- syscall (54, FreeBSD ELF32, sys_ioctl), eip = 0x2820988f, esp = 0xbfbfe5f8, ebp = 0xbfbfe628 --- db:0:kdb.enter.default> ps pid ppid pgrp uid state wmesg wchan cmd 30208 24894 273 0 R CPU 0 ifconfig 30151 29938 18646 0 S sbwait 0xc731d408 fcgicli 29938 18646 18646 0 S wait 0xc6cc9c20 sh 27859 1 27859 0 Ss (threaded) dpinger 101138 S accept 0xc6ff8bd6 dpinger 101137 S nanslp 0xc1f7ac00 dpinger 101136 S nanslp 0xc1f7ac00 dpinger 101135 S sbwait 0xc731c758 dpinger 101130 S uwait 0xc6d25000 dpinger 27678 1 27678 0 Ss (threaded) dpinger 101134 S accept 0xc6ff8536 dpinger 101133 S nanslp 0xc1f7ac00 dpinger 101132 S nanslp 0xc1f7ac00 dpinger 101131 S sbwait 0xc731bc50 dpinger 101126 S uwait 0xc6fe1500 dpinger 24894 273 273 0 S piperd 0xc6d544c8 php-fpm 18456 1 18456 0 Ss kqread 0xc7122480 dhcpleases 18198 273 273 0 R php-fpm 17505 1 17505 59 Ss kqread 0xc712f880 unbound 3719 273 273 0 S accept 0xc6ea1536 php-fpm 36456 65676 22 0 S nanslp 0xc1f7ac00 sleep 47284 47133 47284 0 S+ ttyin 0xc67b6a70 sh 47133 43557 47133 0 S+ wait 0xc670fc20 sh 46794 10469 46794 0 Ss (threaded) sshlockout_pf 100223 S nanslp 0xc1f7ac00 sshlockout_pf 100217 S piperd 0xc6d55330 sshlockout_pf 43557 1 43557 0 Ss+ wait 0xc7360c20 login 14501 14200 14200 0 S nanslp 0xc1f7ac00 minicron 14200 1 14200 0 Ss wait 0xc7361308 minicron 14107 13476 13476 0 S nanslp 0xc1f7ac00 minicron 13476 1 13476 0 Ss wait 0xc7216c20 minicron 13234 12890 12890 0 S nanslp 0xc1f7ac00 minicron 12890 1 12890 0 Ss wait 0xc6dfec20 minicron 10469 1 10469 0 Ss select 0xc72be3a4 syslogd 65676 1 22 0 S+ wait 0xc7216308 sh 58300 58108 58300 0 Ss (threaded) charon 100167 S uwait 0xc72a6a80 charon 100166 S uwait 0xc72a6a00 charon 100165 S uwait 0xc6c9d880 charon 100164 S uwait 0xc6c9d680 charon 100163 S uwait 0xc7123280 charon 100162 S uwait 0xc7123600 charon 100161 S uwait 0xc7123400 charon 100160 S uwait 0xc6d28680 charon 100159 S uwait 0xc7123680 charon 100158 S uwait 0xc7201200 charon 100157 S uwait 0xc71c6c80 charon 100156 S uwait 0xc71c6c00 charon 100155 S uwait 0xc71c5a80 charon 100154 S select 0xc6ea9124 charon 100153 S select 0xc68e55a4 charon 100152 S uwait 0xc71c6b00 charon 100137 S sigwait 0xc6fcb000 charon 58108 1 58108 0 Ss select 0xc68e59e4 starter 51422 1 51343 0 S select 0xc6ea62a4 radvd 46825 1 46825 0 Ss (threaded) ntpd 100104 S select 0xc6ea67a4 ntpd 41432 1 41432 0 Ss nanslp 0xc1f7ac00 cron 41274 40846 40846 0 S kqread 0xc71c3380 nginx 41090 40846 40846 0 S kqread 0xc71c3300 nginx 40846 1 40846 0 Ss pause 0xc711d36c nginx 18646 1 18646 0 Ss wait 0xc711e000 dhcp6c 16777 1 16777 0 Ss select 0xc6ea79e4 xinetd 16441 1 16441 0 Ss bpf 0xc6eda800 filterlog 11841 1 11841 65 Ss select 0xc6ea8da4 dhclient 7139 1 7139 0 Ss select 0xc6a95da4 dhclient 6187 1 6187 0 Ss (threaded) sshlockout_pf 100080 S nanslp 0xc1f7ac00 sshlockout_pf 100074 S uwait 0xc6c9df00 sshlockout_pf 5923 1 5923 0 Ss select 0xc6a932e4 sshd 324 1 324 0 Ss select 0xc6ea87a4 devd 313 311 311 0 S kqread 0xc6d28c00 check_reload_status 311 1 311 0 Ss kqread 0xc6d28d80 check_reload_status 273 1 273 0 Ss kqread 0xc6e52380 php-fpm 56 0 0 0 DL mdwait 0xc6cee800 [md0] 21 0 0 0 DL syncer 0xc1f99f44 [syncer] 20 0 0 0 DL vlruwt 0xc6cc7c20 [vnlru] 19 0 0 0 DL (threaded) [bufdaemon] 100064 D sdflush 0xc690d284 [/ worker] 100057 D psleep 0xc1f99684 [bufdaemon] 18 0 0 0 DL pollid 0xc1f79770 [idlepoll] 17 0 0 0 DL pgzero 0xc1fa4128 [pagezero] 9 0 0 0 DL psleep 0xc1fa3e44 [vmdaemon] 8 0 0 0 DL (threaded) [pagedaemon] 100060 D umarcl 0xc1fa3a24 [uma] 100053 D psleep 0xc2027584 [pagedaemon] 7 0 0 0 DL waiting_ 0xc2021ef4 [sctp_iterator] 6 0 0 0 DL pftm 0xc0f94ba0 [pf purge] 5 0 0 0 DL - 0xc67b743c [fdc0] 16 0 0 0 DL (threaded) [usb] 100041 D - 0xc68d1ddc [usbus1] 100040 D - 0xc68d1dac [usbus1] 100039 D - 0xc68d1d7c [usbus1] 100038 D - 0xc68d1d4c [usbus1] 100037 D - 0xc68d1d1c [usbus1] 100035 D - 0xc687cddc [usbus0] 100034 D - 0xc687cdac [usbus0] 100033 D - 0xc687cd7c [usbus0] 100032 D - 0xc687cd4c [usbus0] 100031 D - 0xc687cd1c [usbus0] 4 0 0 0 DL (threaded) [cam] 100052 D - 0xc1eb8e2c [scanner] 100018 D - 0xc1eb8f80 [doneq0] 15 0 0 0 DL - 0xc1ed6048 [rand_harvestq] 3 0 0 0 DL crypto_r 0xc1fa2da8 [crypto returns] 2 0 0 0 DL crypto_w 0xc1fa2ce8 [crypto] 14 0 0 0 DL (threaded) [geom] 100010 D - 0xc201cf58 [g_down] 100009 D - 0xc201cf54 [g_up] 100008 D - 0xc201cf50 [g_event] 13 0 0 0 DL sleep 0xc1e907dc [ng_queue0] 12 0 0 0 WL (threaded) [intr] 100050 I [swi1: pfsync] 100048 I [swi1: pf send] 100045 I [irq7: ppc0] 100044 I [swi0: uart uart] 100043 I [irq1: atkbd0] 100036 I [irq23: uhci1] 100030 I [irq19: uhci0] 100029 I [irq15: ata1] 100028 I [irq14: ata0] 100027 I [irq18: xl0] 100026 I [irq17: pcm0 rl1] 100025 I [irq16: rl0] 100023 I [swi6: task queue] 100017 I [swi5: fast taskq] 100015 I [swi6: Giant taskq] 100006 I [swi1: netisr 0] 100005 I [swi3: vm] 100004 I [swi4: clock] 11 0 0 0 RL [idle: cpu0] 1 0 1 0 SLs wait 0xc670f308 [init] 10 0 0 0 DL audit_wo 0xc2025e5c [audit] 0 0 0 0 DLs (threaded) [kernel] 100051 D - 0xc6711180 [CAM taskq] 100046 D - 0xc68de980 [mca taskq] 100024 D - 0xc65c8b80 [ffs_trim taskq] 100022 D - 0xc65c8e00 [kqueue taskq] 100021 D - 0xc65c8e80 [acpi_task_2] 100020 D - 0xc65c8e80 [acpi_task_1] 100019 D - 0xc65c8e80 [acpi_task_0] 100016 D - 0xc6711300 [thread] 100011 D - 0xc6712780 [firmware taskq] 100000 D swapin 0xc201cfd4 [swapper] db:0:kdb.enter.default> alltrace Tracing command ifconfig pid 30208 tid 100215 td 0xc7345320 sctp_find_ifa_by_addr(c75bec98,0,1,3,c75bb300,...) at sctp_find_ifa_by_addr+0x93/frame 0xef381784 sctp_del_addr_from_vrf(0,c75bec98,1,c6894014,0,0,0,0,c6ebbe00,0,0) at sctp_del_addr_from_vrf+0x91/frame 0xef3817b0 sctp_addr_change(c75bec00,2,0,ef381870,ffffffff,...) at sctp_addr_change+0x184/frame 0xef3817f0 rt_newaddrmsg_fib(2,c75bec00,0,ef381870,ffffffff) at rt_newaddrmsg_fib+0x9b/frame 0xef381818 rt_newaddrmsg(2,c75bec00,0,ef381870,0,...) at rt_newaddrmsg+0x2f/frame 0xef381838 in6_purgeaddr(c75bec00,19e,0,c272a1b4,1,...) at in6_purgeaddr+0x202/frame 0xef381970 in6_control(c7220848,81206919,c75bea00,c6894000,c7345320,...) at in6_control+0x73b/frame 0xef381a38 ifioctl(c7220848,81206919,c75bea00,c7345320,10000000,...) at ifioctl+0x11ed/frame 0xef381ae4 soo_ioctl(c6d45620,81206919,c75bea00,c6d25a00,c7345320,...) at soo_ioctl+0x2e8/frame 0xef381b10 kern_ioctl(c7345320,3,81206919,c75bea00,c12eb311,...) at kern_ioctl+0x258/frame 0xef381b58 sys_ioctl(c7345320,ef381ca8,c7360000,ef381c64,c0d1bba2,...) at sys_ioctl+0xfc/frame 0xef381c18 syscall(ef381ce8) at syscall+0x5c9/frame 0xef381cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef381cdc --- syscall (54, FreeBSD ELF32, sys_ioctl), eip = 0x2820988f, esp = 0xbfbfe5f8, ebp = 0xbfbfe628 --- Tracing command fcgicli pid 30151 tid 100172 td 0xc72eb960 sched_switch(c72eb960,0,104,c272ccf4,0,...) at sched_switch+0x2d2/frame 0xef351970 mi_switch(104,0,ef351a20,c0cf40f2,c1f86e00,c72eb960) at mi_switch+0x122/frame 0xef3519a8 sleepq_switch(c72eb960,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef3519d0 sleepq_catch_signals(0,c1480b5a,154,c0d8fb7c,c72eb960,...) at sleepq_catch_signals+0x5be/frame 0xef351a24 sleepq_wait_sig(c731d408,68,c1484807,100,0,...) at sleepq_wait_sig+0x14/frame 0xef351a50 _sleep(c731d408,c731d3c8,168,c1484807,0,0,0,0,0) at _sleep+0x29b/frame 0xef351a98 sbwait(c731d3a4,1,0,0,78,...) at sbwait+0x71/frame 0xef351acc soreceive_generic(c731d350,ef351b80,ef351b88,0,0,...) at soreceive_generic+0x273/frame 0xef351b38 kern_recvit(c72eb960,3,ef351be8,0,0,...) at kern_recvit+0x1ea/frame 0xef351bc0 sys_recvfrom(c72eb960,ef351ca8,c7215c20,c1f86e9c,c72eb960,...) at sys_recvfrom+0xa5/frame 0xef351c18 syscall(ef351ce8) at syscall+0x5c9/frame 0xef351cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef351cdc --- syscall (29, FreeBSD ELF32, sys_recvfrom), eip = 0x2814074b, esp = 0xbfbfe664, ebp = 0xbfbfe684 --- Tracing command sh pid 29938 tid 100271 td 0xc72eb640 sched_switch(c72eb640,0,104,c105926c,0,...) at sched_switch+0x2d2/frame 0xef2e8960 mi_switch(104,0,ef2e8a10,c0cf40f2,c1f86e00,c72eb640) at mi_switch+0x122/frame 0xef2e8998 sleepq_switch(c72eb640,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef2e89c0 sleepq_catch_signals(0,c1480b5a,154,0,c72eb640,...) at sleepq_catch_signals+0x5be/frame 0xef2e8a14 sleepq_wait_sig(c6cc9c20,6c,c1481b93,100,0,...) at sleepq_wait_sig+0x14/frame 0xef2e8a40 _sleep(c6cc9c20,c6cc9cb8,16c,c1481b93,0,...) at _sleep+0x29b/frame 0xef2e8a88 kern_wait6(c72eb640,7,0,0,ef2e8b24,...) at kern_wait6+0x71a/frame 0xef2e8af8 sys_wait4(c72eb640,ef2e8ca8,c6cc9c20,0,0,...) at sys_wait4+0x94/frame 0xef2e8c18 syscall(ef2e8ce8) at syscall+0x5c9/frame 0xef2e8cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef2e8cdc --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x281acf43, esp = 0xbfbfeb90, ebp = 0xbfbfeba8 --- Tracing command dpinger pid 27859 tid 101138 td 0xc72eb000 sched_switch(c72eb000,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef35ca20 mi_switch(104,0,ef35cad0,c0cf40f2,c1f86e00,c72eb000) at mi_switch+0x122/frame 0xef35ca58 sleepq_switch(c72eb000,0,c1480b5a,1a8,c711ec20,...) at sleepq_switch+0x15b/frame 0xef35ca80 sleepq_catch_signals(0,c1480b5a,154,ef35cb18,c0cc075a,...) at sleepq_catch_signals+0x5be/frame 0xef35cad4 sleepq_wait_sig(c6ff8bd6,68,c1481c03,100,0,...) at sleepq_wait_sig+0x14/frame 0xef35cb00 _sleep(c6ff8bd6,c20216d4,168,c1481c03,0,...) at _sleep+0x29b/frame 0xef35cb48 kern_accept4(c72eb000,5,0,0,1,...) at kern_accept4+0x1dd/frame 0xef35cbc4 accept1(0,0,1) at accept1+0xad/frame 0xef35cc00 sys_accept(c72eb000,ef35cca8,c711ec20,c72eb000,c711ec20,...) at sys_accept+0x29/frame 0xef35cc18 syscall(ef35cce8) at syscall+0x5c9/frame 0xef35ccdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef35ccdc --- syscall (30, FreeBSD ELF32, sys_accept), eip = 0x28156073, esp = 0xbb8fae9c, ebp = 0xbb8faeb8 --- Tracing command dpinger pid 27859 tid 101137 td 0xc7460c80 sched_switch(c7460c80,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef2dba4c mi_switch(104,0,20e,10000000,0,c7460c80) at mi_switch+0x122/frame 0xef2dba84 sleepq_switch(c7460c80,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef2dbaac sleepq_catch_signals(10000000,0,c0d5f550,c7460c80,0,...) at sleepq_catch_signals+0x5be/frame 0xef2dbb00 sleepq_timedwait_sig(c1f7ac00,6c,20e,10000000,0,...) at sleepq_timedwait_sig+0x14/frame 0xef2dbb30 _sleep(c1f7ac00,0,16c,c147d390,9e01e04,...) at _sleep+0x26b/frame 0xef2dbb78 kern_nanosleep(bb9fbf78,ef2dbbf0,ef2dbbf8,ef2dbce8,1,...) at kern_nanosleep+0x14b/frame 0xef2dbbd8 sys_nanosleep(c7460c80,ef2dbca8,0,c7460c80,c711ec20,...) at sys_nanosleep+0x69/frame 0xef2dbc18 syscall(ef2dbce8) at syscall+0x5c9/frame 0xef2dbcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef2dbcdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28156023, esp = 0xbb9fbf18, ebp = 0xbb9fbf30 --- Tracing command dpinger pid 27859 tid 101136 td 0xc7461320 sched_switch(c7461320,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef3a1a4c mi_switch(104,0,20d,7ffffff,0,c7461320) at mi_switch+0x122/frame 0xef3a1a84 sleepq_switch(c7461320,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef3a1aac sleepq_catch_signals(7ffffff,0,c0d5f550,c7461320,0,...) at sleepq_catch_signals+0x5be/frame 0xef3a1b00 sleepq_timedwait_sig(c1f7ac00,6c,20d,7ffffff,0,...) at sleepq_timedwait_sig+0x14/frame 0xef3a1b30 _sleep(c1f7ac00,0,16c,c147d390,89e01e03,...) at _sleep+0x26b/frame 0xef3a1b78 kern_nanosleep(bbafcf88,ef3a1bf0,ef3a1bf8,ef3a1ce8,0,...) at kern_nanosleep+0x14b/frame 0xef3a1bd8 sys_nanosleep(c7461320,ef3a1ca8,c711ec20,c7461320,c711ec20,...) at sys_nanosleep+0x69/frame 0xef3a1c18 syscall(ef3a1ce8) at syscall+0x5c9/frame 0xef3a1cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef3a1cdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28156023, esp = 0xbbafcf50, ebp = 0xbbafcf68 --- Tracing command dpinger pid 27859 tid 101135 td 0xc7460960 sched_switch(c7460960,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef38f970 mi_switch(104,0,ef38fa20,c0cf40f2,c1f86e00,c7460960) at mi_switch+0x122/frame 0xef38f9a8 sleepq_switch(c7460960,0,c1480b5a,1a8,2811c000,...) at sleepq_switch+0x15b/frame 0xef38f9d0 sleepq_catch_signals(0,c1480b5a,154,c4835a7c,28123000,...) at sleepq_catch_signals+0x5be/frame 0xef38fa24 sleepq_wait_sig(c731c758,68,c1484807,100,0,...) at sleepq_wait_sig+0x14/frame 0xef38fa50 _sleep(c731c758,c731c718,168,c1484807,0,0,0,0,0) at _sleep+0x29b/frame 0xef38fa98 sbwait(c731c6f4,1,c48bdb20,c4835b24,a0,...) at sbwait+0x71/frame 0xef38facc soreceive_generic(c731c6a0,ef38fb80,ef38fb88,0,0,...) at soreceive_generic+0x273/frame 0xef38fb38 kern_recvit(c7460960,4,ef38fbe8,0,0,...) at kern_recvit+0x1ea/frame 0xef38fbc0 sys_recvfrom(c7460960,ef38fca8,c711ec20,c7460960,c711ec20,...) at sys_recvfrom+0xa5/frame 0xef38fc18 syscall(ef38fce8) at syscall+0x5c9/frame 0xef38fcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef38fcdc --- syscall (29, FreeBSD ELF32, sys_recvfrom), eip = 0x2816074b, esp = 0xbbbfde98, ebp = 0xbbbfdec0 --- Tracing command dpinger pid 27859 tid 101130 td 0xc6df8000 sched_switch(c6df8000,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef395a08 mi_switch(104,0,ef395ab8,c0cf40f2,c1f86e00,c6df8000) at mi_switch+0x122/frame 0xef395a40 sleepq_switch(c6df8000,0,c1480b5a,1a8,ffffffff,...) at sleepq_switch+0x15b/frame 0xef395a68 sleepq_catch_signals(0,c1480b5a,154,c730e260,ef395af8,...) at sleepq_catch_signals+0x5be/frame 0xef395abc sleepq_wait_sig(c6d25000,0,c147d7d2,100,0,...) at sleepq_wait_sig+0x14/frame 0xef395ae8 _sleep(c6d25000,c1f85d54,300,c147d7d2,0,...) at _sleep+0x29b/frame 0xef395b30 umtxq_sleep(0,ef395bbc,2,c6d25008,c711ec20,...) at umtxq_sleep+0x125/frame 0xef395b90 do_wait(18b12,0,0,0,14,...) at do_wait+0x371/frame 0xef395be8 __umtx_op_wait(c6df8000,ef395ca8,c711ec20,c6df8000,c714c000,...) at __umtx_op_wait+0x9f/frame 0xef395c18 syscall(ef395ce8) at syscall+0x5c9/frame 0xef395cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef395cdc --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x280841a3, esp = 0xbfbfe8fc, ebp = 0xbfbfe918 --- Tracing command dpinger pid 27678 tid 101134 td 0xc7232000 sched_switch(c7232000,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef345a20 mi_switch(104,0,ef345ad0,c0cf40f2,c1f86e00,c7232000) at mi_switch+0x122/frame 0xef345a58 sleepq_switch(c7232000,0,c1480b5a,1a8,c7215308,...) at sleepq_switch+0x15b/frame 0xef345a80 sleepq_catch_signals(0,c1480b5a,154,ef345b18,c0cc075a,...) at sleepq_catch_signals+0x5be/frame 0xef345ad4 sleepq_wait_sig(c6ff8536,68,c1481c03,100,0,...) at sleepq_wait_sig+0x14/frame 0xef345b00 _sleep(c6ff8536,c20216d4,168,c1481c03,0,...) at _sleep+0x29b/frame 0xef345b48 kern_accept4(c7232000,5,0,0,1,...) at kern_accept4+0x1dd/frame 0xef345bc4 accept1(0,0,1) at accept1+0xad/frame 0xef345c00 sys_accept(c7232000,ef345ca8,c7215308,c7232000,c7215308,...) at sys_accept+0x29/frame 0xef345c18 syscall(ef345ce8) at syscall+0x5c9/frame 0xef345cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef345cdc --- syscall (30, FreeBSD ELF32, sys_accept), eip = 0x28156073, esp = 0xbb8fae9c, ebp = 0xbb8faeb8 --- Tracing command dpinger pid 27678 tid 101133 td 0xc6df7640 sched_switch(c6df7640,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef2f2a4c mi_switch(104,0,20e,10000000,0,c6df7640) at mi_switch+0x122/frame 0xef2f2a84 sleepq_switch(c6df7640,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef2f2aac sleepq_catch_signals(10000000,0,c0d5f550,c6df7640,0,...) at sleepq_catch_signals+0x5be/frame 0xef2f2b00 sleepq_timedwait_sig(c1f7ac00,6c,20e,10000000,0,...) at sleepq_timedwait_sig+0x14/frame 0xef2f2b30 _sleep(c1f7ac00,0,16c,c147d390,7926479,...) at _sleep+0x26b/frame 0xef2f2b78 kern_nanosleep(bb9fbf78,ef2f2bf0,ef2f2bf8,ef2f2ce8,1,...) at kern_nanosleep+0x14b/frame 0xef2f2bd8 sys_nanosleep(c6df7640,ef2f2ca8,0,c6df7640,c7215308,...) at sys_nanosleep+0x69/frame 0xef2f2c18 syscall(ef2f2ce8) at syscall+0x5c9/frame 0xef2f2cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef2f2cdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28156023, esp = 0xbb9fbf18, ebp = 0xbb9fbf30 --- Tracing command dpinger pid 27678 tid 101132 td 0xc7461960 sched_switch(c7461960,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef2d5a4c mi_switch(104,0,20d,7ffffff,0,c7461960) at mi_switch+0x122/frame 0xef2d5a84 sleepq_switch(c7461960,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef2d5aac sleepq_catch_signals(7ffffff,0,c0d5f550,c7461960,0,...) at sleepq_catch_signals+0x5be/frame 0xef2d5b00 sleepq_timedwait_sig(c1f7ac00,6c,20d,7ffffff,0,...) at sleepq_timedwait_sig+0x14/frame 0xef2d5b30 _sleep(c1f7ac00,0,16c,c147d390,87926478,...) at _sleep+0x26b/frame 0xef2d5b78 kern_nanosleep(bbafcf88,ef2d5bf0,ef2d5bf8,ef2d5ce8,0,...) at kern_nanosleep+0x14b/frame 0xef2d5bd8 sys_nanosleep(c7461960,ef2d5ca8,c7215308,c7461960,c7215308,...) at sys_nanosleep+0x69/frame 0xef2d5c18 syscall(ef2d5ce8) at syscall+0x5c9/frame 0xef2d5cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef2d5cdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28156023, esp = 0xbbafcf50, ebp = 0xbbafcf68 --- Tracing command dpinger pid 27678 tid 101131 td 0xc7461640 sched_switch(c7461640,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef38b970 mi_switch(104,0,ef38ba20,c0cf40f2,c1f86e00,c7461640) at mi_switch+0x122/frame 0xef38b9a8 sleepq_switch(c7461640,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef38b9d0 sleepq_catch_signals(0,c1480b5a,154,0,0,...) at sleepq_catch_signals+0x5be/frame 0xef38ba24 sleepq_wait_sig(c731bc50,68,c1484807,100,0,...) at sleepq_wait_sig+0x14/frame 0xef38ba50 _sleep(c731bc50,c731bc10,168,c1484807,0,0,0,0,0) at _sleep+0x29b/frame 0xef38ba98 sbwait(c731bbec,1,0,0,a0,...) at sbwait+0x71/frame 0xef38bacc soreceive_generic(c731bb98,ef38bb80,ef38bb88,0,0,...) at soreceive_generic+0x273/frame 0xef38bb38 kern_recvit(c7461640,4,ef38bbe8,0,0,...) at kern_recvit+0x1ea/frame 0xef38bbc0 sys_recvfrom(c7461640,ef38bca8,c7215308,c7461640,c7215308,...) at sys_recvfrom+0xa5/frame 0xef38bc18 syscall(ef38bce8) at syscall+0x5c9/frame 0xef38bcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef38bcdc --- syscall (29, FreeBSD ELF32, sys_recvfrom), eip = 0x2816074b, esp = 0xbbbfde98, ebp = 0xbbbfdec0 --- Tracing command dpinger pid 27678 tid 101126 td 0xc6fdd960 sched_switch(c6fdd960,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xe0ffda08 mi_switch(104,0,e0ffdab8,c0cf40f2,c1f86e00,c6fdd960) at mi_switch+0x122/frame 0xe0ffda40 sleepq_switch(c6fdd960,0,c1480b5a,1a8,ffffffff,...) at sleepq_switch+0x15b/frame 0xe0ffda68 sleepq_catch_signals(0,c1480b5a,154,c730e850,e0ffdaf8,...) at sleepq_catch_signals+0x5be/frame 0xe0ffdabc sleepq_wait_sig(c6fe1500,0,c147d7d2,100,0,...) at sleepq_wait_sig+0x14/frame 0xe0ffdae8 _sleep(c6fe1500,c1f83144,300,c147d7d2,0,...) at _sleep+0x29b/frame 0xe0ffdb30 umtxq_sleep(0,e0ffdbbc,2,c6fe1508,c7215308,...) at umtxq_sleep+0x125/frame 0xe0ffdb90 do_wait(18b0e,0,0,0,14,...) at do_wait+0x371/frame 0xe0ffdbe8 __umtx_op_wait(c6fdd960,e0ffdca8,c7215308,c6fdd960,c714c000,...) at __umtx_op_wait+0x9f/frame 0xe0ffdc18 syscall(e0ffdce8) at syscall+0x5c9/frame 0xe0ffdcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xe0ffdcdc --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x280841a3, esp = 0xbfbfe97c, ebp = 0xbfbfe998 --- Tracing command php-fpm pid 24894 tid 100818 td 0xc7232c80 sched_switch(c7232c80,0,104,634c6e,ffffff,...) at sched_switch+0x2d2/frame 0xef35f9dc mi_switch(104,0,c7232c80,ef35fa8c,c0cf40f2,c7232c80) at mi_switch+0x122/frame 0xef35fa14 sleepq_switch(c7232c80,0,c1480b5a,1a8,c7232c80,...) at sleepq_switch+0x15b/frame 0xef35fa3c sleepq_catch_signals(0,c1480b5a,154,0,0,...) at sleepq_catch_signals+0x5be/frame 0xef35fa90 sleepq_wait_sig(c6d544c8,5c,c14816f3,100,0,...) at sleepq_wait_sig+0x14/frame 0xef35fabc _sleep(c6d544c8,c6d54648,15c,c14816f3,0,...) at _sleep+0x29b/frame 0xef35fb04 pipe_read(c6d452a0,ef35fbe8,c6d25a00,0,c7232c80,...) at pipe_read+0x31a/frame 0xef35fb48 dofileread(c6d452a0,ef35fbe8,ffffffff,ffffffff,0,...) at dofileread+0x9e/frame 0xef35fb80 kern_readv(c7232c80,c,ef35fbe8,ef35fc18,28f83e58,...) at kern_readv+0x96/frame 0xef35fbc8 sys_read(c7232c80,ef35fca8,fffffed8,0,0,...) at sys_read+0x5c/frame 0xef35fc18 syscall(ef35fce8) at syscall+0x5c9/frame 0xef35fcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef35fcdc --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x2875885f, esp = 0xbfbfab20, ebp = 0xbfbfab3c --- Tracing command dhcpleases pid 18456 tid 100272 td 0xc7462000 sched_switch(c7462000,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef39e8e8 mi_switch(104,0,ef39e998,c0cf40f2,c1f86e00,c7462000) at mi_switch+0x122/frame 0xef39e920 sleepq_switch(c7462000,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef39e948 sleepq_catch_signals(0,c1480b5a,154,ef39e9e8,ef39e9f8,...) at sleepq_catch_signals+0x5be/frame 0xef39e99c sleepq_wait_sig(c7122480,68,c1476c60,100,0,...) at sleepq_wait_sig+0x14/frame 0xef39e9c8 _sleep(c7122480,c7122480,168,c1476c60,0,...) at _sleep+0x29b/frame 0xef39ea10 kern_kevent_fp(c7462000,c6d46d58,1,1,ef39ebe8,...) at kern_kevent_fp+0x3bc/frame 0xef39eb48 kern_kevent(c7462000,3,1,1,ef39ebe8,...) at kern_kevent+0xdf/frame 0xef39eb98 sys_kevent(c7462000,ef39eca8,c735f308,c7462000,c714c000,...) at sys_kevent+0x131/frame 0xef39ec18 syscall(ef39ece8) at syscall+0x5c9/frame 0xef39ecdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef39ecdc --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2813d75b, esp = 0xbfbfec64, ebp = 0xbfbfed54 --- Tracing command php-fpm pid 18198 tid 100124 td 0xc6fe2640 sched_switch(c6fe2640,0,602,282,c1f86e00,...) at sched_switch+0x2d2/frame 0xef307c2c mi_switch(602,0,c147caec,d3,c6fe2640,...) at mi_switch+0x122/frame 0xef307c64 critical_exit(c66f4ecc,0,c6fe2640,12,c66f4ecc,...) at critical_exit+0x89/frame 0xef307c80 intr_event_handle(c6713100,ef307ce8,2d05fc00,ef307ce8,28f69e50,...) at intr_event_handle+0xeb/frame 0xef307ca0 intr_execute_handlers(c66f4ecc,ef307ce8,0) at intr_execute_handlers+0x42/frame 0xef307cc0 lapic_handle_intr(35,ef307ce8) at lapic_handle_intr+0x40/frame 0xef307cd8 Xapic_isr1() at Xapic_isr1+0x43/frame 0xef307cd8 --- interrupt, eip = 0x28758ef0, esp = 0xbfbfa564, ebp = 0xbfbfa590 --- Tracing command unbound pid 17505 tid 100177 td 0xc72f0320 sched_switch(c72f0320,0,104,ad,98,...) at sched_switch+0x2d2/frame 0xef34b8e4 mi_switch(104,0,9a79,f0000000,986,c72f0320) at mi_switch+0x122/frame 0xef34b91c sleepq_switch(c72f0320,0,c1480b5a,1a8,986,...) at sleepq_switch+0x15b/frame 0xef34b944 sleepq_catch_signals(f0000000,986,c0d5f550,c72f0320,0,...) at sleepq_catch_signals+0x5be/frame 0xef34b998 sleepq_timedwait_sig(c712f880,68,9a79,f0000000,986,...) at sleepq_timedwait_sig+0x14/frame 0xef34b9c8 _sleep(c712f880,c712f880,168,c1476c60,ce7c3077,...) at _sleep+0x26b/frame 0xef34ba10 kern_kevent_fp(c72f0320,c6d48310,0,40,ef34bbe8,...) at kern_kevent_fp+0x3bc/frame 0xef34bb48 kern_kevent(c72f0320,d,0,40,ef34bbe8,...) at kern_kevent+0xdf/frame 0xef34bb98 sys_kevent(c72f0320,ef34bca8,c0d3d592,c72f0320,c714c000,...) at sys_kevent+0x131/frame 0xef34bc18 syscall(ef34bce8) at syscall+0x5c9/frame 0xef34bcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef34bcdc --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2843975b, esp = 0xbfbfeb8c, ebp = 0xbfbfebb8 --- Tracing command php-fpm pid 3719 tid 100214 td 0xc7345640 sched_switch(c7345640,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef37ea20 mi_switch(104,0,ef37ead0,c0cf40f2,c1f86e00,c7345640) at mi_switch+0x122/frame 0xef37ea58 sleepq_switch(c7345640,0,c1480b5a,1a8,c7360308,...) at sleepq_switch+0x15b/frame 0xef37ea80 sleepq_catch_signals(0,c1480b5a,154,ef37eb18,c0cc075a,...) at sleepq_catch_signals+0x5be/frame 0xef37ead4 sleepq_wait_sig(c6ea1536,68,c1481c03,100,0,...) at sleepq_wait_sig+0x14/frame 0xef37eb00 _sleep(c6ea1536,c20216d4,168,c1481c03,0,...) at _sleep+0x29b/frame 0xef37eb48 kern_accept4(c7345640,0,ef37ebf0,ef37ebec,1,...) at kern_accept4+0x1dd/frame 0xef37ebc4 accept1(bfbeca88,bfbeca84,1) at accept1+0x63/frame 0xef37ec00 sys_accept(c7345640,ef37eca8,c7360308,c7345640,c7345320,...) at sys_accept+0x29/frame 0xef37ec18 syscall(ef37ece8) at syscall+0x5c9/frame 0xef37ecdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef37ecdc --- syscall (30, FreeBSD ELF32, sys_accept), eip = 0x286f6073, esp = 0xbfbeca1c, ebp = 0xbfbeca38 --- Tracing command sleep pid 36456 tid 100184 td 0xc7232960 sched_switch(c7232960,0,104,c0,98,...) at sched_switch+0x2d2/frame 0xef362a4c mi_switch(104,0,21a,c0000000,3,c7232960) at mi_switch+0x122/frame 0xef362a84 sleepq_switch(c7232960,0,c1480b5a,1a8,3,...) at sleepq_switch+0x15b/frame 0xef362aac sleepq_catch_signals(c0000000,3,c0d5f550,c7232960,0,...) at sleepq_catch_signals+0x5be/frame 0xef362b00 sleepq_timedwait_sig(c1f7ac00,6c,21a,c0000000,3,...) at sleepq_timedwait_sig+0x14/frame 0xef362b30 _sleep(c1f7ac00,0,16c,c147d390,3771a391,...) at _sleep+0x26b/frame 0xef362b78 kern_nanosleep(bfbfede0,ef362bf0,ef362bf8,ef362ce8,3c,...) at kern_nanosleep+0x14b/frame 0xef362bd8 sys_nanosleep(c7232960,ef362ca8,c7215000,c7232960,c72ebc80,...) at sys_nanosleep+0x69/frame 0xef362c18 syscall(ef362ce8) at syscall+0x5c9/frame 0xef362cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef362cdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28131023, esp = 0xbfbfedac, ebp = 0xbfbfedf4 --- Tracing command sh pid 47284 tid 100213 td 0xc7345960 sched_switch(c7345960,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef37b984 mi_switch(104,0,c7345960,ef37ba34,c0cf40f2,c7345960) at mi_switch+0x122/frame 0xef37b9bc sleepq_switch(c7345960,0,c1480b5a,1a8,c1eabba0,...) at sleepq_switch+0x15b/frame 0xef37b9e4 sleepq_catch_signals(0,c1480b5a,154,c1ed68a4,c1fb9cf0,...) at sleepq_catch_signals+0x5be/frame 0xef37ba38 sleepq_wait_sig(c67b6a70,0,c148347b,101,0,...) at sleepq_wait_sig+0x14/frame 0xef37ba64 _cv_wait_sig(c67b6a70,c201d924,c67b6a00,0,ef37baec,...) at _cv_wait_sig+0x18c/frame 0xef37ba94 tty_wait(c67b6a00,c67b6a70,1,ef37badb,c65e5000,...) at tty_wait+0x1f/frame 0xef37baac ttydisc_read(c67b6a00,ef37bbe8,0,c0dd3df0,c6d487a8,...) at ttydisc_read+0xfe/frame 0xef37baec ttydev_read(c68e6c00,ef37bbe8,0,ef37bb40,c7345960,...) at ttydev_read+0x8f/frame 0xef37bb10 devfs_read_f(c6d487a8,ef37bbe8,c7319c80,0,c7345960,...) at devfs_read_f+0xb4/frame 0xef37bb48 dofileread(c6d487a8,ef37bbe8,ffffffff,ffffffff,0,...) at dofileread+0x9e/frame 0xef37bb80 kern_readv(c7345960,0,ef37bbe8,ef37bc18,bfbfead3,...) at kern_readv+0x96/frame 0xef37bbc8 sys_read(c7345960,ef37bca8,c7360610,0,0,...) at sys_read+0x5c/frame 0xef37bc18 syscall(ef37bce8) at syscall+0x5c9/frame 0xef37bcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef37bcdc --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x2820f85f, esp = 0xbfbfea0c, ebp = 0xbfbfeae8 --- Tracing command sh pid 47133 tid 100065 td 0xc68ec640 sched_switch(c68ec640,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xe0fbf960 mi_switch(104,0,e0fbfa10,c0cf40f2,c1f86e00,c68ec640) at mi_switch+0x122/frame 0xe0fbf998 sleepq_switch(c68ec640,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xe0fbf9c0 sleepq_catch_signals(0,c1480b5a,154,1,c68ec640,...) at sleepq_catch_signals+0x5be/frame 0xe0fbfa14 sleepq_wait_sig(c670fc20,6c,c1481b93,100,0,...) at sleepq_wait_sig+0x14/frame 0xe0fbfa40 _sleep(c670fc20,c670fcb8,16c,c1481b93,0,...) at _sleep+0x29b/frame 0xe0fbfa88 kern_wait6(c68ec640,7,0,0,e0fbfb24,...) at kern_wait6+0x71a/frame 0xe0fbfaf8 sys_wait4(c68ec640,e0fbfca8,c670fc20,c1f86e00,28,...) at sys_wait4+0x94/frame 0xe0fbfc18 syscall(e0fbfce8) at syscall+0x5c9/frame 0xe0fbfcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xe0fbfcdc --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x281acf43, esp = 0xbfbfeaa0, ebp = 0xbfbfeab8 --- Tracing command sshlockout_pf pid 46794 tid 100223 td 0xc6fdcc80 sched_switch(c6fdcc80,0,104,0,0,...) at sched_switch+0x2d2/frame 0xef39ba4c mi_switch(104,0,730,80000000,70,c6fdcc80) at mi_switch+0x122/frame 0xef39ba84 sleepq_switch(c6fdcc80,0,c1480b5a,1a8,70,...) at sleepq_switch+0x15b/frame 0xef39baac sleepq_catch_signals(80000000,70,c0d5f550,c6fdcc80,0,...) at sleepq_catch_signals+0x5be/frame 0xef39bb00 sleepq_timedwait_sig(c1f7ac00,6c,730,80000000,70,...) at sleepq_timedwait_sig+0x14/frame 0xef39bb30 _sleep(c1f7ac00,0,16c,c147d390,1b0ee9e,...) at _sleep+0x26b/frame 0xef39bb78 kern_nanosleep(bbbfdf78,ef39bbf0,ef39bbf8,ef39bce8,708,...) at kern_nanosleep+0x14b/frame 0xef39bbd8 sys_nanosleep(c6fdcc80,ef39bca8,c735f918,c6fdcc80,c735f918,...) at sys_nanosleep+0x69/frame 0xef39bc18 syscall(ef39bce8) at syscall+0x5c9/frame 0xef39bcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef39bcdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28153023, esp = 0xbbbfdf48, ebp = 0xbbbfdf60 --- Tracing command sshlockout_pf pid 46794 tid 100217 td 0xc714cc80 sched_switch(c714cc80,0,104,0,c714cf70,...) at sched_switch+0x2d2/frame 0xef3879dc mi_switch(104,0,c714cc80,ef387a8c,c0cf40f2,c714cc80) at mi_switch+0x122/frame 0xef387a14 sleepq_switch(c714cc80,0,c1480b5a,1a8,ef387a8c,...) at sleepq_switch+0x15b/frame 0xef387a3c sleepq_catch_signals(0,c1480b5a,154,0,0,...) at sleepq_catch_signals+0x5be/frame 0xef387a90 sleepq_wait_sig(c6d55330,5c,c14816f3,100,0,...) at sleepq_wait_sig+0x14/frame 0xef387abc _sleep(c6d55330,c6d554b0,15c,c14816f3,0,...) at _sleep+0x29b/frame 0xef387b04 pipe_read(c6d45700,ef387be8,c6d25a00,0,c714cc80,...) at pipe_read+0x31a/frame 0xef387b48 dofileread(c6d45700,ef387be8,ffffffff,ffffffff,0,...) at dofileread+0x9e/frame 0xef387b80 kern_readv(c714cc80,0,ef387be8,ef387c18,2881a000,...) at kern_readv+0x96/frame 0xef387bc8 sys_read(c714cc80,ef387ca8,c735f918,c714cc80,28,...) at sys_read+0x5c/frame 0xef387c18 syscall(ef387ce8) at syscall+0x5c9/frame 0xef387cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef387cdc --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281b585f, esp = 0xbfbfe934, ebp = 0xbfbfe948 --- Tracing command login pid 43557 tid 100211 td 0xc6df9960 sched_switch(c6df9960,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef375960 mi_switch(104,0,ef375a10,c0cf40f2,c1f86e00,c6df9960) at mi_switch+0x122/frame 0xef375998 sleepq_switch(c6df9960,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef3759c0 sleepq_catch_signals(0,c1480b5a,154,0,2,...) at sleepq_catch_signals+0x5be/frame 0xef375a14 sleepq_wait_sig(c7360c20,6c,c1481b93,100,0,...) at sleepq_wait_sig+0x14/frame 0xef375a40 _sleep(c7360c20,c7360cb8,16c,c1481b93,0,...) at _sleep+0x29b/frame 0xef375a88 kern_wait6(c6df9960,0,b81d,0,ef375b24,...) at kern_wait6+0x71a/frame 0xef375af8 sys_wait4(c6df9960,ef375ca8,c7360c20,c2427c80,28,...) at sys_wait4+0x94/frame 0xef375c18 syscall(ef375ce8) at syscall+0x5c9/frame 0xef375cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef375cdc --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28151f43, esp = 0xbfbfed28, ebp = 0xbfbfed40 --- Tracing command minicron pid 14501 tid 100210 td 0xc7346960 sched_switch(c7346960,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef2d2a4c mi_switch(104,0,151a5,0,1518,c7346960) at mi_switch+0x122/frame 0xef2d2a84 sleepq_switch(c7346960,0,c1480b5a,1a8,1518,...) at sleepq_switch+0x15b/frame 0xef2d2aac sleepq_catch_signals(0,1518,c0d5f550,c7346960,0,...) at sleepq_catch_signals+0x5be/frame 0xef2d2b00 sleepq_timedwait_sig(c1f7ac00,6c,151a5,0,1518,...) at sleepq_timedwait_sig+0x14/frame 0xef2d2b30 _sleep(c1f7ac00,0,16c,c147d390,6f42c1a6,...) at _sleep+0x26b/frame 0xef2d2b78 kern_nanosleep(bfbfed18,ef2d2bf0,ef2d2bf8,ef2d2ce8,15180,...) at kern_nanosleep+0x14b/frame 0xef2d2bd8 sys_nanosleep(c7346960,ef2d2ca8,c7361000,c7346960,c7361000,...) at sys_nanosleep+0x69/frame 0xef2d2c18 syscall(ef2d2ce8) at syscall+0x5c9/frame 0xef2d2cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef2d2cdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28131023, esp = 0xbfbfed0c, ebp = 0xbfbfed40 --- Tracing command minicron pid 14200 tid 100209 td 0xc7346640 sched_switch(c7346640,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef36a960 mi_switch(104,0,ef36aa10,c0cf40f2,c1f86e00,c7346640) at mi_switch+0x122/frame 0xef36a998 sleepq_switch(c7346640,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef36a9c0 sleepq_catch_signals(0,c1480b5a,154,0,c7346640,...) at sleepq_catch_signals+0x5be/frame 0xef36aa14 sleepq_wait_sig(c7361308,6c,c1481b93,100,0,...) at sleepq_wait_sig+0x14/frame 0xef36aa40 _sleep(c7361308,c73613a0,16c,c1481b93,0,...) at _sleep+0x29b/frame 0xef36aa88 kern_wait6(c7346640,7,0,0,ef36ab24,...) at kern_wait6+0x71a/frame 0xef36aaf8 sys_wait4(c7346640,ef36aca8,c7361308,c7346640,c7361308,...) at sys_wait4+0x94/frame 0xef36ac18 syscall(ef36ace8) at syscall+0x5c9/frame 0xef36acdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef36acdc --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28130f43, esp = 0xbfbfed28, ebp = 0xbfbfed40 --- Tracing command minicron pid 14107 tid 100208 td 0xc7346320 sched_switch(c7346320,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef36da4c mi_switch(104,0,e35,0,e1,c7346320) at mi_switch+0x122/frame 0xef36da84 sleepq_switch(c7346320,0,c1480b5a,1a8,e1,...) at sleepq_switch+0x15b/frame 0xef36daac sleepq_catch_signals(0,e1,c0d5f550,c7346320,0,...) at sleepq_catch_signals+0x5be/frame 0xef36db00 sleepq_timedwait_sig(c1f7ac00,6c,e35,0,e1,...) at sleepq_timedwait_sig+0x14/frame 0xef36db30 _sleep(c1f7ac00,0,16c,c147d390,6dfb2057,...) at _sleep+0x26b/frame 0xef36db78 kern_nanosleep(bfbfed28,ef36dbf0,ef36dbf8,ef36dce8,e10,...) at kern_nanosleep+0x14b/frame 0xef36dbd8 sys_nanosleep(c7346320,ef36dca8,c7361610,c7346320,c7361610,...) at sys_nanosleep+0x69/frame 0xef36dc18 syscall(ef36dce8) at syscall+0x5c9/frame 0xef36dcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef36dcdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28131023, esp = 0xbfbfed1c, ebp = 0xbfbfed50 --- Tracing command minicron pid 13476 tid 100131 td 0xc6fdd000 sched_switch(c6fdd000,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef2d8960 mi_switch(104,0,ef2d8a10,c0cf40f2,c1f86e00,c6fdd000) at mi_switch+0x122/frame 0xef2d8998 sleepq_switch(c6fdd000,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef2d89c0 sleepq_catch_signals(0,c1480b5a,154,0,c6fdd000,...) at sleepq_catch_signals+0x5be/frame 0xef2d8a14 sleepq_wait_sig(c7216c20,6c,c1481b93,100,0,...) at sleepq_wait_sig+0x14/frame 0xef2d8a40 _sleep(c7216c20,c7216cb8,16c,c1481b93,0,...) at _sleep+0x29b/frame 0xef2d8a88 kern_wait6(c6fdd000,7,0,0,ef2d8b24,...) at kern_wait6+0x71a/frame 0xef2d8af8 sys_wait4(c6fdd000,ef2d8ca8,c7216c20,c6fdd000,c7216c20,...) at sys_wait4+0x94/frame 0xef2d8c18 syscall(ef2d8ce8) at syscall+0x5c9/frame 0xef2d8cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef2d8cdc --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28130f43, esp = 0xbfbfed38, ebp = 0xbfbfed50 --- Tracing command minicron pid 13234 tid 100115 td 0xc6fe2960 sched_switch(c6fe2960,0,104,30,0,...) at sched_switch+0x2d2/frame 0xef302a4c mi_switch(104,0,2f5,0,f,c6fe2960) at mi_switch+0x122/frame 0xef302a84 sleepq_switch(c6fe2960,0,c1480b5a,1a8,f,...) at sleepq_switch+0x15b/frame 0xef302aac sleepq_catch_signals(0,f,c0d5f550,c6fe2960,0,...) at sleepq_catch_signals+0x5be/frame 0xef302b00 sleepq_timedwait_sig(c1f7ac00,6c,2f5,0,f,...) at sleepq_timedwait_sig+0x14/frame 0xef302b30 _sleep(c1f7ac00,0,16c,c147d390,914c9bcb,...) at _sleep+0x26b/frame 0xef302b78 kern_nanosleep(bfbfed48,ef302bf0,ef302bf8,ef302ce8,f0,...) at kern_nanosleep+0x14b/frame 0xef302bd8 sys_nanosleep(c6fe2960,ef302ca8,fffffed8,0,0,...) at sys_nanosleep+0x69/frame 0xef302c18 syscall(ef302ce8) at syscall+0x5c9/frame 0xef302cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef302cdc --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28131023, esp = 0xbfbfed3c, ebp = 0xbfbfed70 --- Tracing command minicron pid 12890 tid 100195 td 0xc7347640 sched_switch(c7347640,0,104,ac,98,...) at sched_switch+0x2d2/frame 0xef370960 mi_switch(104,0,ef370a10,c0cf40f2,c1f86e00,c7347640) at mi_switch+0x122/frame 0xef370998 sleepq_switch(c7347640,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef3709c0 sleepq_catch_signals(0,c1480b5a,154,0,c7347640,...) at sleepq_catch_signals+0x5be/frame 0xef370a14 sleepq_wait_sig(c6dfec20,6c,c1481b93,100,0,...) at sleepq_wait_sig+0x14/frame 0xef370a40 _sleep(c6dfec20,c6dfecb8,16c,c1481b93,0,...) at _sleep+0x29b/frame 0xef370a88 kern_wait6(c7347640,7,0,0,ef370b24,...) at kern_wait6+0x71a/frame 0xef370af8 sys_wait4(c7347640,ef370ca8,c6dfec20,c7347640,c6dfec20,...) at sys_wait4+0x94/frame 0xef370c18 syscall(ef370ce8) at syscall+0x5c9/frame 0xef370cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef370cdc --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28130f43, esp = 0xbfbfed58, ebp = 0xbfbfed70 --- Tracing command syslogd pid 10469 tid 100105 td 0xc714c000 sched_switch(c714c000,0,104,100000,0,...) at sched_switch+0x2d2/frame 0xef2fa8e0 mi_switch(104,0,ef2fa990,c0cf40f2,c1f86e00,c714c000) at mi_switch+0x122/frame 0xef2fa918 sleepq_switch(c714c000,0,c1480b5a,1a8,ef2fa9c8,...) at sleepq_switch+0x15b/frame 0xef2fa940 sleepq_catch_signals(0,c1480b5a,154,ef2faba8,2,...) at sleepq_catch_signals+0x5be/frame 0xef2fa994 sleepq_wait_sig(c72be3a4,0,c14c2473,101,0,...) at sleepq_wait_sig+0x14/frame 0xef2fa9c0 _cv_wait_sig(c72be3a4,c72be390,c0bb978b,c680a000,0,...) at _cv_wait_sig+0x18c/frame 0xef2fa9f0 seltdwait(ffffffff,0,0,c714c000,ef2faba0,...) at seltdwait+0xcf/frame 0xef2faa28 kern_select(c714c000,a,2882e118,0,0,...) at kern_select+0x8c2/frame 0xef2fabd0 sys_select(c714c000,ef2faca8,eb,ef2fac64,c0d1bba2,...) at sys_select+0x69/frame 0xef2fac18 syscall(ef2face8) at syscall+0x5c9/frame 0xef2facdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef2facdc --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281ad84f, esp = 0xbfbfdde4, ebp = 0xbfbfed74 --- Tracing command sh pid 65676 tid 100171 td 0xc72ebc80 sched_switch(c72ebc80,0,104,5,0,...) at sched_switch+0x2d2/frame 0xef34e960 mi_switch(104,0,ef34ea10,c0cf40f2,c1f86e00,c72ebc80) at mi_switch+0x122/frame 0xef34e998 sleepq_switch(c72ebc80,0,c1480b5a,1a8,0,...) at sleepq_switch+0x15b/frame 0xef34e9c0 sleepq_catch_signals(0,c1480b5a,154,c72ebc80,c72ebc80,...) at sleepq_catch_signals+0x5be/frame 0xef34ea14 sleepq_wait_sig(c7216308,6c,c1481b93,100,0,...) at sleepq_wait_sig+0x14/frame 0xef34ea40 _sleep(c7216308,c72163a0,16c,c1481b93,0,...) at _sleep+0x29b/frame 0xef34ea88 kern_wait6(c72ebc80,7,0,0,ef34eb24,...) at kern_wait6+0x71a/frame 0xef34eaf8 sys_wait4(c72ebc80,ef34eca8,fffffed8,0,0,...) at sys_wait4+0x94/frame 0xef34ec18 syscall(ef34ece8) at syscall+0x5c9/frame 0xef34ecdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef34ecdc --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x281acf43, esp = 0xbfbfead0, ebp = 0xbfbfeae8 --- Tracing command charon pid 58300 tid 100167 td 0xc72f0960 sched_switch(c72f0960,0,104,ef342a28,0,...) at sched_switch+0x2d2/frame 0xef342a08 mi_switch(104,0,ef342ab8,c0cf40f2,c1f86e00,c72f0960) at mi_switch+0x122/frame 0xef342a40 sleepq_switch(c72f0960,0,c1480b5a,1a8,c72f0960,...) at sleepq_switch+0x15b/frame 0xef342a68 sleepq_catch_signals(0,c1480b5a,154,200007f,1fffff,...) at sleepq_catch_signals+0x5be/frame 0xef342abc sleepq_wait_sig(c72a6a80,0,c147d7d2,100,0,...) at sleepq_wait_sig+0x14/frame 0xef342ae8 _sleep(c72a6a80,c1f84314,300,c147d7d2,0,...) at _sleep+0x29b/frame 0xef342b30 umtxq_sleep(0,ef342bb8,0,c72a6a88,ef342c18,...) at umtxq_sleep+0x125/frame 0xef342b90 do_wait(0,0,1,1,14,...) at do_wait+0x371/frame 0xef342be8 __umtx_op_wait_uint_private(c72f0960,ef342ca8,c0d3d592,c72f0960,c7231000,...) at __umtx_op_wait_uint_private+0x9f/frame 0xef342c18 syscall(ef342ce8) at syscall+0x5c9/frame 0xef342cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef342cdc --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x281751a3, esp = 0xbaceedf4, ebp = 0xbaceee38 --- Tracing command charon pid 58300 tid 100166 td 0xc72f0c80 sched_switch(c72f0c80,0,104,ef33fa28,0,...) at sched_switch+0x2d2/frame 0xef33fa08 mi_switch(104,0,ef33fab8,c0cf40f2,c1f86e00,c72f0c80) at mi_switch+0x122/frame 0xef33fa40 sleepq_switch(c72f0c80,0,c1480b5a,1a8,c72f0c80,...) at sleepq_switch+0x15b/frame 0xef33fa68 sleepq_catch_signals(0,c1480b5a,154,200007f,1fffff,...) at sleepq_catch_signals+0x5be/frame 0xef33fabc sleepq_wait_sig(c72a6a00,0,c147d7d2,100,0,...) at sleepq_wait_sig+0x14/frame 0xef33fae8 _sleep(c72a6a00,c1f82094,300,c147d7d2,0,...) at _sleep+0x29b/frame 0xef33fb30 umtxq_sleep(0,ef33fbb8,0,c72a6a08,ef33fc18,...) at umtxq_sleep+0x125/frame 0xef33fb90 do_wait(0,0,1,1,14,...) at do_wait+0x371/frame 0xef33fbe8 __umtx_op_wait_uint_private(c72f0c80,ef33fca8,c0d3d592,c72f0c80,c7231000,...) at __umtx_op_wait_uint_private+0x9f/frame 0xef33fc18 syscall(ef33fce8) at syscall+0x5c9/frame 0xef33fcdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef33fcdc --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x281751a3, esp = 0xbadefdf4, ebp = 0xbadefe38 --- Tracing command charon pid 58300 tid 100165 td 0xc6fdc640 sched_switch(c6fdc640,0,104,ef33ca28,0,...) at sched_switch+0x2d2/frame 0xef33ca08 mi_switch(104,0,ef33cab8,c0cf40f2,c1f86e00,c6fdc640) at mi_switch+0x122/frame 0xef33ca40 sleepq_switch(c6fdc640,0,c1480b5a,1a8,10041880,...) at sleepq_switch+0x15b/frame 0xef33ca68 sleepq_catch_signals(0,c1480b5a,154,0,101,...) at sleepq_catch_signals+0x5be/frame 0xef33cabc sleepq_wait_sig(c6c9d880,0,c147d7d2,100,0,...) at sleepq_wait_sig+0x14/frame 0xef33cae8 _sleep(c6c9d880,c1f85de4,300,c147d7d2,0,...) at _sleep+0x29b/frame 0xef33cb30 umtxq_sleep(0,ef33cbb8,0,c6c9d888,ef33cbf0,...) at umtxq_sleep+0x125/frame 0xef33cb90 do_wait(0,0,1,1,14,...) at do_wait+0x371/frame 0xef33cbe8 __umtx_op_wait_uint_private(c6fdc640,ef33cca8,c0d3d592,c6fdc640,c7231000,...) at __umtx_op_wait_uint_private+0x9f/frame 0xef33cc18 syscall(ef33cce8) at syscall+0x5c9/frame 0xef33ccdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef33ccdc --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x281751a3, esp = 0xbaef0df4, ebp = 0xbaef0e38 --- Tracing command charon pid 58300 tid 100164 td 0xc6fdc960 sched_switch(c6fdc960,0,104,ef339a28,0,...) at sched_switch+0x2d2/frame 0xef339a08 mi_switch(104,0,ef339ab8,c0cf40f2,c1f86e00,c6fdc960) at mi_switch+0x122/frame 0xef339a40 sleepq_switch(c6fdc960,0,c1480b5a,1a8,c6fdc960,...) at sleepq_switch+0x15b/frame 0xef339a68 sleepq_catch_signals(0,c1480b5a,154,200007f,1fffff,...) at sleepq_catch_signals+0x5be/frame 0xef339abc sleepq_wait_sig(c6c9d680,0,c147d7d2,100,0,...) at sleepq_wait_sig+0x14/frame 0xef339ae8 _sleep(c6c9d680,c1f83b34,300,c147d7d2,0,...) at _sleep+0x29b/frame 0xef339b30 umtxq_sleep(0,ef339bb8,0,c6c9d688,ef339c18,...) at umtxq_sleep+0x125/frame 0xef339b90 do_wait(0,0,1,1,14,...) at do_wait+0x371/frame 0xef339be8 __umtx_op_wait_uint_private(c6fdc960,ef339ca8,c0d3d592,c6fdc960,c7231000,...) at __umtx_op_wait_uint_private+0x9f/frame 0xef339c18 syscall(ef339ce8) at syscall+0x5c9/frame 0xef339cdc Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xef339cdc --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x281751a3, esp = 0xbaff1df4, ebp = 0xbaff1e38 --- Tracing command charon pid 58300 tid 100163 td 0xc714c640 sched_switch(c714c640,0,104,ef336a28,0,...) at sched_switch+0x2d2/frame 0xef336a08 mi_switch(104,0,ef336ab8,c0cf40f2,c1f86e00,c714c640) at mi_switch+0x122/frame 0xef336a40 sleepq_switch(c714c640,0,c1480b5a,1a8,c714c640,...) at sleepq_switch+0x15b/frame 0xef336a68 sleepq_catch_signals(0,c1480b5a,154,200007f,1fffff,...) at sleepq_catch_signals+0x5be/frame 0xef336abc sleepq_wait_sig(c7123280,0,c147d7d2,100,0,...) at sleepq_wait_sig+0x14/frame 0xef336ae8 _sleep(c7123280,c1f818b4,300,c147d7d2,0,...) atconfig.txt0600001367012721125337 7545 ustarrootwheeloptions CONFIG_AUTOGENERATED ident pfSense machine i386 cpu I686_CPU cpu I586_CPU cpu I486_CPU makeoptions WITH_CTF=1 makeoptions DEBUG=-g options CONSPEED=115200 options MROUTING options DEVICE_POLLING options HIFN_RNDTEST options HIFN_DEBUG options MSGTQL=2048 options MSGSSZ=32 options MSGSEG=512 options MSGMNI=40 options MSGMNB=8192 options ALTQ_CODEL options ALTQ_NOPCC options ALTQ_FAIRQ options ALTQ_PRIQ options ALTQ_HFSC options ALTQ_RIO options ALTQ_RED options ALTQ_CBQ opt[/thread]
-
I haven't seen that one before, but it would appear at a glance to be crashing in handling of SCTP on IPv6. Are you actually using anything like that or allowing it through the firewall?