Random chrashes
-
Hello!
since a while our pfsense installation reboots because of a crash. I read the log but I am not sure about the error. It would be nice im somebody could help me to find the
problem. Here is what the log said:Crash report begins. Anonymous machine information: i386 8.3-RELEASE-p11 FreeBSD 8.3-RELEASE-p11 #1: Wed Sep 11 18:47:57 EDT 2013 root@snapshots-8_3-i386.builders.pfsense.org:/usr/obj.pfSense/usr/pfSensesrc/src/sys/pfSense_SMP.8 Crash report details: Filename: /var/crash/bounds 1 Filename: /var/crash/info.0 Dump header from device /dev/ad0s1b Architecture: i386 Architecture Version: 1 Dump Length: 74752B (0 MB) Blocksize: 512 Dumptime: Thu Nov 28 16:44:35 2013 Hostname: pfsense.sgf Magic: FreeBSD Text Dump Version String: FreeBSD 8.3-RELEASE-p11 #1: Wed Sep 11 18:47:57 EDT 2013 root@snapshots-8_3-i386.builders.pfsense.org:/usr/obj.pfSense/usr/pfSensesrc/src/sys/pfSense_SMP.8 Panic String: Dump Parity: 2953902428 Bounds: 0 Dump Status: good Filename: /var/crash/textdump.tar.0 ddb.txt06000014000012245662343 7101 ustarrootwheeldb:0:kdb.enter.default> run lockinfo db:1:lockinfo> show locks No such command db:1:locks> show alllocks No such command db:1:alllocks> show lockedvnods Locked vnodes db:0:kdb.enter.default> show pcpu cpuid = 1 dynamic pcpu = 0x44ae080 curthread = 0xc6287000: pid 0 "run0 taskq" curpcb = 0xc5bcdd80 fpcurthread = none idlethread = 0xc5d968a0: tid 100003 "idle: cpu1" APIC ID = 1 currentldt = 0x50 db:0:kdb.enter.default> bt Tracing pid 0 tid 100064 td 0xc6287000 run_drain_fifo(c6277000,c6287000,0,c0f9592c,c625db58,...) at run_drain_fifo+0x5f run_ratectl_cb(c6277000,1,0,0,0,...) at run_ratectl_cb+0x5f taskqueue_run_locked(c625db40,c625db58,0,c0f87c6b,0,...) at taskqueue_run_locked+0x6d taskqueue_thread_loop(c6283074,c5bcdd28,0,0,c6283074,...) at taskqueue_thread_loop+0x44 fork_exit(c0adcea0,c6283074,c5bcdd28) at fork_exit+0x87 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xc5bcdd60, ebp = 0 --- db:0:kdb.enter.default> ps pid ppid pgrp uid state wmesg wchan cmd 7242 35461 25 0 S nanslp 0xc1550844 sleep 4667 3483 4667 0 S+ ttyin 0xc5e93870 sh 3483 2427 3483 0 S+ wait 0xc83bd810 sh 3170 52268 3170 0 Ss (threaded) sshlockout_pf 100166 S nanslp 0xc1550844 sshlockout_pf 100135 S piperd 0xc62e5310 initial thread 2427 1 2427 0 Ss+ wait 0xc834d810 login 99199 98784 98784 0 S nanslp 0xc1550844 minicron 98784 1 98784 0 Ss wait 0xc68a12b0 minicron 98772 98230 98230 0 S nanslp 0xc1550844 minicron 98230 1 98230 0 Ss wait 0xc68a02b0 minicron 97973 97421 97421 0 S nanslp 0xc1550844 minicron 97421 1 97421 0 Ss wait 0xc68a0ac0 minicron 96278 1 96278 0 Ss nanslp 0xc1550844 cron 94211 93537 263 0 S bpf 0xc76d8d00 bandwidthd 93892 93537 263 0 S bpf 0xc6b8c300 bandwidthd 93623 93537 263 0 S bpf 0xc6b8cd00 bandwidthd 93537 1 263 0 S bpf 0xc76d9700 bandwidthd 87682 87300 25 0 S+ bpf 0xc76d8e00 bandwidthd 87670 87300 25 0 S+ bpf 0xc7757d00 bandwidthd 87527 87300 25 0 S+ bpf 0xc76d9100 bandwidthd 87300 1 25 0 S+ bpf 0xc7328b00 bandwidthd 86242 85563 263 0 S bpf 0xc68b4600 bandwidthd 86046 85563 263 0 S bpf 0xc834b300 bandwidthd 85823 85563 263 0 S bpf 0xc834a400 bandwidthd 85563 1 263 0 S bpf 0xc62d5400 bandwidthd 54715 1 54715 0 Ss select 0xc76d6924 ntpd 52268 1 52268 0 Ss select 0xc838c6e4 syslogd 35461 1 25 0 S+ wait 0xc679b2b0 sh 95202 1 95202 0 Ss select 0xc678d564 openvpn 85584 1 85584 0 Ss select 0xc678ee24 racoon 83751 1 83751 1002 Ss select 0xc624f8a4 dhcpd 72775 1 72587 65534 S select 0xc76d6764 dnsmasq 70778 66176 66176 0 S (threaded) php 100077 S accept 0xc7411b82 php 67212 65257 65257 0 S accept 0xc6ae9b82 php 67149 1 67149 0 Ss kqread 0xc839b900 dhcpleases 66176 64915 66176 0 Ss wait 0xc62ed2b0 initial thread 65257 64915 65257 0 Ss wait 0xc683d2b0 initial thread 64915 1 64181 0 S kqread 0xc7409100 lighttpd 34443 1 34443 0 Ss select 0xc625eb24 inetd 30632 1 25 0 S+ piperd 0xc62e6498 logger 30312 1 25 0 S+ bpf 0xc68b4700 tcpdump 23813 23601 23601 0 S piperd 0xc62eb930 rrdtool 23601 1 23601 0 Ss select 0xc625ea24 apinger 22498 1 22498 0 Ss select 0xc62a01a4 hostapd 20323 1 20323 0 Ss select 0xc62a11e4 dhcp6c 16520 1 16520 65 Ss select 0xc678dd64 dhclient 9759 1 9759 0 Ss select 0xc62a03e4 dhclient 9582 1 9582 0 Ss (threaded) sshlockout_pf 100129 S nanslp 0xc1550844 sshlockout_pf 100092 S uwait 0xc62a1480 initial thread 9265 1 9265 0 Ss select 0xc62a1164 sshd 274 1 274 0 Ss select 0xc625eca4 devd 265 263 263 0 S kqread 0xc628ae80 check_reload_status 263 1 263 0 Ss kqread 0xc628a480 check_reload_status 71 0 0 0 SL mdwait 0xc6240800 [md0] 37 0 0 0 SL (threaded) zfskern 100089 D l2arc_fe 0xc648ab04 [l2arc_feed_thread] 100088 D arc_recl 0xc647b97c [arc_reclaim_thread] 24 0 0 0 SL sdflush 0xc157f1e0 [softdepflush] 23 0 0 0 SL syncer 0xc15645b8 [syncer] 22 0 0 0 SL vlruwt 0xc628cac0 [vnlru] 21 0 0 0 SL psleep 0xc15642e8 [bufdaemon] 20 0 0 0 SL pgzero 0xc157fe94 [pagezero] 19 0 0 0 SL pollid 0xc154fd7c [idlepoll] 18 0 0 0 SL psleep 0xc157fabc [vmdaemon] 17 0 0 0 SL psleep 0xc157fa84 [pagedaemon] 16 0 0 0 SL ccb_scan 0xc1518c54 [xpt_thrd] 9 0 0 0 SL pftm 0xc050c170 [pfpurge] 8 0 0 0 SL waiting_ 0xc156b9b8 [sctp_iterator] 7 0 0 0 SL - 0xc5f9f000 [fw0_probe] 15 0 0 0 SL (threaded) usb 100050 D - 0xc5f60d34 [usbus3] 100049 D - 0xc5f60d04 [usbus3] 100048 D - 0xc5f60cd4 [usbus3] 100047 D - 0xc5f60ca4 [usbus3] 100045 D - 0xc5f55b5c [usbus2] 100044 D - 0xc5f55b2c [usbus2] 100043 D - 0xc5f55afc [usbus2] 100042 D - 0xc5f55acc [usbus2] 100040 D - 0xc5f50b5c [usbus1] 100039 D - 0xc5f50b2c [usbus1] 100038 D - 0xc5f50afc [usbus1] 100037 D - 0xc5f50acc [usbus1] 100035 D - 0xc5f42b5c [usbus0] 100034 D - 0xc5f42b2c [usbus0] 100033 D - 0xc5f42afc [usbus0] 100032 D - 0xc5f42acc [usbus0] 14 0 0 0 SL - 0xc15506a4 [yarrow] 6 0 0 0 SL crypto_r 0xc157e78c [crypto returns] 5 0 0 0 SL crypto_w 0xc157e768 [crypto] 4 0 0 0 SL - 0xc154de64 [g_down] 3 0 0 0 SL - 0xc154de60 [g_up] 2 0 0 0 SL - 0xc154de58 [g_event] 13 0 0 0 SL (threaded) ng_queue 100010 D sleep 0xc14ec5e0 [ng_queue1] 100009 D sleep 0xc14ec5e0 [ng_queue0] 12 0 0 0 WL (threaded) intr 100073 I [swi1: netisr 1] 100060 I [irq1: atkbd0] 100059 I [irq7: ppc0] 100058 I [swi0: uart] 100055 I [irq16: fwohci0] 100052 I [irq17: em1 atapci1] 100051 I [irq19: sis0 em0] 100046 I [irq23: ehci0] 100041 I [irq22: ohci2] 100036 I [irq21: ohci1] 100031 I [irq20: ohci0] 100030 I [irq15: ata1] 100029 I [irq14: ata0] 100028 I [irq9: acpi0] 100027 I [swi6: task queue] 100026 I [swi6: Giant taskq] 100024 I [swi5: +] 100021 I [swi2: cambio] 100008 I [swi3: vm] 100007 I [swi4: clock] 100006 I [swi4: clock] 100005 I [swi1: netisr 0] 11 0 0 0 RL (threaded) idle 100004 Run CPU 0 [idle: cpu0] 100003 CanRun [idle: cpu1] 1 0 1 0 SLs wait 0xc5d94ac0 [init] 10 0 0 0 SL audit_wo 0xc157eb80 [audit] 0 0 0 0 RLs (threaded) kernel 100130 D - 0xc625e740 [dummynet] 100087 D - 0xc625e280 [system_taskq_1] 100086 D - 0xc625e280 [system_taskq_0] 100064 Run CPU 1 [run0 taskq] 100056 D - 0xc5f9da80 [fw0_taskq] 100054 D - 0xc5f7bc40 [em1 taskq] 100053 D - 0xc5f7a5c0 [em0 taskq] 100025 D - 0xc5ee6180 [thread] 100023 D - 0xc5ee6240 [ffs_trim taskq] 100022 D - 0xc5ee63c0 [kqueue taskq] 100020 D - 0xc5ee6500 [acpi_task_2] 100019 D - 0xc5ee6500 [acpi_task_1] 100018 D - 0xc5ee6500 [acpi_task_0] 100014 D - 0xc5d748c0 [firmware taskq] 100000 D sched 0xc154df60 [swapper] 83248 93623 263 0 Z bandwidthd 83058 85823 263 0 Z bandwidthd 82949 87527 25 0 Z bandwidthd 82851 93537 263 0 Z bandwidthd 82575 85563 263 0 Z bandwidthd 82807 87300 25 0 Z bandwidthd 15279 86046 263 0 Z bandwidthd 14998 87670 25 0 Z bandwidthd 14674 93892 263 0 Z bandwidthd 3689 87682 25 0 Z bandwidthd 3670 86242 263 0 Z bandwidthd 3565 94211 263 0 Z bandwidthd db:0:kdb.enter.default> alltrace Tracing command sleep pid 7242 tid 100194 td 0xc83c1b80 sched_switch(c83c1b80,0,104,a4f9e12,80569,...) at sched_switch+0xbc mi_switch(104,0,0,0,eea60ba0,...) at mi_switch+0xd6 sleepq_switch(c83c1b80,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(0,0,ea61,eea60bf4,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c1550844,5c,c0f9319e,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c1550844,0,15c,c0f9319e,ea61,...) at _sleep+0x192 kern_nanosleep(c83c1b80,eea60c48,eea60c50,3c,0,...) at kern_nanosleep+0xeb nanosleep(c83c1b80,eea60cec,1,c0ac35b6,c,...) at nanosleep+0x6c syscall(eea60d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815c567, esp = 0xbfbfeddc, ebp = 0xbfbfee28 --- Tracing command sh pid 4667 tid 100169 td 0xc86aa8a0 sched_switch(c86aa8a0,0,104,6debb724,72,...) at sched_switch+0xbc mi_switch(104,0,c86aa8a0,1,eea15ab8,...) at mi_switch+0xd6 sleepq_switch(c86aa8a0,0,c0f9592c,1a0,c5f80b80,...) at sleepq_switch+0xae sleepq_catch_signals(c86aa8a0,1,eea15afc,c0a54147,c5e93870,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c5e93870,0,c0f97f32,101,0,...) at sleepq_wait_sig+0x18 _cv_wait_sig(c5e93870,c154fad0,0,0,eea15b68,...) at _cv_wait_sig+0x127 tty_wait(c5e93800,c5e93870,1,eea15b5b,eea15b50,...) at tty_wait+0x23 ttydisc_read(c5e93800,eea15c3c,0,c8673324,eea15bbc,...) at ttydisc_read+0xa5 ttydev_read(c5f77000,eea15c3c,0,0,1,...) at ttydev_read+0x10f devfs_read_f(c8365118,eea15c3c,c8324d80,0,c86aa8a0,...) at devfs_read_f+0x7e dofileread(eea15c3c,ffffffff,ffffffff,0,c8365118,...) at dofileread+0x90 kern_readv(c86aa8a0,0,eea15c3c,eea15c5c,1,...) at kern_readv+0x6a read(c86aa8a0,eea15cec,2,c0ac35b6,c,...) at read+0x50 syscall(eea15d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x281f2e73, esp = 0xbfbfe97c, ebp = 0xbfbfea58 --- Tracing command sh pid 3483 tid 100151 td 0xc83c18a0 sched_switch(c83c18a0,0,104,baf05df4,71,...) at sched_switch+0xbc mi_switch(104,0,c83bd898,0,ee9dfb60,...) at mi_switch+0xd6 sleepq_switch(c83c18a0,0,c0f9592c,1a0,c62ef780,...) at sleepq_switch+0xae sleepq_catch_signals(c83bd898,0,ee9dfbb0,c0aa816a,c83bd810,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c83bd810,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c83bd810,c83bd898,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c83c18a0,ffffffff,ee9dfc58,2,0,...) at kern_wait+0x6ef wait4(c83c18a0,ee9dfcec,2,c0ac35ec,c,...) at wait4+0x3b syscall(ee9dfd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2816e53b, esp = 0xbfbfea8c, ebp = 0xbfbfeaa8 --- Tracing command sshlockout_pf pid 3170 tid 100166 td 0xc86ab2e0 sched_switch(c86ab2e0,0,104,2cd1acbe,80063,...) at sched_switch+0xbc mi_switch(104,0,0,0,eea0cba0,...) at mi_switch+0xd6 sleepq_switch(c86ab2e0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(0,0,1b7741,eea0cbf4,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c1550844,5c,c0f9319e,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c1550844,0,15c,c0f9319e,1b7741,...) at _sleep+0x192 kern_nanosleep(c86ab2e0,eea0cc48,eea0cc50,708,0,...) at kern_nanosleep+0xeb nanosleep(c86ab2e0,eea0ccec,2,0,c,...) at nanosleep+0x6c syscall(eea0cd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28172567, esp = 0xbf9fef4c, ebp = 0xbf9fef68 --- Tracing command sshlockout_pf pid 3170 tid 100135 td 0xc83525c0 sched_switch(c83525c0,0,104,3053935a,6cee0,...) at sched_switch+0xbc mi_switch(104,0,c62e5480,0,ee997b40,...) at mi_switch+0xd6 sleepq_switch(c83525c0,0,c0f9592c,1a0,c5d7f240,...) at sleepq_switch+0xae sleepq_catch_signals(c62e5480,0,ee997b90,c0aa816a,c62e5310,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c62e5310,4c,c0f96015,100,0,...) at sleepq_wait_sig+0x18 _sleep(c62e5310,c62e5480,14c,c0f96015,0,...) at _sleep+0x27a pipe_read(c771d8f8,ee997c3c,c5d91400,0,c83525c0,...) at pipe_read+0x44a dofileread(ee997c3c,ffffffff,ffffffff,0,c771d8f8,...) at dofileread+0x90 kern_readv(c83525c0,0,ee997c3c,ee997c5c,1,...) at kern_readv+0x6a read(c83525c0,ee997cec,2,c0ac35ec,c,...) at read+0x50 syscall(ee997d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x2818ee73, esp = 0xbfbfe8ec, ebp = 0xbfbfe908 --- Tracing command login pid 2427 tid 100132 td 0xc67a58a0 sched_switch(c67a58a0,0,104,97e81cc8,71,...) at sched_switch+0xbc mi_switch(104,0,c834d898,0,ee98bb60,...) at mi_switch+0xd6 sleepq_switch(c67a58a0,0,c0f9592c,1a0,c5d7f1c0,...) at sleepq_switch+0xae sleepq_catch_signals(c834d898,0,ee98bbb0,c0aa816a,c834d810,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c834d810,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c834d810,c834d898,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c67a58a0,d9b,ee98bc58,0,0,...) at kern_wait+0x6ef wait4(c67a58a0,ee98bcec,2,c0ac35ec,c,...) at wait4+0x3b syscall(ee98bd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2810e53b, esp = 0xbfbfed3c, ebp = 0xbfbfed58 --- Tracing command minicron pid 99199 tid 100157 td 0xc83c05c0 sched_switch(c83c05c0,0,104,deb868c,7e60f,...) at sched_switch+0xbc mi_switch(104,0,0,0,ee9f1ba0,...) at mi_switch+0xd6 sleepq_switch(c83c05c0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(0,0,5265c01,ee9f1bf4,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c1550844,5c,c0f9319e,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c1550844,0,15c,c0f9319e,5265c01,...) at _sleep+0x192 kern_nanosleep(c83c05c0,ee9f1c48,ee9f1c50,15180,0,...) at kern_nanosleep+0xeb nanosleep(c83c05c0,ee9f1cec,2,0,c,...) at nanosleep+0x6c syscall(ee9f1d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815c567, esp = 0xbfbfed2c, ebp = 0xbfbfed58 --- Tracing command minicron pid 98784 tid 100144 td 0xc83508a0 sched_switch(c83508a0,0,104,6b609934,6f,...) at sched_switch+0xbc mi_switch(104,0,c68a1338,0,ee9b3b60,...) at mi_switch+0xd6 sleepq_switch(c83508a0,0,c0f9592c,1a0,c62efd80,...) at sleepq_switch+0xae sleepq_catch_signals(c68a1338,0,ee9b3bb0,c0aa816a,c68a12b0,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c68a12b0,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c68a12b0,c68a1338,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c83508a0,ffffffff,ee9b3c58,0,0,...) at kern_wait+0x6ef wait4(c83508a0,ee9b3cec,2,c0ac35ec,c,...) at wait4+0x3b syscall(ee9b3d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f453b, esp = 0xbfbfed3c, ebp = 0xbfbfed58 --- Tracing command minicron pid 98772 tid 100140 td 0xc83515c0 sched_switch(c83515c0,0,104,7cc8812c,7fb39,...) at sched_switch+0xbc mi_switch(104,0,0,0,ee9a7ba0,...) at mi_switch+0xd6 sleepq_switch(c83515c0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(0,0,36ee81,ee9a7bf4,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c1550844,5c,c0f9319e,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c1550844,0,15c,c0f9319e,36ee81,...) at _sleep+0x192 kern_nanosleep(c83515c0,ee9a7c48,ee9a7c50,e10,0,...) at kern_nanosleep+0xeb nanosleep(c83515c0,ee9a7cec,2,c0ac35b6,c,...) at nanosleep+0x6c syscall(ee9a7d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815c567, esp = 0xbfbfed3c, ebp = 0xbfbfed68 --- Tracing command minicron pid 98230 tid 100120 td 0xc68a4b80 sched_switch(c68a4b80,0,104,6acd9170,6f,...) at sched_switch+0xbc mi_switch(104,0,c68a0338,0,ee92fb60,...) at mi_switch+0xd6 sleepq_switch(c68a4b80,0,c0f9592c,1a0,c62efb80,...) at sleepq_switch+0xae sleepq_catch_signals(c68a0338,0,ee92fbb0,c0aa816a,c68a02b0,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c68a02b0,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c68a02b0,c68a0338,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c68a4b80,ffffffff,ee92fc58,0,0,...) at kern_wait+0x6ef wait4(c68a4b80,ee92fcec,2,c0ac35b6,c,...) at wait4+0x3b syscall(ee92fd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f453b, esp = 0xbfbfed4c, ebp = 0xbfbfed68 --- Tracing command minicron pid 97973 tid 100150 td 0xc68a65c0 sched_switch(c68a65c0,0,104,51c846f4,80529,...) at sched_switch+0xbc mi_switch(104,0,0,0,ee9dcba0,...) at mi_switch+0xd6 sleepq_switch(c68a65c0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(0,0,3a981,ee9dcbf4,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c1550844,5c,c0f9319e,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c1550844,0,15c,c0f9319e,3a981,...) at _sleep+0x192 kern_nanosleep(c68a65c0,ee9dcc48,ee9dcc50,f0,0,...) at kern_nanosleep+0xeb nanosleep(c68a65c0,ee9dccec,2,c0ac35b6,c,...) at nanosleep+0x6c syscall(ee9dcd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2815c567, esp = 0xbfbfed3c, ebp = 0xbfbfed68 --- Tracing command minicron pid 97421 tid 100117 td 0xc67a1000 sched_switch(c67a1000,0,104,6a601b80,6f,...) at sched_switch+0xbc mi_switch(104,0,c68a0b48,0,ee923b60,...) at mi_switch+0xd6 sleepq_switch(c67a1000,0,c0f9592c,1a0,c5d7f640,...) at sleepq_switch+0xae sleepq_catch_signals(c68a0b48,0,ee923bb0,c0aa816a,c68a0ac0,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c68a0ac0,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c68a0ac0,c68a0b48,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c67a1000,ffffffff,ee923c58,0,0,...) at kern_wait+0x6ef wait4(c67a1000,ee923cec,2,c0ac35ec,c,...) at wait4+0x3b syscall(ee923d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x280f453b, esp = 0xbfbfed4c, ebp = 0xbfbfed68 --- Tracing command cron pid 96278 tid 100142 td 0xc8351000 sched_switch(c8351000,0,104,f233d21e,8054f,...) at sched_switch+0xbc mi_switch(104,0,0,0,ee9adba0,...) at mi_switch+0xd6 sleepq_switch(c8351000,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(0,0,ea61,ee9adbf4,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c1550844,5c,c0f9319e,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c1550844,0,15c,c0f9319e,ea61,...) at _sleep+0x192 kern_nanosleep(c8351000,ee9adc48,ee9adc50,3c,0,...) at kern_nanosleep+0xeb nanosleep(c8351000,ee9adcec,2,14,c,...) at nanosleep+0x6c syscall(ee9add28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x2817a567, esp = 0xbfbfec6c, ebp = 0xbfbfec98 --- Tracing command bandwidthd pid 94211 tid 100085 td 0xc62ee8a0 sched_switch(c62ee8a0,0,104,b0fd2626,80569,...) at sched_switch+0xbc mi_switch(104,0,c76d8d84,0,ee8abb10,...) at mi_switch+0xd6 sleepq_switch(c62ee8a0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c76d8d84,0,3e8,ee8abb64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c76d8d00,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c76d8d00,c76d8d84,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee8abc3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c62df9a0,ee8abc3c,c5d91400,0,c62ee8a0,...) at devfs_read_f+0x7e dofileread(ee8abc3c,ffffffff,ffffffff,0,c62df9a0,...) at dofileread+0x90 kern_readv(c62ee8a0,14,ee8abc3c,ee8abc5c,1,...) at kern_readv+0x6a read(c62ee8a0,ee8abcec,14,1e,c62eeb4c,...) at read+0x50 syscall(ee8abd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 93892 tid 100102 td 0xc67a5000 sched_switch(c67a5000,0,104,d4a31516,80569,...) at sched_switch+0xbc mi_switch(104,0,c6b8c384,0,ee8f3b10,...) at mi_switch+0xd6 sleepq_switch(c67a5000,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c6b8c384,0,3e8,ee8f3b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c6b8c300,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c6b8c300,c6b8c384,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee8f3c3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c62d77e0,ee8f3c3c,c5d91400,0,c67a5000,...) at devfs_read_f+0x7e dofileread(ee8f3c3c,ffffffff,ffffffff,0,c62d77e0,...) at dofileread+0x90 kern_readv(c67a5000,14,ee8f3c3c,ee8f3c5c,1,...) at kern_readv+0x6a read(c67a5000,ee8f3cec,ee8f3cc0,c0ac35b6,c155482c,...) at read+0x50 syscall(ee8f3d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 93623 tid 100084 td 0xc62eeb80 sched_switch(c62eeb80,0,104,d4d40372,80569,...) at sched_switch+0xbc mi_switch(104,0,c6b8cd84,0,ee8a7b10,...) at mi_switch+0xd6 sleepq_switch(c62eeb80,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c6b8cd84,0,3e8,ee8a7b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c6b8cd00,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c6b8cd00,c6b8cd84,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee8a7c3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c8365428,ee8a7c3c,c5d91400,0,c62eeb80,...) at devfs_read_f+0x7e dofileread(ee8a7c3c,ffffffff,ffffffff,0,c8365428,...) at dofileread+0x90 kern_readv(c62eeb80,14,ee8a7c3c,ee8a7c5c,1,...) at kern_readv+0x6a read(c62eeb80,ee8a7cec,2,c0ac35b6,c,...) at read+0x50 syscall(ee8a7d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 93537 tid 100149 td 0xc68a68a0 sched_switch(c68a68a0,0,104,daf61d9e,80569,...) at sched_switch+0xbc mi_switch(104,0,c76d9784,0,ee9d9b10,...) at mi_switch+0xd6 sleepq_switch(c68a68a0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c76d9784,0,3e8,ee9d9b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c76d9700,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c76d9700,c76d9784,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee9d9c3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c62d83b8,ee9d9c3c,c5d91400,0,c68a68a0,...) at devfs_read_f+0x7e dofileread(ee9d9c3c,ffffffff,ffffffff,0,c62d83b8,...) at dofileread+0x90 kern_readv(c68a68a0,14,ee9d9c3c,ee9d9c5c,1,...) at kern_readv+0x6a read(c68a68a0,ee9d9cec,2,c0ac35ec,c,...) at read+0x50 syscall(ee9d9d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 87682 tid 100133 td 0xc67a55c0 sched_switch(c67a55c0,0,104,95641db4,80569,...) at sched_switch+0xbc mi_switch(104,0,c76d8e84,0,ee98fb10,...) at mi_switch+0xd6 sleepq_switch(c67a55c0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c76d8e84,0,3e8,ee98fb64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c76d8e00,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c76d8e00,c76d8e84,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee98fc3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c62df8f8,ee98fc3c,c5d91400,0,c67a55c0,...) at devfs_read_f+0x7e dofileread(ee98fc3c,ffffffff,ffffffff,0,c62df8f8,...) at dofileread+0x90 kern_readv(c67a55c0,a,ee98fc3c,ee98fc5c,1,...) at kern_readv+0x6a read(c67a55c0,ee98fcec,46,c0ac35ec,c67a55c0,...) at read+0x50 syscall(ee98fd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 87670 tid 100114 td 0xc67a18a0 sched_switch(c67a18a0,0,104,fa33907a,80569,...) at sched_switch+0xbc mi_switch(104,0,c7757d84,0,ee91ab10,...) at mi_switch+0xd6 sleepq_switch(c67a18a0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c7757d84,0,3e8,ee91ab64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c7757d00,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c7757d00,c7757d84,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee91ac3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c62d71c0,ee91ac3c,c5d91400,0,c67a18a0,...) at devfs_read_f+0x7e dofileread(ee91ac3c,ffffffff,ffffffff,0,c62d71c0,...) at dofileread+0x90 kern_readv(c67a18a0,a,ee91ac3c,ee91ac5c,1,...) at kern_readv+0x6a read(c67a18a0,ee91acec,ee91acc0,c0ac35b6,c155482c,...) at read+0x50 syscall(ee91ad28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 87527 tid 100147 td 0xc8350000 sched_switch(c8350000,0,104,8db99056,80569,...) at sched_switch+0xbc mi_switch(104,0,c76d9184,0,ee9d3b10,...) at mi_switch+0xd6 sleepq_switch(c8350000,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c76d9184,0,3e8,ee9d3b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c76d9100,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c76d9100,c76d9184,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee9d3c3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c83cf2a0,ee9d3c3c,c5d91400,0,c8350000,...) at devfs_read_f+0x7e dofileread(ee9d3c3c,ffffffff,ffffffff,0,c83cf2a0,...) at dofileread+0x90 kern_readv(c8350000,a,ee9d3c3c,ee9d3c5c,1,...) at kern_readv+0x6a read(c8350000,ee9d3cec,2,c0ac35b6,c,...) at read+0x50 syscall(ee9d3d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 87300 tid 100094 td 0xc63105c0 sched_switch(c63105c0,0,104,8424159e,80569,...) at sched_switch+0xbc mi_switch(104,0,c7328b84,0,ee8cfb10,...) at mi_switch+0xd6 sleepq_switch(c63105c0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c7328b84,0,3e8,ee8cfb64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c7328b00,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c7328b00,c7328b84,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee8cfc3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c8365e38,ee8cfc3c,c5d91400,0,c63105c0,...) at devfs_read_f+0x7e dofileread(ee8cfc3c,ffffffff,ffffffff,0,c8365e38,...) at dofileread+0x90 kern_readv(c63105c0,a,ee8cfc3c,ee8cfc5c,1,...) at kern_readv+0x6a read(c63105c0,ee8cfcec,2,c0ac35b6,c,...) at read+0x50 syscall(ee8cfd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 86242 tid 100100 td 0xc630f2e0 sched_switch(c630f2e0,0,104,9a90d8ea,80569,...) at sched_switch+0xbc mi_switch(104,0,c68b4684,0,ee8ecb10,...) at mi_switch+0xd6 sleepq_switch(c630f2e0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c68b4684,0,3e8,ee8ecb64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c68b4600,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c68b4600,c68b4684,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee8ecc3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c7412968,ee8ecc3c,c5d91400,0,c630f2e0,...) at devfs_read_f+0x7e dofileread(ee8ecc3c,ffffffff,ffffffff,0,c7412968,...) at dofileread+0x90 kern_readv(c630f2e0,14,ee8ecc3c,ee8ecc5c,1,...) at kern_readv+0x6a read(c630f2e0,ee8eccec,46,c0ac35ec,c630f2e0,...) at read+0x50 syscall(ee8ecd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 86046 tid 100146 td 0xc83502e0 sched_switch(c83502e0,0,104,9594f574,80569,...) at sched_switch+0xbc mi_switch(104,0,c834b384,0,ee9d0b10,...) at mi_switch+0xd6 sleepq_switch(c83502e0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c834b384,0,3e8,ee9d0b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c834b300,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c834b300,c834b384,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee9d0c3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c8365d90,ee9d0c3c,c5d91400,0,c83502e0,...) at devfs_read_f+0x7e dofileread(ee9d0c3c,ffffffff,ffffffff,0,c8365d90,...) at dofileread+0x90 kern_readv(c83502e0,14,ee9d0c3c,ee9d0c5c,1,...) at kern_readv+0x6a read(c83502e0,ee9d0cec,46,c0ac35ec,c83502e0,...) at read+0x50 syscall(ee9d0d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 85823 tid 100158 td 0xc83c02e0 sched_switch(c83c02e0,0,104,b4a18982,80569,...) at sched_switch+0xbc mi_switch(104,0,c834a484,0,ee9f4b10,...) at mi_switch+0xd6 sleepq_switch(c83c02e0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c834a484,0,3e8,ee9f4b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c834a400,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c834a400,c834a484,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee9f4c3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c8365c40,ee9f4c3c,c5d91400,0,c83c02e0,...) at devfs_read_f+0x7e dofileread(ee9f4c3c,ffffffff,ffffffff,0,c8365c40,...) at dofileread+0x90 kern_readv(c83c02e0,14,ee9f4c3c,ee9f4c5c,1,...) at kern_readv+0x6a read(c83c02e0,ee9f4cec,2,c0ac35b6,c,...) at read+0x50 syscall(ee9f4d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command bandwidthd pid 85563 tid 100108 td 0xc67a3b80 sched_switch(c67a3b80,0,104,8113a5aa,80569,...) at sched_switch+0xbc mi_switch(104,0,c62d5484,0,ee908b10,...) at mi_switch+0xd6 sleepq_switch(c67a3b80,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c62d5484,0,3e8,ee908b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c62d5400,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c62d5400,c62d5484,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee908c3c,0,0,80000,...) at bpfread+0x258 devfs_read_f(c62d8150,ee908c3c,c5d91400,0,c67a3b80,...) at devfs_read_f+0x7e dofileread(ee908c3c,ffffffff,ffffffff,0,c62d8150,...) at dofileread+0x90 kern_readv(c67a3b80,14,ee908c3c,ee908c5c,1,...) at kern_readv+0x6a read(c67a3b80,ee908cec,2,c0ac35b6,c,...) at read+0x50 syscall(ee908d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28353e73, esp = 0xbfbfeaec, ebp = 0xbfbfebe8 --- Tracing command ntpd pid 54715 tid 100143 td 0xc8350b80 sched_switch(c8350b80,0,104,78edbf2a,80569,...) at sched_switch+0xbc mi_switch(104,0,c76d6900,0,ee9b0a1c,...) at mi_switch+0xd6 sleepq_switch(c8350b80,0,c0f9592c,1a0,c62efc80,...) at sleepq_switch+0xae sleepq_catch_signals(c76d6900,0,ee9b0a60,c0a54147,c76d6924,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c76d6924,0,c0f9725c,101,0,...) at sleepq_wait_sig+0x18 _cv_wait_sig(c76d6924,c76d6910,ee9b0a8c,c0ae1570,c624c380,...) at _cv_wait_sig+0x127 seltdwait(c682d80c,58,c5d91400,c8350b80,0,...) at seltdwait+0x132 kern_select(c8350b80,25,bfbfed0c,0,0,0,20,c68a1560,0) at kern_select+0x61e select(c8350b80,ee9b0cec,0,e,0,...) at select+0x63 syscall(ee9b0d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, select), eip = 0x283d2df3, esp = 0xbfbfecec, ebp = 0xbfbfeda8 --- Tracing command syslogd pid 52268 tid 100076 td 0xc62f1b80 sched_switch(c62f1b80,0,104,623ff918,80569,...) at sched_switch+0xbc mi_switch(104,0,c838c6c0,0,c5bf4a1c,...) at mi_switch+0xd6 sleepq_switch(c62f1b80,0,c0f9592c,1a0,c62ef040,...) at sleepq_switch+0xae sleepq_catch_signals(c838c6c0,0,c5bf4a60,c0a54147,c838c6e4,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c838c6e4,0,c0f9725c,101,0,...) at sleepq_wait_sig+0x18 _cv_wait_sig(c838c6e4,c838c6d0,c62f1b80,0,c14e0e80,...) at _cv_wait_sig+0x127 seltdwait(c7412d58,58,c5d91400,c62f1b80,c5bf4ac4,...) at seltdwait+0x132 kern_select(c62f1b80,f,2822c098,0,0,0,20,c62edac0,0) at kern_select+0x61e select(c62f1b80,c5bf4cec,c5bf4cc0,c0ac35b6,c155482c,...) at select+0x63 syscall(c5bf4d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, select), eip = 0x2818fdf3, esp = 0xbfbfe24c, ebp = 0xbfbfedb8 --- Tracing command sh pid 35461 tid 100097 td 0xc630fb80 sched_switch(c630fb80,0,104,a1fc2a6,80569,...) at sched_switch+0xbc mi_switch(104,0,c679b338,0,ee8dbb60,...) at mi_switch+0xd6 sleepq_switch(c630fb80,0,c0f9592c,1a0,c5d7f140,...) at sleepq_switch+0xae sleepq_catch_signals(c679b338,0,ee8dbbb0,c0aa816a,c679b2b0,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c679b2b0,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c679b2b0,c679b338,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c630fb80,ffffffff,ee8dbc58,2,0,...) at kern_wait+0x6ef wait4(c630fb80,ee8dbcec,2,c0ac35b6,c,...) at wait4+0x3b syscall(ee8dbd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2816e53b, esp = 0xbfbfeafc, ebp = 0xbfbfeb18 --- Tracing command openvpn pid 95202 tid 100136 td 0xc83522e0 sched_switch(c83522e0,0,104,24ce6e1e,80568,...) at sched_switch+0xbc mi_switch(104,0,c678d540,0,ee99ba78,...) at mi_switch+0xd6 sleepq_switch(c83522e0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c678d540,0,c83522e0,ee99bac0,c0a53c79,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c678d564,0,c0f9725c,101,0,...) at sleepq_timedwait_sig+0x1c _cv_timedwait_sig(c678d564,c678d550,2711,c0ae1570,c624c380,...) at _cv_timedwait_sig+0x139 seltdwait(ee99bc40,ee99bc48,c5d91400,c83522e0,c0ae8ea0,...) at seltdwait+0x84 poll(c83522e0,ee99bcec,ee99bcc0,c0ac35b6,c155482c,...) at poll+0x47f syscall(ee99bd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (209, FreeBSD ELF32, poll), eip = 0x28397097, esp = 0xbfbfdacc, ebp = 0xbfbfdb08 --- Tracing command racoon pid 85584 tid 100113 td 0xc67a1b80 sched_switch(c67a1b80,0,104,622bf18c,80569,...) at sched_switch+0xbc mi_switch(104,0,c678ee00,0,ee917a18,...) at mi_switch+0xd6 sleepq_switch(c67a1b80,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c678ee00,0,c67a1b80,ee917a60,c0a53c79,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c678ee24,0,c0f9725c,101,0,...) at sleepq_timedwait_sig+0x1c _cv_timedwait_sig(c678ee24,c678ee10,41f0,c0ae1570,c624c380,...) at _cv_timedwait_sig+0x139 seltdwait(ee917c0c,ee917c14,c5d91400,c67a1b80,0,...) at seltdwait+0x84 kern_select(c67a1b80,14,80c1d40,0,0,ee917c54,20,10,d685f) at kern_select+0x61e select(c67a1b80,ee917cec,ee917cc0,c0ac35b6,c155482c,...) at select+0x63 syscall(ee917d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, select), eip = 0x283e1df3, esp = 0xbfbfe95c, ebp = 0xbfbfeda8 --- Tracing command dhcpd pid 83751 tid 100112 td 0xc67a3000 sched_switch(c67a3000,0,104,144867f4,8056a,...) at sched_switch+0xbc mi_switch(104,0,c624f880,0,ee914a18,...) at mi_switch+0xd6 sleepq_switch(c67a3000,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c624f880,0,c67a3000,ee914a60,c0a53c79,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c624f8a4,0,c0f9725c,101,0,...) at sleepq_timedwait_sig+0x1c _cv_timedwait_sig(c624f8a4,c624f890,19b386,c0ae1570,c624c380,...) at _cv_timedwait_sig+0x139 seltdwait(ee914c0c,ee914c14,c5d91400,c67a3000,c69d4400,...) at seltdwait+0x84 kern_select(c67a3000,16,2845b080,2845b180,0,ee914c54,20,694,56eb9) at kern_select+0x61e select(c67a3000,ee914cec,46,c0ac35ec,c67a3000,...) at select+0x63 syscall(ee914d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, select), eip = 0x28335df3, esp = 0xbfbfeb7c, ebp = 0xbfbfeba8 --- Tracing command dnsmasq pid 72775 tid 100128 td 0xc68a32e0 sched_switch(c68a32e0,0,104,d2e83084,80567,...) at sched_switch+0xbc mi_switch(104,0,c76d6740,0,ee94da1c,...) at mi_switch+0xd6 sleepq_switch(c68a32e0,0,c0f9592c,1a0,c5d7f100,...) at sleepq_switch+0xae sleepq_catch_signals(c76d6740,0,ee94da60,c0a54147,c76d6764,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c76d6764,0,c0f9725c,101,0,...) at sleepq_wait_sig+0x18 _cv_wait_sig(c76d6764,c76d6750,ee94da8c,c0ae1570,0,...) at _cv_wait_sig+0x127 seltdwait(c62d8690,58,c7409380,c68a32e0,c1d8c778,...) at seltdwait+0x132 kern_select(c68a32e0,8,bfbfec60,bfbfebe0,bfbfeb60,0,20,c683e810,bfbfeb60) at kern_select+0x61e select(c68a32e0,ee94dcec,ee94dcc0,c0ac35b6,c1554eac,...) at select+0x63 syscall(ee94dd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, select), eip = 0x281dddf3, esp = 0xbfbfeb0c, ebp = 0xbfbfed38 --- Tracing command php pid 70778 tid 100077 td 0xc62f18a0 sched_switch(c62f18a0,0,104,9d1ed2d0,6d64d,...) at sched_switch+0xbc mi_switch(104,0,c1564130,0,c5bf8b9c,...) at mi_switch+0xd6 sleepq_switch(c62f18a0,0,c0f9592c,1a0,c5d7fb00,...) at sleepq_switch+0xae sleepq_catch_signals(c1564130,0,c5bf8bec,c0aa816a,c7411b82,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c7411b82,58,c0f963b7,100,0,...) at sleepq_wait_sig+0x18 _sleep(c7411b82,c1564130,158,c0f963b7,0,...) at _sleep+0x27a kern_accept(c62f18a0,0,c5bf8c54,c5bf8c50,c5bf8c4c,...) at kern_accept+0x183 accept(c62f18a0,c5bf8cec,c5bf8cc0,c0ac35b6,c1554a34,...) at accept+0x73 syscall(c5bf8d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (30, FreeBSD ELF32, accept), eip = 0x286a7c5f, esp = 0xbfbeca9c, ebp = 0xbfbecab8 --- Tracing command php pid 67212 tid 100079 td 0xc6311b80 sched_switch(c6311b80,0,104,a0c6e00e,6d64d,...) at sched_switch+0xbc mi_switch(104,0,c1564130,0,ee895b9c,...) at mi_switch+0xd6 sleepq_switch(c6311b80,0,c0f9592c,1a0,c5f80e00,...) at sleepq_switch+0xae sleepq_catch_signals(c1564130,0,ee895bec,c0aa816a,c6ae9b82,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c6ae9b82,58,c0f963b7,100,0,...) at sleepq_wait_sig+0x18 _sleep(c6ae9b82,c1564130,158,c0f963b7,0,...) at _sleep+0x27a kern_accept(c6311b80,0,ee895c54,ee895c50,ee895c4c,...) at kern_accept+0x183 accept(c6311b80,ee895cec,ee895cc0,c0ac35b6,c1554a34,...) at accept+0x73 syscall(ee895d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (30, FreeBSD ELF32, accept), eip = 0x286a7c5f, esp = 0xbfbeca9c, ebp = 0xbfbecab8 --- Tracing command dhcpleases pid 67149 tid 100090 td 0xc62ee5c0 sched_switch(c62ee5c0,0,104,8b5ee788,80284,...) at sched_switch+0xbc mi_switch(104,0,c839b900,0,ee8bca84,...) at mi_switch+0xd6 sleepq_switch(c62ee5c0,0,c0f9592c,1a0,c62ef0c0,...) at sleepq_switch+0xae sleepq_catch_signals(c839b900,0,ee8bcad4,c0aa816a,c839b900,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c839b900,58,c0f8ea33,100,0,...) at sleepq_wait_sig+0x18 _sleep(c839b900,c839b900,158,c0f8ea33,0,...) at _sleep+0x27a kern_kevent(c62ee5c0,71,0,1,ee8bcc3c,...) at kern_kevent+0x325 kevent(c62ee5c0,ee8bccec,ee8bccc0,c0ac35b6,c155482c,...) at kevent+0x88 syscall(ee8bcd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (363, FreeBSD ELF32, kevent), eip = 0x2812dbf7, esp = 0xbfbfecec, ebp = 0xbfbfed68 --- Tracing command php pid 66176 tid 100081 td 0xc62f12e0 sched_switch(c62f12e0,0,104,d341cd64,5b,...) at sched_switch+0xbc mi_switch(104,0,c62ed338,0,ee89bb60,...) at mi_switch+0xd6 sleepq_switch(c62f12e0,0,c0f9592c,1a0,c5f80e80,...) at sleepq_switch+0xae sleepq_catch_signals(c62ed338,0,ee89bbb0,c0aa816a,c62ed2b0,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c62ed2b0,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c62ed2b0,c62ed338,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c62f12e0,ffffffff,ee89bc58,0,0,...) at kern_wait+0x6ef wait4(c62f12e0,ee89bcec,2,c5d91400,c,...) at wait4+0x3b syscall(ee89bd28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2868753b, esp = 0xbfbfcb5c, ebp = 0xbfbfcb78 --- Tracing command php pid 65257 tid 100110 td 0xc67a35c0 sched_switch(c67a35c0,0,104,9b2bef30,5b,...) at sched_switch+0xbc mi_switch(104,0,c683d338,0,ee90eb60,...) at mi_switch+0xd6 sleepq_switch(c67a35c0,0,c0f9592c,1a0,c5d7f480,...) at sleepq_switch+0xae sleepq_catch_signals(c683d338,0,ee90ebb0,c0aa816a,c683d2b0,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c683d2b0,5c,c0f96347,100,0,...) at sleepq_wait_sig+0x18 _sleep(c683d2b0,c683d338,15c,c0f96347,0,...) at _sleep+0x27a kern_wait(c67a35c0,ffffffff,ee90ec58,0,0,...) at kern_wait+0x6ef wait4(c67a35c0,ee90ecec,2,c0ac35ec,c,...) at wait4+0x3b syscall(ee90ed28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (7, FreeBSD ELF32, wait4), eip = 0x2868753b, esp = 0xbfbfcb5c, ebp = 0xbfbfcb78 --- Tracing command lighttpd pid 64915 tid 100116 td 0xc67a12e0 sched_switch(c67a12e0,0,104,ad8a7ba6,80569,...) at sched_switch+0xbc mi_switch(104,0,c7409100,0,ee920a80,...) at mi_switch+0xd6 sleepq_switch(c67a12e0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c7409100,0,3e9,ee920ad4,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c7409100,58,c0f8ea33,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c7409100,c7409100,158,c0f8ea33,3e9,...) at _sleep+0x192 kern_kevent(c67a12e0,8,0,2b58,ee920c3c,...) at kern_kevent+0x325 kevent(c67a12e0,ee920cec,200046,c0ac35ec,c67a12e0,...) at kevent+0x88 syscall(ee920d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (363, FreeBSD ELF32, kevent), eip = 0x283a4bf7, esp = 0xbfbfea9c, ebp = 0xbfbfead8 --- Tracing command inetd pid 34443 tid 100123 td 0xc68a42e0 sched_switch(c68a42e0,0,104,b816da40,802db,...) at sched_switch+0xbc mi_switch(104,0,c625eb00,0,ee939a1c,...) at mi_switch+0xd6 sleepq_switch(c68a42e0,0,c0f9592c,1a0,c68a5200,...) at sleepq_switch+0xae sleepq_catch_signals(c625eb00,0,ee939a60,c0a54147,c625eb24,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c625eb24,0,c0f9725c,101,0,...) at sleepq_wait_sig+0x18 _cv_wait_sig(c625eb24,c625eb10,ee939a8c,c0ae1570,c624c380,...) at _cv_wait_sig+0x127 seltdwait(c6ae9670,58,c5d91400,c68a42e0,c925f594,...) at seltdwait+0x132 kern_select(c68a42e0,c,bfbfdf90,0,0,0,20,c689f810,bfbfe0b4) at kern_select+0x61e select(c68a42e0,ee939cec,0,1,0,...) at select+0x63 syscall(ee939d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (93, FreeBSD ELF32, select), eip = 0x2819fdf3, esp = 0xbfbfdf4c, ebp = 0xbfbfedd8 --- Tracing command logger pid 30632 tid 100127 td 0xc68a35c0 sched_switch(c68a35c0,0,104,edd0b6ee,80568,...) at sched_switch+0xbc mi_switch(104,0,c62e6608,0,ee94ab40,...) at mi_switch+0xd6 sleepq_switch(c68a35c0,0,c0f9592c,1a0,c5f80740,...) at sleepq_switch+0xae sleepq_catch_signals(c62e6608,0,ee94ab90,c0aa816a,c62e6498,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c62e6498,4c,c0f96015,100,0,...) at sleepq_wait_sig+0x18 _sleep(c62e6498,c62e6608,14c,c0f96015,0,...) at _sleep+0x27a pipe_read(c62df770,ee94ac3c,c5d91400,0,c68a35c0,...) at pipe_read+0x44a dofileread(ee94ac3c,ffffffff,ffffffff,0,c62df770,...) at dofileread+0x90 kern_readv(c68a35c0,0,ee94ac3c,ee94ac5c,1,...) at kern_readv+0x6a read(c68a35c0,ee94acec,ee94acc0,c0ac35b6,c1554eac,...) at read+0x50 syscall(ee94ad28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28179e73, esp = 0xbfbfe91c, ebp = 0xbfbfe938 --- Tracing command tcpdump pid 30312 tid 100098 td 0xc630f8a0 sched_switch(c630f8a0,0,104,ad6eac16,80569,...) at sched_switch+0xbc mi_switch(104,0,c68b4784,0,ee8e4b10,...) at mi_switch+0xd6 sleepq_switch(c630f8a0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c68b4784,0,3e8,ee8e4b64,c0aa8082,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c68b4700,1a,c0fa46e1,100,0,...) at sleepq_timedwait_sig+0x1c _sleep(c68b4700,c68b4784,11a,c0fa46e1,3e8,...) at _sleep+0x192 bpfread(c5d83a00,ee8e4c3c,0,0,8000,...) at bpfread+0x258 devfs_read_f(c62d8af0,ee8e4c3c,c624bc00,0,c630f8a0,...) at devfs_read_f+0x7e dofileread(ee8e4c3c,ffffffff,ffffffff,0,c62d8af0,...) at dofileread+0x90 kern_readv(c630f8a0,8,ee8e4c3c,ee8e4c5c,1,...) at kern_readv+0x6a read(c630f8a0,ee8e4cec,2,c0ac35b6,c,...) at read+0x50 syscall(ee8e4d28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x28392e73, esp = 0xbfbfeafc, ebp = 0xbfbfebf8 --- Tracing command rrdtool pid 23813 tid 100093 td 0xc62ee000 sched_switch(c62ee000,0,104,a7511cee,8055e,...) at sched_switch+0xbc mi_switch(104,0,c62ebaa0,0,ee8cab40,...) at mi_switch+0xd6 sleepq_switch(c62ee000,0,c0f9592c,1a0,c62ef140,...) at sleepq_switch+0xae sleepq_catch_signals(c62ebaa0,0,ee8cab90,c0aa816a,c62eb930,...) at sleepq_catch_signals+0x33b sleepq_wait_sig(c62eb930,4c,c0f96015,100,0,...) at sleepq_wait_sig+0x18 _sleep(c62eb930,c62ebaa0,14c,c0f96015,0,...) at _sleep+0x27a pipe_read(c62d85e8,ee8cac3c,c6a2b880,0,c62ee000,...) at pipe_read+0x44a dofileread(ee8cac3c,ffffffff,ffffffff,0,c62d85e8,...) at dofileread+0x90 kern_readv(c62ee000,0,ee8cac3c,ee8cac5c,1,...) at kern_readv+0x6a read(c62ee000,ee8cacec,ee8cacc0,c0ac35b6,c155482c,...) at read+0x50 syscall(ee8cad28) at syscall+0x1fb Xint0x80_syscall() at Xint0x80_syscall+0x21 --- syscall (3, FreeBSD ELF32, read), eip = 0x282bbe73, esp = 0xbfbfec9c, ebp = 0xbfbfecb8 --- Tracing command apinger pid 23601 tid 100122 td 0xc68a45c0 sched_switch(c68a45c0,0,104,15797ab4,8056a,...) at sched_switch+0xbc mi_switch(104,0,c625ea00,0,ee935a78,...) at mi_switch+0xd6 sleepq_switch(c68a45c0,0,c0f9592c,1a0,c0ab2e4a,...) at sleepq_switch+0xae sleepq_catch_signals(c625ea00,0,c68a45c0,ee935ac0,c0a53c79,...) at sleepq_catch_signals+0x33b sleepq_timedwait_sig(c625ea24,0,c0f9725c,101,0,...) at sleepq_timedwait_sig+0x1c _cv_timedwait_sig(c62[/thread] -
Not sure if this is the case but I had random crashes when I upgraded to 2.1. I fixed it by doing a backup, doing a fresh install instead of the upgrade and restoring the backup. No crashes since so if you did an upgrade to 2.1 i'd suggest doing a fresh install.