[RESOLVED] 2.2.1 to 2.2.2 upgraded & working but crash reports every now & then



  • i upgraded one full install box from 2.2.1 to 2.2.2 and it works fine also but every few days i keep getting crash report

    Crash report begins.  Anonymous machine information:
    
    i386
    10.1-RELEASE-p9
    FreeBSD 10.1-RELEASE-p9 #0 57b23e7(releng/10.1)-dirty: Mon Apr 13 20:33:39 CDT 2015     root@pfs22-i386-builder:/usr/obj.i386/usr/pfSensesrc/src/sys/pfSense_SMP.10
    
    Crash report details:
    
    Filename: /var/crash/bounds
    2
    
    Filename: /var/crash/info.0
    Dump header from device /dev/ada0s1b
      Architecture: i386
      Architecture Version: 1
      Dump Length: 72192B (0 MB)
      Blocksize: 512
      Dumptime: Wed Apr 22 19:03:15 2015
      Hostname: na
      Magic: FreeBSD Text Dump
      Version String: FreeBSD 10.1-RELEASE-p9 #0 57b23e7(releng/10.1)-dirty: Mon Apr 13 20:33:39 CDT 2015
        root@pfs22-i386-builder:/usr/obj.i386/usr/pfSensesrc/src/sys/pfSense_SMP.10
      Panic String: 
      Dump Parity: 1921864290
      Bounds: 0
      Dump Status: good
    
    Filename: /var/crash/info.1
    Dump header from device /dev/ada0s1b
      Architecture: i386
      Architecture Version: 1
      Dump Length: 74240B (0 MB)
      Blocksize: 512
      Dumptime: Wed Apr 22 19:07:00 2015
      Hostname: na
      Magic: FreeBSD Text Dump
      Version String: FreeBSD 10.1-RELEASE-p9 #0 57b23e7(releng/10.1)-dirty: Mon Apr 13 20:33:39 CDT 2015
        root@pfs22-i386-builder:/usr/obj.i386/usr/pfSensesrc/src/sys/pfSense_SMP.10
      Panic String: sleeping thread
      Dump Parity: 3853664804
      Bounds: 1
      Dump Status: good
    
    Filename: /var/crash/info.last
    Dump header from device /dev/ada0s1b
      Architecture: i386
      Architecture Version: 1
      Dump Length: 74240B (0 MB)
      Blocksize: 512
      Dumptime: Wed Apr 22 19:07:00 2015
      Hostname: na
      Magic: FreeBSD Text Dump
      Version String: FreeBSD 10.1-RELEASE-p9 #0 57b23e7(releng/10.1)-dirty: Mon Apr 13 20:33:39 CDT 2015
        root@pfs22-i386-builder:/usr/obj.i386/usr/pfSensesrc/src/sys/pfSense_SMP.10
      Panic String: sleeping thread
      Dump Parity: 3853664804
      Bounds: 1
      Dump Status: good
    
    Filename: /var/crash/minfree
    2048
    
    Filename: /var/crash/textdump.tar.0
    ddb.txt���������������������������������������������������������������������������������������������0600����0�������0�������140000������12515721433�  7075� �����������������������������������������������������������������������������������������������������ustar���root����������������������������wheel������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������db:0:kdb.enter.default>  run lockinfo
    db:1:lockinfo> show locks
    No such command
    db:1:locks>  show alllocks
    No such command
    db:1:alllocks>  show lockedvnods
    Locked vnodes
    db:0:kdb.enter.default>  show pcpu
    cpuid        = 0
    dynamic pcpu = 0x5dc880
    curthread    = 0xc786d310: pid 12 "irq257: re0"
    curpcb       = 0xef3fdd60
    fpcurthread  = none
    idlethread   = 0xc7700000: tid 100003 "idle: cpu0"
    APIC ID      = 0
    currentldt   = 0x50
    db:0:kdb.enter.default>  bt
    Tracing pid 12 tid 100029 td 0xc786d310
    pppoe_findsession(c82968c0,0,c82c0000,0,0,...) at pppoe_findsession+0x90/frame 0xef3fd8a8
    ng_pppoe_rcvdata_ether(c82c0100,c8296880,0,1000001,14,...) at ng_pppoe_rcvdata_ether+0x2bc/frame 0xef3fd924
    ng_apply_item(0,c7e86c00,c80ee500,0,ef3fd9b4,...) at ng_apply_item+0x22d/frame 0xef3fd980
    ng_snd_item(c8296880,0,c82c0180,0,c7e86c00,...) at ng_snd_item+0x1a0/frame 0xef3fd9b4
    ng_ether_input_orphan(c7e86c00,c80ee500,5ae39d77,3296e054,5535e223,...) at ng_ether_input_orphan+0x66/frame 0xef3fd9d4
    ether_demux(c7e86c00,c80ee500,6,0,c12b5e9e,...) at ether_demux+0x1f9/frame 0xef3fda00
    ether_nh_input(c80ee500,0,0,0,0,...) at ether_nh_input+0x37e/frame 0xef3fda50
    netisr_dispatch_src(9,0,c80ee500) at netisr_dispatch_src+0x8b/frame 0xef3fda98
    netisr_dispatch(9,c80ee500) at netisr_dispatch+0x20/frame 0xef3fdaac
    ether_input(c7e86c00,c80ee500,0,32d6dbc0,c8147700,...) at ether_input+0x19/frame 0xef3fdabc
    vlan_input(c789bc00,c80ee500,c8438fc8,ef3fdb40,c102a2dd,...) at vlan_input+0x1a8/frame 0xef3fdaec
    ether_demux(c789bc00,c80ee500,6,c78cf240,ef3fdbb0,...) at ether_demux+0xaf/frame 0xef3fdb18
    ether_nh_input(c80ee500,c789cdc4,c789c000,c7cfa100,ef3fdbd4,...) at ether_nh_input+0x37e/frame 0xef3fdb68
    netisr_dispatch_src(9,0,c80ee500) at netisr_dispatch_src+0x8b/frame 0xef3fdbb0
    netisr_dispatch(9,c80ee500) at netisr_dispatch+0x20/frame 0xef3fdbc4
    ether_input(c789bc00,c80ee500,c0cda523,c7700000,0,...) at ether_input+0x19/frame 0xef3fdbd4
    re_rxeof(c1f14900,c7700000,2710,0,0,...) at re_rxeof+0x48b/frame 0xef3fdc28
    re_intr_msi(c789c000,0,246,0,11d12d51,...) at re_intr_msi+0x107/frame 0xef3fdc5c
    intr_event_execute_handlers(109,c76fed00,c14323d3,55a,0,...) at intr_event_execute_handlers+0xaa/frame 0xef3fdc88
    ithread_loop(c78a92d0,ef3fdd08,0,0,0,...) at ithread_loop+0x80/frame 0xef3fdcc4
    fork_exit(c0cbe9e0,c78a92d0,ef3fdd08) at fork_exit+0xa3/frame 0xef3fdcf4
    fork_trampoline() at fork_trampoline+0x8/frame 0xef3fdcf4
    --- trap 0, eip = 0, esp = 0xef3fdd40, ebp = 0 ---
    db:0:kdb.enter.default>  ps
      pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
    89778     1 89778     0  Ss      (threaded)          ntpd
    100220                   S       usem     0xc8527980 ntpd
    100152                   S       select   0xc868d0e4 ntpd
    87545     1 87545     0  Ss      nanslp   0xc1f086f8 cron
    81244     1 81244  1002  Ss      select   0xc868ff64 dhcpd
    75219     1 75219    59  Ss      (threaded)          unbound
    100219                   S       kqread   0xc8940880 unbound
    100078                   S       kqread   0xc8942100 unbound
    68745   246   246     0  S       accept   0xc80361e6 php-fpm
    52565     1 52565     0  Ss      select   0xc8783264 openvpn
    55493 55424 55424     0  S       piperd   0xc8869198 rrdtool
    55424     1 55424     0  Ss      select   0xc87820a4 apinger
    22815     1 22495     0  S       kqread   0xc8453380 lighttpd
    69871 69850 69871     0  S+      ttyin    0xc77df470 sh
    69850 69601 69850     0  S+      wait     0xc85febc0 sh
    69847 60173 69847     0  Ss      (threaded)          sshlockout_pf
    100139                   S       nanslp   0xc1f086f9 sshlockout_pf
    100115                   S       piperd   0xc82fb000 sshlockout_pf
    69601     1 69601     0  Ss+     wait     0xc85ff000 login
    68298 67787 67787     0  S       nanslp   0xc1f086f9 minicron
    67787     1 67787     0  Ss      wait     0xc850c000 minicron
    67718 67455 67455     0  S       nanslp   0xc1f086f8 minicron
    67455     1 67455     0  Ss      wait     0xc850c5e0 minicron
    67290 66962 66962     0  S       nanslp   0xc1f086f9 minicron
    66962     1 66962     0  Ss      wait     0xc810c8d0 minicron
    60173     1 60173     0  Ss      select   0xc80d8624 syslogd
    56017     1 56017     0  Ss      select   0xc7e6df64 powerd
    29837     1 29837     0  Ss      select   0xc80d77a4 inetd
    29163     1 29163     0  Ss      bpf      0xc7976a00 filterlog
    27182     1 27182     0  Ss      kqread   0xc7e92580 choparp
    23101     1 23101     0  Ss      (threaded)          mpd5
    100120                   S       select   0xc80d7f24 mpd5
    18713     1 18713     0  Ss      (threaded)          mpd5
    100119                   S       select   0xc7e6c9a4 mpd5
    14790     1 14790     0  Ss      (threaded)          mpd5
    100118                   S       select   0xc7e6cb24 mpd5
     7906     1  7906     0  Ss      (threaded)          sshlockout_pf
    100113                   S       nanslp   0xc1f086f9 sshlockout_pf
    100105                   S       uwait    0xc7e6fd80 sshlockout_pf
     7822     1  7822     0  Ss      select   0xc80d8164 sshd
     6907     1  6907     0  Ss      (threaded)          mpd5
    100112                   S       select   0xc80d85a4 mpd5
      274     1   274     0  Ss      select   0xc7ddab24 devd
      264   262   262     0  S       kqread   0xc7e94880 check_reload_status
      262     1   262     0  Ss      kqread   0xc7e70c00 check_reload_status
      246     1   246     0  Ss      kqread   0xc7975d00 php-fpm
       63     0     0     0  DL      mdwait   0xc7dfb000 [md0]
       20     0     0     0  DL      syncer   0xc1f27c44 [syncer]
       19     0     0     0  DL      vlruwt   0xc7e10000 [vnlru]
       18     0     0     0  DL      psleep   0xc1f27384 [bufdaemon]
       17     0     0     0  DL      pollid   0xc1f070b8 [idlepoll]
        9     0     0     0  DL      pgzero   0xc2052ea0 [pagezero]
        8     0     0     0  DL      psleep   0xc2052bc4 [vmdaemon]
        7     0     0     0  DL      psleep   0xc20b2404 [pagedaemon]
        6     0     0     0  DL      waiting_ 0xc20acd0c [sctp_iterator]
        5     0     0     0  DL      pftm     0xc0f6af60 [pf purge]
       16     0     0     0  DL      (threaded)          [usb]
    100053                   D       -        0xc793bd34 [usbus4]
    100052                   D       -        0xc793bd04 [usbus4]
    100051                   D       -        0xc793bcd4 [usbus4]
    100050                   D       -        0xc793bca4 [usbus4]
    100049                   D       -        0xc791fdac [usbus3]
    100048                   D       -        0xc791fd7c [usbus3]
    100047                   D       -        0xc791fd4c [usbus3]
    100046                   D       -        0xc791fd1c [usbus3]
    100044                   D       -        0xc7900dac [usbus2]
    100043                   D       -        0xc7900d7c [usbus2]
    100042                   D       -        0xc7900d4c [usbus2]
    100041                   D       -        0xc7900d1c [usbus2]
    100039                   D       -        0xc78ecdac [usbus1]
    100038                   D       -        0xc78ecd7c [usbus1]
    100037                   D       -        0xc78ecd4c [usbus1]
    100036                   D       -        0xc78ecd1c [usbus1]
    100034                   D       -        0xc78addac [usbus0]
    100033                   D       -        0xc78add7c [usbus0]
    100032                   D       -        0xc78add4c [usbus0]
    100031                   D       -        0xc78add1c [usbus0]
        4     0     0     0  DL      (threaded)          [cam]
    100065                   D       -        0xc1e49628 [scanner]
    100024                   D       -        0xc1e49780 [doneq0]
        3     0     0     0  DL      crypto_r 0xc2051af8 [crypto returns]
        2     0     0     0  DL      crypto_w 0xc2051a38 [crypto]
       15     0     0     0  DL      -        0xc1e63808 [rand_harvestq]
       14     0     0     0  DL      (threaded)          [geom]
    100013                   D       -        0xc20a8d60 [g_down]
    100012                   D       -        0xc20a8d5c [g_up]
    100011                   D       -        0xc20a8d58 [g_event]
       13     0     0     0  DL      (threaded)          [ng_queue]
    100010                   D       sleep    0xc1e20048 [ng_queue1]
    100009                   D       sleep    0xc1e20048 [ng_queue0]
       12     0     0     0  RL      (threaded)          [intr]
    100073                   I                           [swi1: netisr 1]
    100063                   I                           [swi1: pfsync]
    100061                   I                           [swi1: pf send]
    100058                   I                           [irq1: atkbd0]
    100057                   I                           [irq7: ppc0]
    100056                   I                           [swi0: uart]
    100055                   I                           [irq15: ata1]
    100054                   I                           [irq14: ata0]
    100045                   I                           [irq16: uhci3]
    100040                   I                           [irq18: uhci2]
    100035                   I                           [irq19: uhci1]
    100030                   I                           [irq23: uhci0 ehci0]
    100029                   Run     CPU 0               [irq257: re0]
    100028                   I                           [irq256: hdac0]
    100027                   I                           [swi6: Giant taskq]
    100025                   I                           [swi6: task queue]
    100019                   I                           [swi5: fast taskq]
    100008                   I                           [swi3: vm]
    100007                   I                           [swi4: clock]
    100006                   I                           [swi4: clock]
    100005                   I                           [swi1: netisr 0]
       11     0     0     0  RL      (threaded)          [idle]
    100004                   Run     CPU 1               [idle: cpu1]
    100003                   CanRun                      [idle: cpu0]
        1     0     1     0  SLs     wait     0xc76f92f0 [init]
       10     0     0     0  DL      audit_wo 0xc20b0d08 [audit]
        0     0     0     0  DLs     (threaded)          [kernel]
    100064                   D       -        0xc76fb100 [CAM taskq]
    100059                   D       -        0xc7975080 [mca taskq]
    100026                   D       -        0xc75d9c00 [ffs_trim taskq]
    100023                   D       -        0xc76fb180 [acpi_task_2]
    100022                   D       -        0xc76fb180 [acpi_task_1]
    100021                   D       -        0xc76fb180 [acpi_task_0]
    100020                   D       -        0xc76fb200 [kqueue taskq]
    100018                   D       -        0xc76fb380 [thread]
    100014                   D       -        0xc76fc480 [firmware taskq]
    100000                   D       swapin   0xc20a8de4 [swapper]
    db:0:kdb.enter.default>  alltrace
    
    Tracing command ntpd pid 89778 tid 100220 td 0xc860f930
    sched_switch(c860f930,0,104,0,0,...) at sched_switch+0x2da/frame 0xf1982a40
    mi_switch(104,0,f1982af4,c0cda1e4,f1982c10,c860f930) at mi_switch+0x122/frame 0xf1982a78
    sleepq_switch(c860f930,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf1982aa0
    sleepq_catch_signals(c860f930,0,c143ae9e,154,0,...) at sleepq_catch_signals+0x5be/frame 0xf1982af4
    sleepq_wait_sig(c8527980,0,c1437bb4,100,0,...) at sleepq_wait_sig+0x14/frame 0xf1982b20
    _sleep(c8527980,c1f125ec,300,c1437bb4,0,...) at _sleep+0x29b/frame 0xf1982b68
    umtxq_sleep(0,0,1,c8527988,c1e14cf4,...) at umtxq_sleep+0x125/frame 0xf1982bc8
    __umtx_op_sem_wait(c860f930,f1982cc8,2,c788f064,c788f000,...) at __umtx_op_sem_wait+0x559/frame 0xf1982c40
    syscall(f1982d08) at syscall+0x48b/frame 0xf1982cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf1982cfc
    --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x28361efb, esp = 0xbf9fdeec, ebp = 0xbf9fdf38 ---
    
    Tracing command ntpd pid 89778 tid 100152 td 0xc868b620
    sched_switch(c868b620,0,104,28c03080,0,...) at sched_switch+0x2da/frame 0xf1908908
    mi_switch(104,0,f19089bc,c0cda1e4,80a4eba,c868b620) at mi_switch+0x122/frame 0xf1908940
    sleepq_switch(c868b620,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf1908968
    sleepq_catch_signals(c868b620,0,c143ae9e,154,2,...) at sleepq_catch_signals+0x5be/frame 0xf19089bc
    sleepq_wait_sig(c868d0e4,0,c147b5b6,101,0,...) at sleepq_wait_sig+0x14/frame 0xf19089e8
    _cv_wait_sig(c868d0e4,c868d0d0,c851b848,2b,c868b620,...) at _cv_wait_sig+0x17c/frame 0xf1908a18
    seltdwait(ffffffff,0,0,c868b620,f1908bc8,...) at seltdwait+0xcf/frame 0xf1908a50
    kern_select(c868b620,2c,bfbfddd8,0,0,...) at kern_select+0x8d2/frame 0xf1908bf8
    sys_select(c868b620,f1908cc8,0,0,0,...) at sys_select+0x69/frame 0xf1908c40
    syscall(f1908d08) at syscall+0x48b/frame 0xf1908cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf1908cfc
    --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x2842c5d7, esp = 0xbfbfdd34, ebp = 0xbfbfdd58 ---
    
    Tracing command cron pid 87545 tid 100094 td 0xc7ff2310
    sched_switch(c7ff2310,0,104,0,ebfaba98,...) at sched_switch+0x2da/frame 0xebfaba74
    mi_switch(104,0,c0d07bfa,3ebee53e,1c124,c7ff2310) at mi_switch+0x122/frame 0xebfabaac
    sleepq_switch(c7ff2310,0,c143ae9e,1a8,1c124,...) at sleepq_switch+0x15b/frame 0xebfabad4
    sleepq_catch_signals(bff9c4f6,3,c0d38a10,c7ff2310,0,...) at sleepq_catch_signals+0x5be/frame 0xebfabb28
    sleepq_timedwait_sig(c1f086f8,6c,1c124,bff9c4f6,3,...) at sleepq_timedwait_sig+0x14/frame 0xebfabb58
    _sleep(c1f086f8,0,16c,c14377d8,3ebee53e,...) at _sleep+0x26b/frame 0xebfabba0
    kern_nanosleep(bfbfede8,ebfabc18,ebfabc20,0,3b,...) at kern_nanosleep+0x14a/frame 0xebfabc00
    sys_nanosleep(c7ff2310,ebfabcc8,2,c788f064,c788f000,...) at sys_nanosleep+0x69/frame 0xebfabc40
    syscall(ebfabd08) at syscall+0x48b/frame 0xebfabcfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfabcfc
    --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x281b6c73, esp = 0xbfbfec74, ebp = 0xbfbfee04 ---
    
    Tracing command dhcpd pid 81244 tid 100147 td 0xc8618c40
    sched_switch(c8618c40,0,104,c8618c40,0,...) at sched_switch+0x2da/frame 0xf18f3900
    mi_switch(104,0,1984b013,1c782,d3f3fa6d,c8618c40) at mi_switch+0x122/frame 0xf18f3938
    sleepq_switch(c8618c40,0,c143ae9e,1a8,68,...) at sleepq_switch+0x15b/frame 0xf18f3960
    sleepq_catch_signals(d3f3fa6d,68,c0d38a10,c8618c40,1,...) at sleepq_catch_signals+0x5be/frame 0xf18f39b4
    sleepq_timedwait_sig(c868ff64,0,1c782,d3f3fa6d,68,...) at sleepq_timedwait_sig+0x14/frame 0xf18f39e4
    _cv_timedwait_sig_sbt(c868ff64,c868ff50,1984b013,1c782,d3f3fa6d,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xf18f3a18
    seltdwait(1c782,d3f3fa6d,68,c8618c40,f18f3bc8,...) at seltdwait+0xc1/frame 0xf18f3a50
    kern_select(c8618c40,16,2885d080,2885d180,0,...) at kern_select+0x8d2/frame 0xf18f3bf8
    sys_select(c8618c40,f18f3cc8,c8618c40,3,f18f3cc8,...) at sys_select+0x69/frame 0xf18f3c40
    syscall(f18f3d08) at syscall+0x48b/frame 0xf18f3cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18f3cfc
    --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x283915d7, esp = 0xbfbfec24, ebp = 0xbfbfec48 ---
    
    Tracing command unbound pid 75219 tid 100219 td 0xc860fc40
    sched_switch(c860fc40,0,104,0,0,...) at sched_switch+0x2da/frame 0xf1974948
    mi_switch(104,0,f19749fc,c0cda1e4,f1974a38,c860fc40) at mi_switch+0x122/frame 0xf1974980
    sleepq_switch(c860fc40,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf19749a8
    sleepq_catch_signals(c860fc40,0,c143ae9e,154,c84bf000,...) at sleepq_catch_signals+0x5be/frame 0xf19749fc
    sleepq_wait_sig(c8940880,68,c1431da4,100,0,...) at sleepq_wait_sig+0x14/frame 0xf1974a28
    _sleep(c8940880,c8940880,168,c1431da4,0,...) at _sleep+0x29b/frame 0xf1974a70
    kern_kevent(c860fc40,1a,0,40,f1974c10,...) at kern_kevent+0x48f/frame 0xf1974bc0
    sys_kevent(c860fc40,f1974cc8,c0cfa212,c860fc40,0,...) at sys_kevent+0x131/frame 0xf1974c40
    syscall(f1974d08) at syscall+0x48b/frame 0xf1974cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf1974cfc
    --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x28456773, esp = 0xbf9fde3c, ebp = 0xbf9fdea0 ---
    
    Tracing command unbound pid 75219 tid 100078 td 0xc7e16000
    sched_switch(c7e16000,0,104,c8206c00,0,...) at sched_switch+0x2da/frame 0xebf6e948
    mi_switch(104,0,ebf6e9fc,c0cda1e4,ebf6ea38,c7e16000) at mi_switch+0x122/frame 0xebf6e980
    sleepq_switch(c7e16000,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebf6e9a8
    sleepq_catch_signals(c7e16000,0,c143ae9e,154,c84bf000,...) at sleepq_catch_signals+0x5be/frame 0xebf6e9fc
    sleepq_wait_sig(c8942100,68,c1431da4,100,0,...) at sleepq_wait_sig+0x14/frame 0xebf6ea28
    _sleep(c8942100,c8942100,168,c1431da4,0,...) at _sleep+0x29b/frame 0xebf6ea70
    kern_kevent(c7e16000,19,0,40,ebf6ec10,...) at kern_kevent+0x48f/frame 0xebf6ebc0
    sys_kevent(c7e16000,ebf6ecc8,c0cfa212,c1f15000,c7e16000,...) at sys_kevent+0x131/frame 0xebf6ec40
    syscall(ebf6ed08) at syscall+0x48b/frame 0xebf6ecfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebf6ecfc
    --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x28456773, esp = 0xbfbfebc4, ebp = 0xbfbfec28 ---
    
    Tracing command php-fpm pid 68745 tid 100097 td 0xc7ff1930
    sched_switch(c7ff1930,0,104,ac,98,...) at sched_switch+0x2da/frame 0xebfb4a48
    mi_switch(104,0,ebfb4afc,c0cda1e4,ebfb4af0,c7ff1930) at mi_switch+0x122/frame 0xebfb4a80
    sleepq_switch(c7ff1930,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebfb4aa8
    sleepq_catch_signals(c7ff1930,0,c143ae9e,154,c0ca7b5a,...) at sleepq_catch_signals+0x5be/frame 0xebfb4afc
    sleepq_wait_sig(c80361e6,68,c143be07,100,0,...) at sleepq_wait_sig+0x14/frame 0xebfb4b28
    _sleep(c80361e6,c20ac4d4,168,c143be07,0,...) at _sleep+0x29b/frame 0xebfb4b70
    kern_accept4(c7ff1930,0,ebfb4c18,ebfb4c14,1,...) at kern_accept4+0x1dd/frame 0xebfb4bec
    accept1(bfbeca98,bfbeca94,1) at accept1+0x63/frame 0xebfb4c28
    sys_accept(c7ff1930,ebfb4cc8,c0cfa212,c7ff1930,0,...) at sys_accept+0x29/frame 0xebfb4c40
    syscall(ebfb4d08) at syscall+0x48b/frame 0xebfb4cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfb4cfc
    --- syscall (30, FreeBSD ELF32, sys_accept), eip = 0x28679cff, esp = 0xbfbeca4c, ebp = 0xbfbfcb48 ---
    
    Tracing command openvpn pid 52565 tid 100124 td 0xc8314620
    sched_switch(c8314620,0,104,c81d6dec,0,...) at sched_switch+0x2da/frame 0xf18a8970
    mi_switch(104,0,89a18d9e,1c0fe,8039577c,c8314620) at mi_switch+0x122/frame 0xf18a89a8
    sleepq_switch(c8314620,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18a89d0
    sleepq_catch_signals(8039577c,0,c0d38a10,c8314620,1,...) at sleepq_catch_signals+0x5be/frame 0xf18a8a24
    sleepq_timedwait_sig(c8783264,0,1c0fe,8039577c,0,...) at sleepq_timedwait_sig+0x14/frame 0xf18a8a54
    _cv_timedwait_sig_sbt(c8783264,c8783250,89a18d9e,1c0fe,8039577c,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xf18a8a88
    seltdwait(1c0fe,8039577c,0,c8314620,0,...) at seltdwait+0xc1/frame 0xf18a8ac0
    sys_poll(c8314620,f18a8cc8,c0cfa212,c8314620,0,...) at sys_poll+0x478/frame 0xf18a8c40
    syscall(f18a8d08) at syscall+0x48b/frame 0xf18a8cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18a8cfc
    --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x283e068f, esp = 0xbfbfe430, ebp = 0xbfbfe470 ---
    
    Tracing command rrdtool pid 55493 tid 100076 td 0xc7e16620
    sched_switch(c7e16620,0,104,0,c82aa470,...) at sched_switch+0x2da/frame 0xebf67a04
    mi_switch(104,0,c0cda1e4,2,9af8,c7e16620) at mi_switch+0x122/frame 0xebf67a3c
    sleepq_switch(c7e16620,0,c143ae9e,1a8,ffffffff,...) at sleepq_switch+0x15b/frame 0xebf67a64
    sleepq_catch_signals(c7e16620,0,c143ae9e,154,0,...) at sleepq_catch_signals+0x5be/frame 0xebf67ab8
    sleepq_wait_sig(c8869198,5c,c143b906,100,0,...) at sleepq_wait_sig+0x14/frame 0xebf67ae4
    _sleep(c8869198,c8869318,15c,c143b906,0,...) at _sleep+0x29b/frame 0xebf67b2c
    pipe_read(c7eb8d20,ebf67c10,c8453400,0,c7e16620,...) at pipe_read+0x31a/frame 0xebf67b70
    dofileread(c7eb8d20,ebf67c10,ffffffff,ffffffff,0,...) at dofileread+0x9e/frame 0xebf67ba8
    kern_readv(c7e16620,0,ebf67c10,28803000,28806000,...) at kern_readv+0x96/frame 0xebf67bf0
    sys_read(c7e16620,ebf67cc8,c0cfa212,c7e16620,32d75528,...) at sys_read+0x5c/frame 0xebf67c40
    syscall(ebf67d08) at syscall+0x48b/frame 0xebf67cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebf67cfc
    --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28315617, esp = 0xbfbfec9c, ebp = 0xbfbfecb0 ---
    
    Tracing command apinger pid 55424 tid 100143 td 0xc85e4000
    sched_switch(c85e4000,0,104,c7e79258,0,...) at sched_switch+0x2da/frame 0xf18ea970
    mi_switch(104,0,e4435d07,1c0f7,2fba5dfe,c85e4000) at mi_switch+0x122/frame 0xf18ea9a8
    sleepq_switch(c85e4000,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18ea9d0
    sleepq_catch_signals(2fba5dfe,0,c0d38a10,c85e4000,0,...) at sleepq_catch_signals+0x5be/frame 0xf18eaa24
    sleepq_timedwait_sig(c87820a4,0,1c0f7,2fba5dfe,0,...) at sleepq_timedwait_sig+0x14/frame 0xf18eaa54
    _cv_timedwait_sig_sbt(c87820a4,c8782090,e4435d07,1c0f7,2fba5dfe,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xf18eaa88
    seltdwait(1c0f7,2fba5dfe,0,c85e4000,0,...) at seltdwait+0xc1/frame 0xf18eaac0
    sys_poll(c85e4000,f18eacc8,2,c788f064,c788f000,...) at sys_poll+0x478/frame 0xf18eac40
    syscall(f18ead08) at syscall+0x48b/frame 0xf18eacfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18eacfc
    --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2815a68f, esp = 0xbfbfcd34, ebp = 0xbfbfed90 ---
    
    Tracing command lighttpd pid 22815 tid 100136 td 0xc85e5310
    sched_switch(c85e5310,0,104,c0d0867a,1,...) at sched_switch+0x2da/frame 0xf18d4944
    mi_switch(104,0,c0d07bfa,d461a5bd,1c0f7,c85e5310) at mi_switch+0x122/frame 0xf18d497c
    sleepq_switch(c85e5310,0,c143ae9e,1a8,1c0f7,...) at sleepq_switch+0x15b/frame 0xf18d49a4
    sleepq_catch_signals(10000000,0,c0d38a10,c85e5310,0,...) at sleepq_catch_signals+0x5be/frame 0xf18d49f8
    sleepq_timedwait_sig(c8453380,68,1c0f7,10000000,0,...) at sleepq_timedwait_sig+0x14/frame 0xf18d4a28
    _sleep(c8453380,c8453380,168,c1431da4,d461a5bd,...) at _sleep+0x26b/frame 0xf18d4a70
    kern_kevent(c85e5310,9,0,e19a,f18d4c10,...) at kern_kevent+0x48f/frame 0xf18d4bc0
    sys_kevent(c85e5310,f18d4cc8,c1f15000,200046,f18d4cc4,...) at sys_kevent+0x131/frame 0xf18d4c40
    syscall(f18d4d08) at syscall+0x48b/frame 0xf18d4cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18d4cfc
    --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x283e7773, esp = 0xbfbfec9c, ebp = 0xbfbfecc8 ---
    
    Tracing command sh pid 69871 tid 100141 td 0xc85e4620
    sched_switch(c85e4620,0,104,ac,98,...) at sched_switch+0x2da/frame 0xf18e39b4
    mi_switch(104,0,c0cda1e4,c77df428,0,c85e4620) at mi_switch+0x122/frame 0xf18e39ec
    sleepq_switch(c85e4620,0,c143ae9e,1a8,20202020,...) at sleepq_switch+0x15b/frame 0xf18e3a14
    sleepq_catch_signals(c85e4620,0,c143ae9e,154,c85e4620,...) at sleepq_catch_signals+0x5be/frame 0xf18e3a68
    sleepq_wait_sig(c77df470,0,c143d661,101,0,...) at sleepq_wait_sig+0x14/frame 0xf18e3a94
    _cv_wait_sig(c77df470,c20a9714,c77df400,0,f18e3b1c,...) at _cv_wait_sig+0x17c/frame 0xf18e3ac4
    tty_wait(c77df400,c77df470,1,f18e3b0b,c75df000,...) at tty_wait+0x1f/frame 0xf18e3adc
    ttydisc_read(c77df400,f18e3c10,0,c0dab840,c7eb8dc8,...) at ttydisc_read+0xfe/frame 0xf18e3b1c
    ttydev_read(c794c900,f18e3c10,0,1,0,...) at ttydev_read+0x8f/frame 0xf18e3b40
    devfs_read_f(c7eb8dc8,f18e3c10,c8527b00,0,c85e4620,...) at devfs_read_f+0xbf/frame 0xf18e3b70
    dofileread(c7eb8dc8,f18e3c10,ffffffff,ffffffff,0,...) at dofileread+0x9e/frame 0xf18e3ba8
    kern_readv(c85e4620,0,f18e3c10,2c,bfbfead3,...) at kern_readv+0x96/frame 0xf18e3bf0
    sys_read(c85e4620,f18e3cc8,2,c788f064,c788f000,...) at sys_read+0x5c/frame 0xf18e3c40
    syscall(f18e3d08) at syscall+0x48b/frame 0xf18e3cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18e3cfc
    --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x28230617, esp = 0xbfbfea0c, ebp = 0xbfbfeae8 ---
    
    Tracing command sh pid 69850 tid 100140 td 0xc85e4930
    sched_switch(c85e4930,0,104,ac,98,...) at sched_switch+0x2da/frame 0xf18e0988
    mi_switch(104,0,f18e0a3c,c0cda1e4,f18e0aef,c85e4930) at mi_switch+0x122/frame 0xf18e09c0
    sleepq_switch(c85e4930,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18e09e8
    sleepq_catch_signals(c85e4930,0,c143ae9e,154,c12ce928,...) at sleepq_catch_signals+0x5be/frame 0xf18e0a3c
    sleepq_wait_sig(c85febc0,6c,c143bd97,100,0,...) at sleepq_wait_sig+0x14/frame 0xf18e0a68
    _sleep(c85febc0,c85fec58,16c,c143bd97,0,...) at _sleep+0x29b/frame 0xf18e0ab0
    kern_wait6(c85e4930,7,0,0,f18e0b4c,...) at kern_wait6+0x725/frame 0xf18e0b20
    sys_wait4(c85e4930,f18e0cc8,2,c788f064,c788f000,...) at sys_wait4+0x94/frame 0xf18e0c40
    syscall(f18e0d08) at syscall+0x48b/frame 0xf18e0cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18e0cfc
    --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28180113, esp = 0xbfbfeaa0, ebp = 0xbfbfeab8 ---
    
    Tracing command sshlockout_pf pid 69847 tid 100139 td 0xc8619000
    sched_switch(c8619000,0,104,c0d0867a,c8619000,...) at sched_switch+0x2da/frame 0xf18dda74
    mi_switch(104,0,c0d07bfa,c86015b3,1c251,c8619000) at mi_switch+0x122/frame 0xf18ddaac
    sleepq_switch(c8619000,0,c143ae9e,1a8,1c251,...) at sleepq_switch+0x15b/frame 0xf18ddad4
    sleepq_catch_signals(80000000,70,c0d38a10,c8619000,1,...) at sleepq_catch_signals+0x5be/frame 0xf18ddb28
    sleepq_timedwait_sig(c1f086f9,6c,1c251,80000000,70,...) at sleepq_timedwait_sig+0x14/frame 0xf18ddb58
    _sleep(c1f086f9,0,16c,c14377d8,c86015b3,...) at _sleep+0x26b/frame 0xf18ddba0
    kern_nanosleep(bf9fdf78,f18ddc18,f18ddc20,46,708,...) at kern_nanosleep+0x14a/frame 0xf18ddc00
    sys_nanosleep(c8619000,f18ddcc8,c0cfa212,c8619000,0,...) at sys_nanosleep+0x69/frame 0xf18ddc40
    syscall(f18ddd08) at syscall+0x48b/frame 0xf18ddcfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18ddcfc
    --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x281b5c73, esp = 0xbf9fdf48, ebp = 0xbf9fdf60 ---
    
    Tracing command sshlockout_pf pid 69847 tid 100115 td 0xc8108620
    sched_switch(c8108620,0,104,ac,98,...) at sched_switch+0x2da/frame 0xebff0a04
    mi_switch(104,0,c0cda1e4,c7e95680,146,c8108620) at mi_switch+0x122/frame 0xebff0a3c
    sleepq_switch(c8108620,0,c143ae9e,1a8,c7e95680,...) at sleepq_switch+0x15b/frame 0xebff0a64
    sleepq_catch_signals(c8108620,0,c143ae9e,154,0,...) at sleepq_catch_signals+0x5be/frame 0xebff0ab8
    sleepq_wait_sig(c82fb000,5c,c143b906,100,0,...) at sleepq_wait_sig+0x14/frame 0xebff0ae4
    _sleep(c82fb000,c82fb180,15c,c143b906,0,...) at _sleep+0x29b/frame 0xebff0b2c
    pipe_read(c7ecc038,ebff0c10,c7e95680,0,c8108620,...) at pipe_read+0x31a/frame 0xebff0b70
    dofileread(c7ecc038,ebff0c10,ffffffff,ffffffff,0,...) at dofileread+0x9e/frame 0xebff0ba8
    kern_readv(c8108620,0,ebff0c10,2c,2881a000,...) at kern_readv+0x96/frame 0xebff0bf0
    sys_read(c8108620,ebff0cc8,c8108620,35,3,...) at sys_read+0x5c/frame 0xebff0c40
    syscall(ebff0d08) at syscall+0x48b/frame 0xebff0cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebff0cfc
    --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281d8617, esp = 0xbfbfe934, ebp = 0xbfbfe948 ---
    
    Tracing command login pid 69601 tid 100138 td 0xc85e4c40
    sched_switch(c85e4c40,0,104,ac,98,...) at sched_switch+0x2da/frame 0xf18da988
    mi_switch(104,0,f18daa3c,c0cda1e4,f18daaef,c85e4c40) at mi_switch+0x122/frame 0xf18da9c0
    sleepq_switch(c85e4c40,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18da9e8
    sleepq_catch_signals(c85e4c40,0,c143ae9e,154,2,...) at sleepq_catch_signals+0x5be/frame 0xf18daa3c
    sleepq_wait_sig(c85ff000,6c,c143bd97,100,0,...) at sleepq_wait_sig+0x14/frame 0xf18daa68
    _sleep(c85ff000,c85ff098,16c,c143bd97,0,...) at _sleep+0x29b/frame 0xf18daab0
    kern_wait6(c85e4c40,0,110da,0,f18dab4c,...) at kern_wait6+0x725/frame 0xf18dab20
    sys_wait4(c85e4c40,f18dacc8,2,c788f064,c788f000,...) at sys_wait4+0x94/frame 0xf18dac40
    syscall(f18dad08) at syscall+0x48b/frame 0xf18dacfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18dacfc
    --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28126113, esp = 0xbfbfed28, ebp = 0xbfbfed40 ---
    
    Tracing command minicron pid 68298 tid 100132 td 0xc810ac40
    sched_switch(c810ac40,0,104,100,c85fd0a8,...) at sched_switch+0x2da/frame 0xf18c8a74
    mi_switch(104,0,c0d07bfa,63792b0f,2a350,c810ac40) at mi_switch+0x122/frame 0xf18c8aac
    sleepq_switch(c810ac40,0,c143ae9e,1a8,2a350,...) at sleepq_switch+0x15b/frame 0xf18c8ad4
    sleepq_catch_signals(0,1518,c0d38a10,c810ac40,1,...) at sleepq_catch_signals+0x5be/frame 0xf18c8b28
    sleepq_timedwait_sig(c1f086f9,6c,2a350,0,1518,...) at sleepq_timedwait_sig+0x14/frame 0xf18c8b58
    _sleep(c1f086f9,0,16c,c14377d8,63792b0f,...) at _sleep+0x26b/frame 0xf18c8ba0
    kern_nanosleep(bfbfed18,f18c8c18,f18c8c20,0,15180,...) at kern_nanosleep+0x14a/frame 0xf18c8c00
    sys_nanosleep(c810ac40,f18c8cc8,c0cfa212,c1f15000,c810ac40,...) at sys_nanosleep+0x69/frame 0xf18c8c40
    syscall(f18c8d08) at syscall+0x48b/frame 0xf18c8cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18c8cfc
    --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28192c73, esp = 0xbfbfed0c, ebp = 0xbfbfed40 ---
    
    Tracing command minicron pid 67787 tid 100137 td 0xc85e5000
    sched_switch(c85e5000,0,104,ac,98,...) at sched_switch+0x2da/frame 0xf18d7988
    mi_switch(104,0,f18d7a3c,c0cda1e4,f18d7aef,c85e5000) at mi_switch+0x122/frame 0xf18d79c0
    sleepq_switch(c85e5000,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18d79e8
    sleepq_catch_signals(c85e5000,0,c143ae9e,154,c0,...) at sleepq_catch_signals+0x5be/frame 0xf18d7a3c
    sleepq_wait_sig(c850c000,6c,c143bd97,100,0,...) at sleepq_wait_sig+0x14/frame 0xf18d7a68
    _sleep(c850c000,c850c098,16c,c143bd97,0,...) at _sleep+0x29b/frame 0xf18d7ab0
    kern_wait6(c85e5000,7,0,0,f18d7b4c,...) at kern_wait6+0x725/frame 0xf18d7b20
    sys_wait4(c85e5000,f18d7cc8,0,0,0,...) at sys_wait4+0x94/frame 0xf18d7c40
    syscall(f18d7d08) at syscall+0x48b/frame 0xf18d7cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18d7cfc
    --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28105113, esp = 0xbfbfed28, ebp = 0xbfbfed40 ---
    
    Tracing command minicron pid 67718 tid 100131 td 0xc830f000
    sched_switch(c830f000,0,104,c0d0867a,c85fd688,...) at sched_switch+0x2da/frame 0xf18c4a74
    mi_switch(104,0,c0d07bfa,e2c637ae,1c252,c830f000) at mi_switch+0x122/frame 0xf18c4aac
    sleepq_switch(c830f000,0,c143ae9e,1a8,1c252,...) at sleepq_switch+0x15b/frame 0xf18c4ad4
    sleepq_catch_signals(0,e1,c0d38a10,c830f000,0,...) at sleepq_catch_signals+0x5be/frame 0xf18c4b28
    sleepq_timedwait_sig(c1f086f8,6c,1c252,0,e1,...) at sleepq_timedwait_sig+0x14/frame 0xf18c4b58
    _sleep(c1f086f8,0,16c,c14377d8,e2c637ae,...) at _sleep+0x26b/frame 0xf18c4ba0
    kern_nanosleep(bfbfed28,f18c4c18,f18c4c20,0,e10,...) at kern_nanosleep+0x14a/frame 0xf18c4c00
    sys_nanosleep(c830f000,f18c4cc8,f18c4c7c,c0568180,c788f064,...) at sys_nanosleep+0x69/frame 0xf18c4c40
    syscall(f18c4d08) at syscall+0x48b/frame 0xf18c4cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18c4cfc
    --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28192c73, esp = 0xbfbfed1c, ebp = 0xbfbfed50 ---
    
    Tracing command minicron pid 67455 tid 100135 td 0xc85e5620
    sched_switch(c85e5620,0,104,ac,98,...) at sched_switch+0x2da/frame 0xf18d1988
    mi_switch(104,0,f18d1a3c,c0cda1e4,f18d1aef,c85e5620) at mi_switch+0x122/frame 0xf18d19c0
    sleepq_switch(c85e5620,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18d19e8
    sleepq_catch_signals(c85e5620,0,c143ae9e,154,c0,...) at sleepq_catch_signals+0x5be/frame 0xf18d1a3c
    sleepq_wait_sig(c850c5e0,6c,c143bd97,100,0,...) at sleepq_wait_sig+0x14/frame 0xf18d1a68
    _sleep(c850c5e0,c850c678,16c,c143bd97,0,...) at _sleep+0x29b/frame 0xf18d1ab0
    kern_wait6(c85e5620,7,0,0,f18d1b4c,...) at kern_wait6+0x725/frame 0xf18d1b20
    sys_wait4(c85e5620,f18d1cc8,0,246,f18d1cc8,...) at sys_wait4+0x94/frame 0xf18d1c40
    syscall(f18d1d08) at syscall+0x48b/frame 0xf18d1cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18d1cfc
    --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28105113, esp = 0xbfbfed38, ebp = 0xbfbfed50 ---
    
    Tracing command minicron pid 67290 tid 100110 td 0xc8109310
    sched_switch(c8109310,0,104,100,c810b398,...) at sched_switch+0x2da/frame 0xebfe1a74
    mi_switch(104,0,c0d07bfa,7980ad37,1c196,c8109310) at mi_switch+0x122/frame 0xebfe1aac
    sleepq_switch(c8109310,0,c143ae9e,1a8,1c196,...) at sleepq_switch+0x15b/frame 0xebfe1ad4
    sleepq_catch_signals(0,f,c0d38a10,c8109310,1,...) at sleepq_catch_signals+0x5be/frame 0xebfe1b28
    sleepq_timedwait_sig(c1f086f9,6c,1c196,0,f,...) at sleepq_timedwait_sig+0x14/frame 0xebfe1b58
    _sleep(c1f086f9,0,16c,c14377d8,7980ad37,...) at _sleep+0x26b/frame 0xebfe1ba0
    kern_nanosleep(bfbfed48,ebfe1c18,ebfe1c20,0,f0,...) at kern_nanosleep+0x14a/frame 0xebfe1c00
    sys_nanosleep(c8109310,ebfe1cc8,c0cfa212,c8109310,0,...) at sys_nanosleep+0x69/frame 0xebfe1c40
    syscall(ebfe1d08) at syscall+0x48b/frame 0xebfe1cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfe1cfc
    --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x28192c73, esp = 0xbfbfed3c, ebp = 0xbfbfed70 ---
    
    Tracing command minicron pid 66962 tid 100107 td 0xc8109c40
    sched_switch(c8109c40,0,104,ac,98,...) at sched_switch+0x2da/frame 0xebfd5988
    mi_switch(104,0,ebfd5a3c,c0cda1e4,ebfd5aef,c8109c40) at mi_switch+0x122/frame 0xebfd59c0
    sleepq_switch(c8109c40,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebfd59e8
    sleepq_catch_signals(c8109c40,0,c143ae9e,154,c0,...) at sleepq_catch_signals+0x5be/frame 0xebfd5a3c
    sleepq_wait_sig(c810c8d0,6c,c143bd97,100,0,...) at sleepq_wait_sig+0x14/frame 0xebfd5a68
    _sleep(c810c8d0,c810c968,16c,c143bd97,0,...) at _sleep+0x29b/frame 0xebfd5ab0
    kern_wait6(c8109c40,7,0,0,ebfd5b4c,...) at kern_wait6+0x725/frame 0xebfd5b20
    sys_wait4(c8109c40,ebfd5cc8,c0cfa212,c8109c40,0,...) at sys_wait4+0x94/frame 0xebfd5c40
    syscall(ebfd5d08) at syscall+0x48b/frame 0xebfd5cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfd5cfc
    --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28105113, esp = 0xbfbfed58, ebp = 0xbfbfed70 ---
    
    Tracing command syslogd pid 60173 tid 100133 td 0xc810a930
    sched_switch(c810a930,0,104,0,0,...) at sched_switch+0x2da/frame 0xf18cb908
    mi_switch(104,0,f18cb9bc,c0cda1e4,281a8a0c,c810a930) at mi_switch+0x122/frame 0xf18cb940
    sleepq_switch(c810a930,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18cb968
    sleepq_catch_signals(c810a930,0,c143ae9e,154,2,...) at sleepq_catch_signals+0x5be/frame 0xf18cb9bc
    sleepq_wait_sig(c80d8624,0,c147b5b6,101,0,...) at sleepq_wait_sig+0x14/frame 0xf18cb9e8
    _cv_wait_sig(c80d8624,c80d8610,f18cba50,c0ba28d9,c77ee600,...) at _cv_wait_sig+0x17c/frame 0xf18cba18
    seltdwait(ffffffff,0,0,c810a930,f18cbbc8,...) at seltdwait+0xcf/frame 0xf18cba50
    kern_select(c810a930,10,28835118,0,0,...) at kern_select+0x8d2/frame 0xf18cbbf8
    sys_select(c810a930,f18cbcc8,0,0,0,...) at sys_select+0x69/frame 0xf18cbc40
    syscall(f18cbd08) at syscall+0x48b/frame 0xf18cbcfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18cbcfc
    --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281cf5d7, esp = 0xbfbfdde4, ebp = 0xbfbfed74 ---
    
    Tracing command powerd pid 56017 tid 100125 td 0xc8314310
    sched_switch(c8314310,0,104,c9,0,...) at sched_switch+0x2da/frame 0xf18ab900
    mi_switch(104,0,d14f1997,1c0f7,7ffffff,c8314310) at mi_switch+0x122/frame 0xf18ab938
    sleepq_switch(c8314310,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xf18ab960
    sleepq_catch_signals(7ffffff,0,c0d38a10,c8314310,0,...) at sleepq_catch_signals+0x5be/frame 0xf18ab9b4
    sleepq_timedwait_sig(c7e6df64,0,1c0f7,7ffffff,0,...) at sleepq_timedwait_sig+0x14/frame 0xf18ab9e4
    _cv_timedwait_sig_sbt(c7e6df64,c7e6df50,d14f1997,1c0f7,7ffffff,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xf18aba18
    seltdwait(1c0f7,7ffffff,0,c868cfc8,100,...) at seltdwait+0xc1/frame 0xf18aba50
    kern_select(c8314310,0,bfbfed10,0,bfbfed10,...) at kern_select+0x8d2/frame 0xf18abbf8
    sys_select(c8314310,f18abcc8,c0cfa212,c8314310,0,...) at sys_select+0x69/frame 0xf18abc40
    syscall(f18abd08) at syscall+0x48b/frame 0xf18abcfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xf18abcfc
    --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281c95d7, esp = 0xbfbfec8c, ebp = 0xbfbfedc4 ---
    
    Tracing command inetd pid 29837 tid 100102 td 0xc7ef2930
    sched_switch(c7ef2930,0,104,c7e9a000,0,...) at sched_switch+0x2da/frame 0xebfc3908
    mi_switch(104,0,ebfc39bc,c0cda1e4,0,c7ef2930) at mi_switch+0x122/frame 0xebfc3940
    sleepq_switch(c7ef2930,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebfc3968
    sleepq_catch_signals(c7ef2930,0,c143ae9e,154,96,...) at sleepq_catch_signals+0x5be/frame 0xebfc39bc
    sleepq_wait_sig(c80d77a4,0,c147b5b6,101,0,...) at sleepq_wait_sig+0x14/frame 0xebfc39e8
    _cv_wait_sig(c80d77a4,c80d7790,c8684848,1a,c7ef2930,...) at _cv_wait_sig+0x17c/frame 0xebfc3a18
    seltdwait(ffffffff,0,0,c7ef2930,ebfc3bc8,...) at seltdwait+0xcf/frame 0xebfc3a50
    kern_select(c7ef2930,1b,bfbfe000,0,0,...) at kern_select+0x8d2/frame 0xebfc3bf8
    sys_select(c7ef2930,ebfc3cc8,c76fa930,c1f15000,ebfc3cc8,...) at sys_select+0x69/frame 0xebfc3c40
    syscall(ebfc3d08) at syscall+0x48b/frame 0xebfc3cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfc3cfc
    --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x281df5d7, esp = 0xbfbfdf9c, ebp = 0xbfbfedc4 ---
    
    Tracing command filterlog pid 29163 tid 100096 td 0xc7ff1c40
    sched_switch(c7ff1c40,0,104,ebfb1a00,0,...) at sched_switch+0x2da/frame 0xebfb19d0
    mi_switch(104,0,c5c88512,1c0f7,fffffed,c7ff1c40) at mi_switch+0x122/frame 0xebfb1a08
    sleepq_switch(c7ff1c40,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebfb1a30
    sleepq_catch_signals(0,0,c0d38a10,c7ff1c40,0,...) at sleepq_catch_signals+0x5be/frame 0xebfb1a84
    sleepq_timedwait_sig(c7976a00,1a,0,0,0,...) at sleepq_timedwait_sig+0x14/frame 0xebfb1ab4
    _sleep(c7976a00,c7976a8c,11a,c144e70b,fffffed8,...) at _sleep+0x26b/frame 0xebfb1afc
    bpfread(c75e6400,ebfb1c10,0,80000,0,...) at bpfread+0x218/frame 0xebfb1b40
    devfs_read_f(c7eca188,ebfb1c10,c7e95680,0,c7ff1c40,...) at devfs_read_f+0xbf/frame 0xebfb1b70
    dofileread(c7eca188,ebfb1c10,ffffffff,ffffffff,0,...) at dofileread+0x9e/frame 0xebfb1ba8
    kern_readv(c7ff1c40,a,ebfb1c10,c1f14900,2882d000,...) at kern_readv+0x96/frame 0xebfb1bf0
    sys_read(c7ff1c40,ebfb1cc8,602,c1f14900,c7ff1c40,...) at sys_read+0x5c/frame 0xebfb1c40
    syscall(ebfb1d08) at syscall+0x48b/frame 0xebfb1cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfb1cfc
    --- syscall (3, FreeBSD ELF32, sys_read), eip = 0x281f8617, esp = 0xbfbfec74, ebp = 0xbfbfed60 ---
    
    Tracing command choparp pid 27182 tid 100082 td 0xc7ef1620
    sched_switch(c7ef1620,0,104,394,0,...) at sched_switch+0x2da/frame 0xebf7a948
    mi_switch(104,0,ebf7a9fc,c0cda1e4,ebf7aa38,c7ef1620) at mi_switch+0x122/frame 0xebf7a980
    sleepq_switch(c7ef1620,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebf7a9a8
    sleepq_catch_signals(c7ef1620,0,c143ae9e,154,c7e05800,...) at sleepq_catch_signals+0x5be/frame 0xebf7a9fc
    sleepq_wait_sig(c7e92580,68,c1431da4,100,0,...) at sleepq_wait_sig+0x14/frame 0xebf7aa28
    _sleep(c7e92580,c7e92580,168,c1431da4,0,...) at _sleep+0x29b/frame 0xebf7aa70
    kern_kevent(c7ef1620,a,0,1,ebf7ac10,...) at kern_kevent+0x48f/frame 0xebf7abc0
    sys_kevent(c7ef1620,ebf7acc8,c0cfa212,c1f15000,c7ef1620,...) at sys_kevent+0x131/frame 0xebf7ac40
    syscall(ebf7ad08) at syscall+0x48b/frame 0xebf7acfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebf7acfc
    --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2815f773, esp = 0xbfbfec7c, ebp = 0xbfbfecdc ---
    
    Tracing command mpd5 pid 23101 tid 100120 td 0xc7ef2000
    sched_switch(c7ef2000,0,104,c7ef2000,0,...) at sched_switch+0x2da/frame 0xebf71970
    mi_switch(104,0,c60b375f,1c100,9fffff47,c7ef2000) at mi_switch+0x122/frame 0xebf719a8
    sleepq_switch(c7ef2000,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebf719d0
    sleepq_catch_signals(9fffff47,0,c0d38a10,c7ef2000,1,...) at sleepq_catch_signals+0x5be/frame 0xebf71a24
    sleepq_timedwait_sig(c80d7f24,0,1c100,9fffff47,0,...) at sleepq_timedwait_sig+0x14/frame 0xebf71a54
    _cv_timedwait_sig_sbt(c80d7f24,c80d7f10,c60b375f,1c100,9fffff47,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xebf71a88
    seltdwait(1c100,9fffff47,0,c7ef2000,0,...) at seltdwait+0xc1/frame 0xebf71ac0
    sys_poll(c7ef2000,ebf71cc8,c0cfa212,c7ef2000,0,...) at sys_poll+0x478/frame 0xebf71c40
    syscall(ebf71d08) at syscall+0x48b/frame 0xebf71cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebf71cfc
    --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2848d68f, esp = 0xbf9fdf0c, ebp = 0xbf9fdf28 ---
    
    Tracing command mpd5 pid 18713 tid 100119 td 0xc7ef1310
    sched_switch(c7ef1310,0,104,2,0,...) at sched_switch+0x2da/frame 0xebf7d970
    mi_switch(104,0,108ab789,1c0fd,9fffff47,c7ef1310) at mi_switch+0x122/frame 0xebf7d9a8
    sleepq_switch(c7ef1310,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebf7d9d0
    sleepq_catch_signals(9fffff47,0,c0d38a10,c7ef1310,1,...) at sleepq_catch_signals+0x5be/frame 0xebf7da24
    sleepq_timedwait_sig(c7e6c9a4,0,1c0fd,9fffff47,0,...) at sleepq_timedwait_sig+0x14/frame 0xebf7da54
    _cv_timedwait_sig_sbt(c7e6c9a4,c7e6c990,108ab789,1c0fd,9fffff47,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xebf7da88
    seltdwait(1c0fd,9fffff47,0,c7ef1310,0,...) at seltdwait+0xc1/frame 0xebf7dac0
    sys_poll(c7ef1310,ebf7dcc8,c0cfa212,c7ef1310,0,...) at sys_poll+0x478/frame 0xebf7dc40
    syscall(ebf7dd08) at syscall+0x48b/frame 0xebf7dcfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebf7dcfc
    --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2848d68f, esp = 0xbf9fdf0c, ebp = 0xbf9fdf28 ---
    
    Tracing command mpd5 pid 14790 tid 100118 td 0xc7ff2c40
    sched_switch(c7ff2c40,0,104,c7ff2c40,0,...) at sched_switch+0x2da/frame 0xebffc970
    mi_switch(104,0,2e3cdafa,1c0fe,9fffff47,c7ff2c40) at mi_switch+0x122/frame 0xebffc9a8
    sleepq_switch(c7ff2c40,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebffc9d0
    sleepq_catch_signals(9fffff47,0,c0d38a10,c7ff2c40,0,...) at sleepq_catch_signals+0x5be/frame 0xebffca24
    sleepq_timedwait_sig(c7e6cb24,0,1c0fe,9fffff47,0,...) at sleepq_timedwait_sig+0x14/frame 0xebffca54
    _cv_timedwait_sig_sbt(c7e6cb24,c7e6cb10,2e3cdafa,1c0fe,9fffff47,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xebffca88
    seltdwait(1c0fe,9fffff47,0,c7ff2c40,0,...) at seltdwait+0xc1/frame 0xebffcac0
    sys_poll(c7ff2c40,ebffccc8,ebffcc7c,c0d3bb6b,0,...) at sys_poll+0x478/frame 0xebffcc40
    syscall(ebffcd08) at syscall+0x48b/frame 0xebffccfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebffccfc
    --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2848d68f, esp = 0xbf9fdf0c, ebp = 0xbf9fdf28 ---
    
    Tracing command sshlockout_pf pid 7906 tid 100113 td 0xc8109000
    sched_switch(c8109000,0,104,c0d0867a,c8108f20,...) at sched_switch+0x2da/frame 0xebfe5a74
    mi_switch(104,0,c0d07bfa,5a8dc568,1c21d,c8109000) at mi_switch+0x122/frame 0xebfe5aac
    sleepq_switch(c8109000,0,c143ae9e,1a8,1c21d,...) at sleepq_switch+0x15b/frame 0xebfe5ad4
    sleepq_catch_signals(80000000,70,c0d38a10,c8109000,1,...) at sleepq_catch_signals+0x5be/frame 0xebfe5b28
    sleepq_timedwait_sig(c1f086f9,6c,1c21d,80000000,70,...) at sleepq_timedwait_sig+0x14/frame 0xebfe5b58
    _sleep(c1f086f9,0,16c,c14377d8,5a8dc568,...) at _sleep+0x26b/frame 0xebfe5ba0
    kern_nanosleep(bf9fdf78,ebfe5c18,ebfe5c20,c0d1c100,708,...) at kern_nanosleep+0x14a/frame 0xebfe5c00
    sys_nanosleep(c8109000,ebfe5cc8,c0cfa212,c1f15000,c8109000,...) at sys_nanosleep+0x69/frame 0xebfe5c40
    syscall(ebfe5d08) at syscall+0x48b/frame 0xebfe5cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfe5cfc
    --- syscall (240, FreeBSD ELF32, sys_nanosleep), eip = 0x281b5c73, esp = 0xbf9fdf48, ebp = 0xbf9fdf60 ---
    
    Tracing command sshlockout_pf pid 7906 tid 100105 td 0xc810a310
    sched_switch(c810a310,0,104,146,0,...) at sched_switch+0x2da/frame 0xebfcda40
    mi_switch(104,0,ebfcdaf4,c0cda1e4,1,c810a310) at mi_switch+0x122/frame 0xebfcda78
    sleepq_switch(c810a310,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebfcdaa0
    sleepq_catch_signals(c810a310,0,c143ae9e,154,1,...) at sleepq_catch_signals+0x5be/frame 0xebfcdaf4
    sleepq_wait_sig(c7e6fd80,0,c1437bcd,100,0,...) at sleepq_wait_sig+0x14/frame 0xebfcdb20
    _sleep(c7e6fd80,c1f11cec,300,c1437bcd,0,...) at _sleep+0x29b/frame 0xebfcdb68
    umtxq_sleep(0,0,2,c7e6fd88,ebfcdc10,...) at umtxq_sleep+0x125/frame 0xebfcdbc8
    do_wait(18711,0,0,0,ebfcdcc8,...) at do_wait+0x2ff/frame 0xebfcdc10
    __umtx_op_wait(c810a310,ebfcdcc8,2,c788f064,c788f000,...) at __umtx_op_wait+0x9f/frame 0xebfcdc40
    syscall(ebfcdd08) at syscall+0x48b/frame 0xebfcdcfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfcdcfc
    --- syscall (454, FreeBSD ELF32, sys__umtx_op), eip = 0x280a58e3, esp = 0xbfbfe924, ebp = 0xbfbfe940 ---
    
    Tracing command sshd pid 7822 tid 100108 td 0xc8109930
    sched_switch(c8109930,0,104,ac,98,...) at sched_switch+0x2da/frame 0xebfd9908
    mi_switch(104,0,ebfd99bc,c0cda1e4,ebfd996c,c8109930) at mi_switch+0x122/frame 0xebfd9940
    sleepq_switch(c8109930,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebfd9968
    sleepq_catch_signals(c8109930,0,c143ae9e,154,a,...) at sleepq_catch_signals+0x5be/frame 0xebfd99bc
    sleepq_wait_sig(c80d8164,0,c147b5b6,101,0,...) at sleepq_wait_sig+0x14/frame 0xebfd99e8
    _cv_wait_sig(c80d8164,c80d8150,c801a350,5,c8109930,...) at _cv_wait_sig+0x17c/frame 0xebfd9a18
    seltdwait(ffffffff,0,0,c8109930,ebfd9bc8,...) at seltdwait+0xcf/frame 0xebfd9a50
    kern_select(c8109930,6,28c03150,0,0,...) at kern_select+0x8d2/frame 0xebfd9bf8
    sys_select(c8109930,ebfd9cc8,c7954620,c1f14900,3,...) at sys_select+0x69/frame 0xebfd9c40
    syscall(ebfd9d08) at syscall+0x48b/frame 0xebfd9cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfd9cfc
    --- syscall (93, FreeBSD ELF32, sys_select), eip = 0x284645d7, esp = 0xbfbfe43c, ebp = 0xbfbfee14 ---
    
    Tracing command mpd5 pid 6907 tid 100112 td 0xc8108c40
    sched_switch(c8108c40,0,104,c8108c40,0,...) at sched_switch+0x2da/frame 0xebfe8970
    mi_switch(104,0,f732c1b8,1c0fb,9fffff47,c8108c40) at mi_switch+0x122/frame 0xebfe89a8
    sleepq_switch(c8108c40,0,c143ae9e,1a8,0,...) at sleepq_switch+0x15b/frame 0xebfe89d0
    sleepq_catch_signals(9fffff47,0,c0d38a10,c8108c40,0,...) at sleepq_catch_signals+0x5be/frame 0xebfe8a24
    sleepq_timedwait_sig(c80d85a4,0,1c0fb,9fffff47,0,...) at sleepq_timedwait_sig+0x14/frame 0xebfe8a54
    _cv_timedwait_sig_sbt(c80d85a4,c80d8590,f732c1b8,1c0fb,9fffff47,...) at _cv_timedwait_sig_sbt+0x1a7/frame 0xebfe8a88
    seltdwait(1c0fb,9fffff47,0,c8108c40,0,...) at seltdwait+0xc1/frame 0xebfe8ac0
    sys_poll(c8108c40,ebfe8cc8,2,c788f064,c788f000,...) at sys_poll+0x478/frame 0xebfe8c40
    syscall(ebfe8d08) at syscall+0x48b/frame 0xebfe8cfc
    Xint0x80_syscall() at Xint0x80_syscall+0x21/frame 0xebfe8cfc
    --- syscall (209, FreeBSD ELF32, sys_poll), eip = 0x2848d68f, esp = 0xbf9fdf0c, ebp = 0xbf9fdf28 ---
    
    Tracing command devd pid 274 tid 100106 td 0xc810a000
    sched_switch(c810a000,0,104,c76ff620,0,...) at sched_switch+0x2da/frame 0xebfd0900
    mi_switch(104,0,12142fb0,1c108,c0000000,c810a000) at mi_switch+0x122/frame 0xebfd0938
    sleepq_switch(c810a000,0,c143ae9e,1a8,3,...) at sleepq_switch+0x15b/frame 0xebfd0960
    sleepq_catch_signals(c0000000,3,c0d38a10,c810a000,0,...) at sleepq_catch_sign[/thread]
    

  • Rebel Alliance Developer Netgate

    Try adding a tunable for net.bpf.zerocopy_enable=1

    We have seen a few of the "sleeping thread" type panics, but not enough of that crash report made it through to tell if it's the same as others.



  • i added that to all my boxes so lets wait and see how it goes, ill report back



  • i can confirm that solved it, probably it should be set as default in the config


Log in to reply