Crash report after update from 2.3.2 early june –> todays build



  • i've submitteed it from from 78.x.192.160

    
    					Crash report begins.  Anonymous machine information:
    
    i386
    10.3-RELEASE-p5
    FreeBSD 10.3-RELEASE-p5 #54 0523c9e(RELENG_2_3): Mon Jun 27 20:29:45 CDT 2016     root@ce23-i386-builder:/builder/pfsense/tmp/obj/builder/pfsense/tmp/FreeBSD-src/sys/pfSense
    
    Crash report details:
    
    Filename: /var/crash/bounds
    1
    
    Filename: /var/crash/info.0
    Dump header from device /dev/ada0s1b
      Architecture: i386
      Architecture Version: 1
      Dump Length: 70656B (0 MB)
      Blocksize: 512
      Dumptime: Tue Jun 28 08:03:43 2016
      Hostname: pfsense.vbees.lan
      Magic: FreeBSD Text Dump
      Version String: FreeBSD 10.3-RELEASE-p5 #35 b8fc376(RELENG_2_3): Thu Jun  9 18:55:25 CDT 2016
        root@ce23-i386-builder:/builder/pfsense/tmp/obj/builder/pfsense/tmp/FreeBSD-src/sys/pfSense
      Panic String: 
      Dump Parity: 1280922957
      Bounds: 0
      Dump Status: good
    
    Filename: /var/crash/info.last
    Dump header from device /dev/ada0s1b
      Architecture: i386
      Architecture Version: 1
      Dump Length: 70656B (0 MB)
      Blocksize: 512
      Dumptime: Tue Jun 28 08:03:43 2016
      Hostname: pfsense.vbees.lan
      Magic: FreeBSD Text Dump
      Version String: FreeBSD 10.3-RELEASE-p5 #35 b8fc376(RELENG_2_3): Thu Jun  9 18:55:25 CDT 2016
        root@ce23-i386-builder:/builder/pfsense/tmp/obj/builder/pfsense/tmp/FreeBSD-src/sys/pfSense
      Panic String: 
      Dump Parity: 1280922957
      Bounds: 0
      Dump Status: good
    
    Filename: /var/crash/minfree
    2048
    
    Filename: /var/crash/textdump.tar.0
    ddb.txt06000014000012734411477  7104 ustarrootwheeldb:0:kdb.enter.default>  run lockinfo
    db:1:lockinfo> show locks
    No such command
    db:1:locks>  show alllocks
    No such command
    db:1:alllocks>  show lockedvnods
    Locked vnodes
    db:0:kdb.enter.default>  show pcpu
    cpuid        = 0
    dynamic pcpu = 0x570200
    curthread    = 0xc54e5320: pid 24423 "sh"
    curpcb       = 0xe852ed40
    fpcurthread  = none
    idlethread   = 0xc4298000: tid 100003 "idle: cpu0"
    APIC ID      = 0
    currentldt   = 0x50
    db:0:kdb.enter.default>  bt
    Tracing pid 24423 tid 100139 td 0xc54e5320
    swap_release_by_cred(2000,0,287f5000,0,c2d4ad28,...) at swap_release_by_cred+0x1a/frame 0xe852e92c
    vm_map_delete(c26bc5a0,1000,bfc00000,0,c26bc654,...) at vm_map_delete+0x2b4/frame 0xe852e98c
    vm_map_remove(c26bc5a0,1000,bfc00000,c4bddafc,0,...) at vm_map_remove+0xc2/frame 0xe852e9b4
    vmspace_exit(c54e5320,c4bdd918,1,0,e852ea24,...) at vmspace_exit+0xda/frame 0xe852e9e0
    exit1(c54e5320,8b,5f67,c4bddb30,0,...) at exit1+0x879/frame 0xe852ea50
    sigexit(c54e5320,b,e852ed18,b,2,...) at sigexit+0x924/frame 0xe852ec08
    postsig(b,0,c1480f47,df,e852ecd8,...) at postsig+0x2de/frame 0xe852eca8
    ast(e852ece8) at ast+0x538/frame 0xe852ecdc
    doreti_ast() at doreti_ast+0x1b/frame 0xbfbfec88
    db:0:kdb.enter.default>  ps
      pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
    83058     1 83058     0  DEs     vmmaps   0xc26bd354 zebra
    24423     1 24423     0  RE+     CPU 0               sh
    10224  9794  9794     0  DE      vmmaps   0xc26bdbc4 minicron
     9794     1  9794     0  Ss      wait     0xc5520c20 minicron
      304   302   302     0  S       kqread   0xc4787c80 check_reload_status
      302     1   302     0  Ss      kqread   0xc4786980 check_reload_status
       56     0     0     0  DL      mdwait   0xc43d0000 [md0]
       21     0     0     0  DL      vlruwt   0xc4724918 [vnlru]
       20     0     0     0  DL      syncer   0xc1f98dc4 [syncer]
       19     0     0     0  DL      qsleep   0xc1f98504 [bufdaemon]
       18     0     0     0  DL      pgzero   0xc1fa2fa8 [pagezero]
       17     0     0     0  DL      pollid   0xc1f785f0 [idlepoll]
        9     0     0     0  DL      psleep   0xc1fa2cc4 [vmdaemon]
        8     0     0     0  DL      (threaded)          [pagedaemon]
    100072                   D       umarcl   0xc1fa28a4 [uma]
    100065                   D       psleep   0xc2026404 [pagedaemon]
        7     0     0     0  DL      waiting_ 0xc2020d74 [sctp_iterator]
        6     0     0     0  DL      pftm     0xc0f94ba0 [pf purge]
        5     0     0     0  DL      -        0xc4314c3c [fdc0]
       16     0     0     0  DL      (threaded)          [usb]
    100048                   D       -        0xc443ad64 [usbus3]
    100047                   D       -        0xc443ad34 [usbus3]
    100046                   D       -        0xc443ad04 [usbus3]
    100045                   D       -        0xc443acd4 [usbus3]
    100044                   D       -        0xc443aca4 [usbus3]
    100042                   D       -        0xc441addc [usbus2]
    100041                   D       -        0xc441adac [usbus2]
    100040                   D       -        0xc441ad7c [usbus2]
    100039                   D       -        0xc441ad4c [usbus2]
    100038                   D       -        0xc441ad1c [usbus2]
    100036                   D       -        0xc4405ddc [usbus1]
    100035                   D       -        0xc4405dac [usbus1]
    100034                   D       -        0xc4405d7c [usbus1]
    100033                   D       -        0xc4405d4c [usbus1]
    100032                   D       -        0xc4405d1c [usbus1]
    100030                   D       -        0xc43d7ddc [usbus0]
    100029                   D       -        0xc43d7dac [usbus0]
    100028                   D       -        0xc43d7d7c [usbus0]
    100027                   D       -        0xc43d7d4c [usbus0]
    100026                   D       -        0xc43d7d1c [usbus0]
        4     0     0     0  DL      (threaded)          [cam]
    100064                   D       -        0xc1eb7cac [scanner]
    100017                   D       -        0xc1eb7e00 [doneq0]
       15     0     0     0  DL      -        0xc1ed4ec8 [rand_harvestq]
        3     0     0     0  DL      crypto_r 0xc1fa1c28 [crypto returns]
        2     0     0     0  DL      crypto_w 0xc1fa1b68 [crypto]
       14     0     0     0  DL      (threaded)          [geom]
    100010                   D       -        0xc201bdd8 [g_down]
    100009                   D       -        0xc201bdd4 [g_up]
    100008                   D       -        0xc201bdd0 [g_event]
       13     0     0     0  DL      sleep    0xc1e8f63c [ng_queue0]
       12     0     0     0  WL      (threaded)          [intr]
    100062                   I                           [swi1: pfsync]
    100060                   I                           [swi1: pf send]
    100057                   I                           [swi0: uart]
    100056                   I                           [irq7: ppc0]
    100054                   I                           [irq1: atkbd0]
    100053                   I                           [irq15: ata1]
    100052                   I                           [irq14: ata0]
    100049                   I                           [irq17: pcm0 rl0]
    100043                   I                           [irq23: dc0 ehci0]
    100037                   I                           [irq18: em0 uhci2]
    100031                   I                           [irq19: em1 uhci1]
    100025                   I                           [irq16: uhci0]
    100024                   I                           [swi6: Giant taskq]
    100022                   I                           [swi6: task queue]
    100016                   I                           [swi5: fast taskq]
    100006                   I                           [swi4: clock]
    100005                   I                           [swi1: netisr 0]
    100004                   I                           [swi3: vm]
       11     0     0     0  RL                          [idle: cpu0]
        1     0     1     0  SLs     wait     0xc4291308 [init]
       10     0     0     0  DL      audit_wo 0xc2024cdc [audit]
        0     0     0     0  DLs     (threaded)          [kernel]
    100126                   D       -        0xc4ce4e80 [dummynet]
    100063                   D       -        0xc4293280 [CAM taskq]
    100058                   D       -        0xc442ed00 [mca taskq]
    100051                   D       -        0xc4450800 [em1 taskq]
    100050                   D       -        0xc4450a80 [em0 taskq]
    100023                   D       -        0xc41c8c80 [ffs_trim taskq]
    100021                   D       -        0xc41c8f00 [kqueue taskq]
    100020                   D       -        0xc4293000 [acpi_task_2]
    100019                   D       -        0xc4293000 [acpi_task_1]
    100018                   D       -        0xc4293000 [acpi_task_0]
    100015                   D       -        0xc4293400 [thread]
    100011                   D       -        0xc4294780 [firmware taskq]
    100000                   D       swapin   0xc201be54 [swapper]
    db:0:kdb.enter.default>  alltrace
    
    Tracing command zebra pid 83058 tid 100157 td 0xc5527000
    sched_switch(c5527000,0,104,e856b9fc,0,...) at sched_switch+0x2d2/frame 0xe856b9d8
    mi_switch(104,0,c5e5d1a8,e856ba60,c5527000,c26bd354) at mi_switch+0x122/frame 0xe856ba10
    sleepq_switch(c5527000,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xe856ba38
    sleepq_wait(c26bd354,54,c14b54ce,0,0,...) at sleepq_wait+0x3f/frame 0xe856ba64
    _sleep(c26bd354,c1fa2c44,254,c14b54ce,0,0,0,0,100) at _sleep+0x2ae/frame 0xe856baac
    _vm_map_unlock_and_wait(c26bd2d0,0,0,0,c2d2bad8,...) at _vm_map_unlock_and_wait+0x121/frame 0xe856bae4
    vm_map_delete(c26bd2d0,1000,bfc00000,0,c26bd384,...) at vm_map_delete+0xf3/frame 0xe856bb44
    vm_map_remove(c26bd2d0,1000,bfc00000,c551fafc,0,...) at vm_map_remove+0xc2/frame 0xe856bb6c
    vmspace_exit(c5527000,c551f918,1,0,c1e841d0,...) at vmspace_exit+0xda/frame 0xe856bb98
    exit1(c5527000,0,e856bcdc,c12e8089,c5527000,...) at exit1+0x879/frame 0xe856bc08
    sys_sys_exit(c5527000,e856bca8,c551f918,c5523000,0,...) at sys_sys_exit+0x1d/frame 0xe856bc18
    syscall(e856bce8) at syscall+0x5c9/frame 0xe856bcdc
    Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xe856bcdc
    --- syscall (1, FreeBSD ELF32, sys_sys_exit), eip = 0x281e2273, esp = 0xbfbfeb34, ebp = 0xbfbfeb44 ---
    
    Tracing command sh pid 24423 tid 100139 td 0xc54e5320
    swap_release_by_cred(2000,0,287f5000,0,c2d4ad28,...) at swap_release_by_cred+0x1a/frame 0xe852e92c
    vm_map_delete(c26bc5a0,1000,bfc00000,0,c26bc654,...) at vm_map_delete+0x2b4/frame 0xe852e98c
    vm_map_remove(c26bc5a0,1000,bfc00000,c4bddafc,0,...) at vm_map_remove+0xc2/frame 0xe852e9b4
    vmspace_exit(c54e5320,c4bdd918,1,0,e852ea24,...) at vmspace_exit+0xda/frame 0xe852e9e0
    exit1(c54e5320,8b,5f67,c4bddb30,0,...) at exit1+0x879/frame 0xe852ea50
    sigexit(c54e5320,b,e852ed18,b,2,...) at sigexit+0x924/frame 0xe852ec08
    postsig(b,0,c1480f47,df,e852ecd8,...) at postsig+0x2de/frame 0xe852eca8
    ast(e852ece8) at ast+0x538/frame 0xe852ecdc
    doreti_ast() at doreti_ast+0x1b/frame 0xbfbfec88
    
    Tracing command minicron pid 10224 tid 100145 td 0xc5529320
    sched_switch(c5529320,0,104,e8546840,0,...) at sched_switch+0x2d2/frame 0xe8546820
    mi_switch(104,0,0,0,c5529320,c26bdbc4) at mi_switch+0x122/frame 0xe8546858
    sleepq_switch(c5529320,0,c1480a56,269,64ab81f9,...) at sleepq_switch+0x15b/frame 0xe8546880
    sleepq_wait(c26bdbc4,54,c14b54ce,0,0,...) at sleepq_wait+0x3f/frame 0xe85468ac
    _sleep(c26bdbc4,c1fa2c44,254,c14b54ce,0,0,0,0,100) at _sleep+0x2ae/frame 0xe85468f4
    _vm_map_unlock_and_wait(c26bdb40,0,0,0,c2c0d2b0,...) at _vm_map_unlock_and_wait+0x121/frame 0xe854692c
    vm_map_delete(c26bdb40,1000,bfc00000,0,c26bdbf4,...) at vm_map_delete+0xf3/frame 0xe854698c
    vm_map_remove(c26bdb40,1000,bfc00000,c5520afc,0,...) at vm_map_remove+0xc2/frame 0xe85469b4
    vmspace_exit(c5529320,c5520918,1,0,e8546a00,...) at vmspace_exit+0xda/frame 0xe85469e0
    exit1(c5529320,f,ac,98,e8546a84,...) at exit1+0x879/frame 0xe8546a50
    sigexit(c5529320,f,e8546cdc,c12e827b,c5529320,...) at sigexit+0x924/frame 0xe8546c08
    postsig(f,0,c1480f47,df,c13070fa,...) at postsig+0x2de/frame 0xe8546ca8
    ast(e8546ce8) at ast+0x538/frame 0xe8546cdc
    doreti_ast() at doreti_ast+0x1b/frame 0xbfbfed70
    
    Tracing command minicron pid 9794 tid 100144 td 0xc5529640
    sched_switch(c5529640,0,104,16,0,...) at sched_switch+0x2d2/frame 0xe8542960
    mi_switch(104,0,e8542a10,c0cf40f2,c1f85c80,c5529640) at mi_switch+0x122/frame 0xe8542998
    sleepq_switch(c5529640,0,c1480a56,1a8,0,...) at sleepq_switch+0x15b/frame 0xe85429c0
    sleepq_catch_signals(0,c1480a56,154,0,0,...) at sleepq_catch_signals+0x5be/frame 0xe8542a14
    sleepq_wait_sig(c5520c20,6c,c1481a73,100,0,...) at sleepq_wait_sig+0x14/frame 0xe8542a40
    _sleep(c5520c20,c5520cb8,16c,c1481a73,0,...) at _sleep+0x29b/frame 0xe8542a88
    kern_wait6(c5529640,7,0,0,e8542b24,...) at kern_wait6+0x71a/frame 0xe8542af8
    sys_wait4(c5529640,e8542ca8,c5520c20,c552a000,0,...) at sys_wait4+0x94/frame 0xe8542c18
    syscall(e8542ce8) at syscall+0x5c9/frame 0xe8542cdc
    Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xe8542cdc
    --- syscall (7, FreeBSD ELF32, sys_wait4), eip = 0x28130f43, esp = 0xbfbfed58, ebp = 0xbfbfed70 ---
    
    Tracing command check_reload_status pid 304 tid 100084 td 0xc47da000
    sched_switch(c47da000,0,104,0,0,...) at sched_switch+0x2d2/frame 0xe8fb08e8
    mi_switch(104,0,e8fb0998,c0cf40f2,c1f85c80,c47da000) at mi_switch+0x122/frame 0xe8fb0920
    sleepq_switch(c47da000,0,c1480a56,1a8,c47da000,...) at sleepq_switch+0x15b/frame 0xe8fb0948
    sleepq_catch_signals(0,c1480a56,154,102,0,...) at sleepq_catch_signals+0x5be/frame 0xe8fb099c
    sleepq_wait_sig(c4787c80,68,c1476bc8,100,0,...) at sleepq_wait_sig+0x14/frame 0xe8fb09c8
    _sleep(c4787c80,c4787c80,168,c1476bc8,0,...) at _sleep+0x29b/frame 0xe8fb0a10
    kern_kevent_fp(c47da000,c47ae968,0,1,e8fb0be8,...) at kern_kevent_fp+0x3bc/frame 0xe8fb0b48
    kern_kevent(c47da000,3,0,1,e8fb0be8,...) at kern_kevent+0xdf/frame 0xe8fb0b98
    sys_kevent(c47da000,e8fb0ca8,c47d6610,c47d2000,7ff6befe,...) at sys_kevent+0x131/frame 0xe8fb0c18
    syscall(e8fb0ce8) at syscall+0x5c9/frame 0xe8fb0cdc
    Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xe8fb0cdc
    --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2818175b, esp = 0xbfbfeb6c, ebp = 0xbfbfeb98 ---
    
    Tracing command check_reload_status pid 302 tid 100082 td 0xc47da640
    sched_switch(c47da640,0,104,0,c4400c80,...) at sched_switch+0x2d2/frame 0xe8faa8e4
    mi_switch(104,0,1055f8,10000000,0,c47da640) at mi_switch+0x122/frame 0xe8faa91c
    sleepq_switch(c47da640,0,c1480a56,1a8,0,...) at sleepq_switch+0x15b/frame 0xe8faa944
    sleepq_catch_signals(10000000,0,c0d5f550,c47da640,0,...) at sleepq_catch_signals+0x5be/frame 0xe8faa998
    sleepq_timedwait_sig(c4786980,68,1055f8,10000000,0,...) at sleepq_timedwait_sig+0x14/frame 0xe8faa9c8
    _sleep(c4786980,c4786980,168,c1476bc8,48a92bd3,...) at _sleep+0x26b/frame 0xe8faaa10
    kern_kevent_fp(c47da640,c47af348,0,40,e8faabe8,...) at kern_kevent_fp+0x3bc/frame 0xe8faab48
    kern_kevent(c47da640,4,0,40,e8faabe8,...) at kern_kevent+0xdf/frame 0xe8faab98
    sys_kevent(c47da640,e8faaca8,c47d6c20,e8faac64,c0d1bba2,...) at sys_kevent+0x131/frame 0xe8faac18
    syscall(e8faace8) at syscall+0x5c9/frame 0xe8faacdc
    Xint0x80_syscall() at Xint0x80_syscall+0x2f/frame 0xe8faacdc
    --- syscall (363, FreeBSD ELF32, sys_kevent), eip = 0x2818175b, esp = 0xbfbfea5c, ebp = 0xbfbfea88 ---
    
    Tracing command md0 pid 56 tid 100079 td 0xc47db000
    sched_switch(c47db000,0,104,e8fa1bb0,0,...) at sched_switch+0x2d2/frame 0xe8fa1b88
    mi_switch(104,0,c3673db0,c4776940,c47db000,c43d0000) at mi_switch+0x122/frame 0xe8fa1bc0
    sleepq_switch(c47db000,0,c1480a56,269,c46df158,...) at sleepq_switch+0x15b/frame 0xe8fa1be8
    sleepq_wait(c43d0000,5c,c1401bc5,0,0,...) at sleepq_wait+0x3f/frame 0xe8fa1c14
    _sleep(c43d0000,c43d0020,25c,c1401bc5,0,...) at _sleep+0x2ae/frame 0xe8fa1c5c
    md_kthread(c43d0000,e8fa1ce8,0,c47db000,e8fa1cdc,...) at md_kthread+0xd7/frame 0xe8fa1ca4
    fork_exit(c08726b0,c43d0000,e8fa1ce8) at fork_exit+0xa3/frame 0xe8fa1cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe8fa1cd4
    --- trap 0, eip = 0, esp = 0xe8fa1d20, ebp = 0 ---
    
    Tracing command vnlru pid 21 tid 100071 td 0xc4758640
    sched_switch(c4758640,0,104,0,c201d3a4,...) at sched_switch+0x2d2/frame 0xe84dcb5c
    mi_switch(104,0,101,c4758640,e84dcbe8,c4724918) at mi_switch+0x122/frame 0xe84dcb94
    sleepq_switch(c4758640,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xe84dcbbc
    sleepq_timedwait(c4724918,60,0,0,0,...) at sleepq_timedwait+0x3f/frame 0xe84dcbe8
    _sleep(c4724918,c1f98d7c,260,c1487d6c,fffffed8,...) at _sleep+0x282/frame 0xe84dcc30
    vnlru_proc(0,e84dcce8,0,0,0,...) at vnlru_proc+0xcf/frame 0xe84dcca4
    fork_exit(c0dc7f10,0,e84dcce8) at fork_exit+0xa3/frame 0xe84dccd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84dccd4
    --- trap 0, eip = 0, esp = 0xe84dcd20, ebp = 0 ---
    
    Tracing command syncer pid 20 tid 100070 td 0xc4758960
    sched_switch(c4758960,0,104,c201d3a4,0,...) at sched_switch+0x2d2/frame 0xe84d9b68
    mi_switch(104,0,101,c4758960,e84d9bf4,c1f98dc4) at mi_switch+0x122/frame 0xe84d9ba0
    sleepq_switch(c4758960,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xe84d9bc8
    sleepq_timedwait(c1f98dc4,0,0,0,0,...) at sleepq_timedwait+0x3f/frame 0xe84d9bf4
    _cv_timedwait_sbt(c1f98dc4,c1f98da4,fffffed8,0,0,...) at _cv_timedwait_sbt+0x1b7/frame 0xe84d9c28
    sched_sync(0,e84d9ce8,0,0,0,...) at sched_sync+0x7a1/frame 0xe84d9ca4
    fork_exit(c0dc75d0,0,e84d9ce8) at fork_exit+0xa3/frame 0xe84d9cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84d9cd4
    --- trap 0, eip = 0, esp = 0xe84d9d20, ebp = 0 ---
    
    Tracing command bufdaemon pid 19 tid 100069 td 0xc4758c80
    sched_switch(c4758c80,0,104,c201d3a4,0,...) at sched_switch+0x2d2/frame 0xe84d6b98
    mi_switch(104,0,101,c4758c80,e84d6c24,c1f98504) at mi_switch+0x122/frame 0xe84d6bd0
    sleepq_switch(c4758c80,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xe84d6bf8
    sleepq_timedwait(c1f98504,54,0,0,0,...) at sleepq_timedwait+0x3f/frame 0xe84d6c24
    _sleep(c1f98504,c1f98480,54,c1485734,1999997c,...) at _sleep+0x282/frame 0xe84d6c6c
    buf_daemon(0,e84d6ce8,0,0,0,...) at buf_daemon+0xac/frame 0xe84d6ca4
    fork_exit(c0dab150,0,e84d6ce8) at fork_exit+0xa3/frame 0xe84d6cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84d6cd4
    --- trap 0, eip = 0, esp = 0xe84d6d20, ebp = 0 ---
    
    Tracing command pagezero pid 18 tid 100068 td 0xc4759000
    sched_switch(c4759000,0,104,c201d3a4,0,...) at sched_switch+0x2d2/frame 0xe84d3ba0
    mi_switch(104,0,101,c4759000,e84d3c2c,c1fa2fa8) at mi_switch+0x122/frame 0xe84d3bd8
    sleepq_switch(c4759000,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xe84d3c00
    sleepq_timedwait(c1fa2fa8,0,12b,0,0,...) at sleepq_timedwait+0x3f/frame 0xe84d3c2c
    _sleep(c1fa2fa8,c2026380,0,c14b6fa6,fffea520,...) at _sleep+0x282/frame 0xe84d3c74
    vm_pagezero(0,e84d3ce8,0,0,0,...) at vm_pagezero+0xd2/frame 0xe84d3ca4
    fork_exit(c107ad10,0,e84d3ce8) at fork_exit+0xa3/frame 0xe84d3cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84d3cd4
    --- trap 0, eip = 0, esp = 0xe84d3d20, ebp = 0 ---
    
    Tracing command idlepoll pid 17 tid 100067 td 0xc4759320
    sched_switch(c4759320,0,104,0,c201d3a4,...) at sched_switch+0x2d2/frame 0xe84d0b8c
    mi_switch(104,0,101,c4759320,e84d0c18,c1f785f0) at mi_switch+0x122/frame 0xe84d0bc4
    sleepq_switch(c4759320,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xe84d0bec
    sleepq_timedwait(c1f785f0,0,2,0,0,...) at sleepq_timedwait+0x3f/frame 0xe84d0c18
    _sleep(c1f785f0,0,0,c147ada1,fffffc88,...) at _sleep+0x282/frame 0xe84d0c60
    poll_idle(0,e84d0ce8,0,0,0,...) at poll_idle+0x125/frame 0xe84d0ca4
    fork_exit(c0cf7de0,0,e84d0ce8) at fork_exit+0xa3/frame 0xe84d0cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84d0cd4
    --- trap 0, eip = 0, esp = 0xe84d0d20, ebp = 0 ---
    
    Tracing command vmdaemon pid 9 tid 100066 td 0xc4759640
    sched_switch(c4759640,0,104,0,0,...) at sched_switch+0x2d2/frame 0xe84cdb54
    mi_switch(104,0,0,0,c4759640,c1fa2cc4) at mi_switch+0x122/frame 0xe84cdb8c
    sleepq_switch(c4759640,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xe84cdbb4
    sleepq_wait(c1fa2cc4,74,c148572d,0,0,...) at sleepq_wait+0x3f/frame 0xe84cdbe0
    _sleep(c1fa2cc4,c1fa2cac,74,c148572d,0,...) at _sleep+0x2ae/frame 0xe84cdc28
    vm_daemon(0,e84cdce8,0,0,0,...) at vm_daemon+0x130/frame 0xe84cdca4
    fork_exit(c1072d80,0,e84cdce8) at fork_exit+0xa3/frame 0xe84cdcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84cdcd4
    --- trap 0, eip = 0, esp = 0xe84cdd20, ebp = 0 ---
    
    Tracing command pagedaemon pid 8 tid 100072 td 0xc4758320
    sched_switch(c4758320,0,104,0,0,...) at sched_switch+0x2d2/frame 0xe8f86ba4
    mi_switch(104,0,0,0,c4758320,c1fa28a4) at mi_switch+0x122/frame 0xe8f86bdc
    sleepq_switch(c4758320,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xe8f86c04
    sleepq_wait(c1fa28a4,54,c14b4ca3,0,0,...) at sleepq_wait+0x3f/frame 0xe8f86c30
    _sleep(c1fa28a4,c1fa2890,54,c14b4ca3,0,...) at _sleep+0x2ae/frame 0xe8f86c78
    uma_reclaim_worker(0,e8f86ce8,0,0,0,...) at uma_reclaim_worker+0xb0/frame 0xe8f86ca4
    fork_exit(c10545c0,0,e8f86ce8) at fork_exit+0xa3/frame 0xe8f86cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe8f86cd4
    --- trap 0, eip = 0, esp = 0xe8f86d20, ebp = 0 ---
    
    Tracing command pagedaemon pid 8 tid 100065 td 0xc4431640
    sched_switch(c4431640,0,104,c201d3a4,0,...) at sched_switch+0x2d2/frame 0xe84cab60
    mi_switch(104,0,101,c4431640,e84cabec,c2026404) at mi_switch+0x122/frame 0xe84cab98
    sleepq_switch(c4431640,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xe84cabc0
    sleepq_timedwait(c2026404,54,0,0,0,...) at sleepq_timedwait+0x3f/frame 0xe84cabec
    _sleep(c2026404,c2026380,54,c148572d,fffffed8,...) at _sleep+0x282/frame 0xe84cac34
    vm_pageout(0,e84cace8,0,0,0,...) at vm_pageout+0x23b/frame 0xe84caca4
    fork_exit(c1073b20,0,e84cace8) at fork_exit+0xa3/frame 0xe84cacd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84cacd4
    --- trap 0, eip = 0, esp = 0xe84cad20, ebp = 0 ---
    
    Tracing command sctp_iterator pid 7 tid 100061 td 0xc4495320
    sched_switch(c4495320,0,104,0,d4784fc8,...) at sched_switch+0x2d2/frame 0xe84beba4
    mi_switch(104,0,c26e8bf0,d4bfcfc8,c4495320,c2020d74) at mi_switch+0x122/frame 0xe84bebdc
    sleepq_switch(c4495320,0,c1480a56,269,c4bfc000,...) at sleepq_switch+0x15b/frame 0xe84bec04
    sleepq_wait(c2020d74,0,c149be9f,0,0,...) at sleepq_wait+0x3f/frame 0xe84bec30
    _sleep(c2020d74,c2020d3c,0,c149be9f,0,...) at _sleep+0x2ae/frame 0xe84bec78
    sctp_iterator_thread(0,e84bece8,0,0,0,...) at sctp_iterator_thread+0x9c/frame 0xe84beca4
    fork_exit(c0ea87d0,0,e84bece8) at fork_exit+0xa3/frame 0xe84becd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84becd4
    --- trap 0, eip = 0, esp = 0xe84bed20, ebp = 0 ---
    
    Tracing command pf purge pid 6 tid 100059 td 0xc4495960
    sched_switch(c4495960,0,104,c201d3a4,0,...) at sched_switch+0x2d2/frame 0xe84b8b98
    mi_switch(104,0,101,c4495960,e84b8c24,c0f94ba0) at mi_switch+0x122/frame 0xe84b8bd0
    sleepq_switch(c4495960,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xe84b8bf8
    sleepq_timedwait(c0f94ba0,0,0,0,0,...) at sleepq_timedwait+0x3f/frame 0xe84b8c24
    _sleep(c0f94ba0,c2023cf4,0,c14a7eb3,1999997c,...) at _sleep+0x282/frame 0xe84b8c6c
    pf_purge_thread(0,e84b8ce8,0,0,0,...) at pf_purge_thread+0xba/frame 0xe84b8ca4
    fork_exit(c0f94ba0,0,e84b8ce8) at fork_exit+0xa3/frame 0xe84b8cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84b8cd4
    --- trap 0, eip = 0, esp = 0xe84b8d20, ebp = 0 ---
    
    Tracing command fdc0 pid 5 tid 100055 td 0xc4496640
    sched_switch(c4496640,0,104,0,c201d3a4,...) at sched_switch+0x2d2/frame 0xc3ffab34
    mi_switch(104,0,101,c4496640,c3ffabc0,c4314c3c) at mi_switch+0x122/frame 0xc3ffab6c
    sleepq_switch(c4496640,0,c1480a56,28c,101,...) at sleepq_switch+0x15b/frame 0xc3ffab94
    sleepq_timedwait(c4314c3c,5c,0,0,0,...) at sleepq_timedwait+0x3f/frame 0xc3ffabc0
    _sleep(c4314c3c,c4314cf0,5c,c1467e28,fffffed8,...) at _sleep+0x282/frame 0xc3ffac08
    fdc_thread(c4314c00,c3fface8,0,0,0,...) at fdc_thread+0x877/frame 0xc3ffaca4
    fork_exit(c124d2e0,c4314c00,c3fface8) at fork_exit+0xa3/frame 0xc3ffacd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3ffacd4
    --- trap 0, eip = 0, esp = 0xc3ffad20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100048 td 0xc442f960
    sched_switch(c442f960,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc37b3bb8
    mi_switch(104,0,0,0,c442f960,c443ad64) at mi_switch+0x122/frame 0xc37b3bf0
    sleepq_switch(c442f960,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc37b3c18
    sleepq_wait(c443ad64,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc37b3c44
    _cv_wait(c443ad64,c443aed0,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc37b3c74
    usb_process(c443ad5c,c37b3ce8,0,0,0,...) at usb_process+0xf6/frame 0xc37b3ca4
    fork_exit(c0ab1850,c443ad5c,c37b3ce8) at fork_exit+0xa3/frame 0xc37b3cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc37b3cd4
    --- trap 0, eip = 0, esp = 0xc37b3d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100047 td 0xc442fc80
    sched_switch(c442fc80,0,104,40006,0,...) at sched_switch+0x2d2/frame 0xc37b0bb8
    mi_switch(104,0,8,28,c442fc80,c443ad34) at mi_switch+0x122/frame 0xc37b0bf0
    sleepq_switch(c442fc80,0,c1480a56,269,500,...) at sleepq_switch+0x15b/frame 0xc37b0c18
    sleepq_wait(c443ad34,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc37b0c44
    _cv_wait(c443ad34,c443aed0,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc37b0c74
    usb_process(c443ad2c,c37b0ce8,0,0,0,...) at usb_process+0xf6/frame 0xc37b0ca4
    fork_exit(c0ab1850,c443ad2c,c37b0ce8) at fork_exit+0xa3/frame 0xc37b0cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc37b0cd4
    --- trap 0, eip = 0, esp = 0xc37b0d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100046 td 0xc4430000
    sched_switch(c4430000,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc37adbb8
    mi_switch(104,0,0,0,c4430000,c443ad04) at mi_switch+0x122/frame 0xc37adbf0
    sleepq_switch(c4430000,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc37adc18
    sleepq_wait(c443ad04,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc37adc44
    _cv_wait(c443ad04,c443aed0,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc37adc74
    usb_process(c443acfc,c37adce8,0,0,0,...) at usb_process+0xf6/frame 0xc37adca4
    fork_exit(c0ab1850,c443acfc,c37adce8) at fork_exit+0xa3/frame 0xc37adcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc37adcd4
    --- trap 0, eip = 0, esp = 0xc37add20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100045 td 0xc4430320
    sched_switch(c4430320,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc37aabb8
    mi_switch(104,0,0,0,c4430320,c443acd4) at mi_switch+0x122/frame 0xc37aabf0
    sleepq_switch(c4430320,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc37aac18
    sleepq_wait(c443acd4,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc37aac44
    _cv_wait(c443acd4,c443aed0,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc37aac74
    usb_process(c443accc,c37aace8,0,0,0,...) at usb_process+0xf6/frame 0xc37aaca4
    fork_exit(c0ab1850,c443accc,c37aace8) at fork_exit+0xa3/frame 0xc37aacd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc37aacd4
    --- trap 0, eip = 0, esp = 0xc37aad20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100044 td 0xc4430640
    sched_switch(c4430640,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc37a7bb8
    mi_switch(104,0,0,0,c4430640,c443aca4) at mi_switch+0x122/frame 0xc37a7bf0
    sleepq_switch(c4430640,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc37a7c18
    sleepq_wait(c443aca4,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc37a7c44
    _cv_wait(c443aca4,c443aed0,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc37a7c74
    usb_process(c443ac9c,c37a7ce8,0,0,0,...) at usb_process+0xf6/frame 0xc37a7ca4
    fork_exit(c0ab1850,c443ac9c,c37a7ce8) at fork_exit+0xa3/frame 0xc37a7cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc37a7cd4
    --- trap 0, eip = 0, esp = 0xc37a7d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100042 td 0xc4430c80
    sched_switch(c4430c80,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc379fbb8
    mi_switch(104,0,0,0,c4430c80,c441addc) at mi_switch+0x122/frame 0xc379fbf0
    sleepq_switch(c4430c80,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc379fc18
    sleepq_wait(c441addc,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc379fc44
    _cv_wait(c441addc,c441af48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc379fc74
    usb_process(c441add4,c379fce8,0,0,0,...) at usb_process+0xf6/frame 0xc379fca4
    fork_exit(c0ab1850,c441add4,c379fce8) at fork_exit+0xa3/frame 0xc379fcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc379fcd4
    --- trap 0, eip = 0, esp = 0xc379fd20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100041 td 0xc4431000
    sched_switch(c4431000,0,104,40002,0,...) at sched_switch+0x2d2/frame 0xc379cbb8
    mi_switch(104,0,c0aa3a12,c201c7b4,c4431000,c441adac) at mi_switch+0x122/frame 0xc379cbf0
    sleepq_switch(c4431000,0,c1480a56,269,100,...) at sleepq_switch+0x15b/frame 0xc379cc18
    sleepq_wait(c441adac,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc379cc44
    _cv_wait(c441adac,c441af48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc379cc74
    usb_process(c441ada4,c379cce8,0,0,0,...) at usb_process+0xf6/frame 0xc379cca4
    fork_exit(c0ab1850,c441ada4,c379cce8) at fork_exit+0xa3/frame 0xc379ccd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc379ccd4
    --- trap 0, eip = 0, esp = 0xc379cd20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100040 td 0xc4431320
    sched_switch(c4431320,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc3799bb8
    mi_switch(104,0,0,0,c4431320,c441ad7c) at mi_switch+0x122/frame 0xc3799bf0
    sleepq_switch(c4431320,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc3799c18
    sleepq_wait(c441ad7c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc3799c44
    _cv_wait(c441ad7c,c441af48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc3799c74
    usb_process(c441ad74,c3799ce8,0,0,0,...) at usb_process+0xf6/frame 0xc3799ca4
    fork_exit(c0ab1850,c441ad74,c3799ce8) at fork_exit+0xa3/frame 0xc3799cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3799cd4
    --- trap 0, eip = 0, esp = 0xc3799d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100039 td 0xc43a8320
    sched_switch(c43a8320,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc3796bb8
    mi_switch(104,0,0,0,c43a8320,c441ad4c) at mi_switch+0x122/frame 0xc3796bf0
    sleepq_switch(c43a8320,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc3796c18
    sleepq_wait(c441ad4c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc3796c44
    _cv_wait(c441ad4c,c441af48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc3796c74
    usb_process(c441ad44,c3796ce8,0,0,0,...) at usb_process+0xf6/frame 0xc3796ca4
    fork_exit(c0ab1850,c441ad44,c3796ce8) at fork_exit+0xa3/frame 0xc3796cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3796cd4
    --- trap 0, eip = 0, esp = 0xc3796d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100038 td 0xc43a8640
    sched_switch(c43a8640,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc3793bb8
    mi_switch(104,0,0,0,c43a8640,c441ad1c) at mi_switch+0x122/frame 0xc3793bf0
    sleepq_switch(c43a8640,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc3793c18
    sleepq_wait(c441ad1c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc3793c44
    _cv_wait(c441ad1c,c441af48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc3793c74
    usb_process(c441ad14,c3793ce8,0,0,0,...) at usb_process+0xf6/frame 0xc3793ca4
    fork_exit(c0ab1850,c441ad14,c3793ce8) at fork_exit+0xa3/frame 0xc3793cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3793cd4
    --- trap 0, eip = 0, esp = 0xc3793d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100036 td 0xc43a8c80
    sched_switch(c43a8c80,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc378dbb8
    mi_switch(104,0,0,0,c43a8c80,c4405ddc) at mi_switch+0x122/frame 0xc378dbf0
    sleepq_switch(c43a8c80,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc378dc18
    sleepq_wait(c4405ddc,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xc378dc44
    _cv_wait(c4405ddc,c4405f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xc378dc74
    usb_process(c4405dd4,c378dce8,0,0,0,...) at usb_process+0xf6/frame 0xc378dca4
    fork_exit(c0ab1850,c4405dd4,c378dce8) at fork_exit+0xa3/frame 0xc378dcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc378dcd4
    --- trap 0, eip = 0, esp = 0xc378dd20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100035 td 0xc43ff000
    sched_switch(c43ff000,0,104,40002,0,...) at sched_switch+0x2d2/frame 0xddfffbb8
    mi_switch(104,0,c4405cf0,1400,c43ff000,c4405dac) at mi_switch+0x122/frame 0xddfffbf0
    sleepq_switch(c43ff000,0,c1480a56,269,100,...) at sleepq_switch+0x15b/frame 0xddfffc18
    sleepq_wait(c4405dac,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddfffc44
    _cv_wait(c4405dac,c4405f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddfffc74
    usb_process(c4405da4,ddfffce8,0,0,0,...) at usb_process+0xf6/frame 0xddfffca4
    fork_exit(c0ab1850,c4405da4,ddfffce8) at fork_exit+0xa3/frame 0xddfffcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddfffcd4
    --- trap 0, eip = 0, esp = 0xddfffd20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100034 td 0xc43ff320
    sched_switch(c43ff320,0,104,0,0,...) at sched_switch+0x2d2/frame 0xddffcbb8
    mi_switch(104,0,0,0,c43ff320,c4405d7c) at mi_switch+0x122/frame 0xddffcbf0
    sleepq_switch(c43ff320,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xddffcc18
    sleepq_wait(c4405d7c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddffcc44
    _cv_wait(c4405d7c,c4405f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddffcc74
    usb_process(c4405d74,ddffcce8,0,0,0,...) at usb_process+0xf6/frame 0xddffcca4
    fork_exit(c0ab1850,c4405d74,ddffcce8) at fork_exit+0xa3/frame 0xddffccd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddffccd4
    --- trap 0, eip = 0, esp = 0xddffcd20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100033 td 0xc43ff640
    sched_switch(c43ff640,0,104,0,0,...) at sched_switch+0x2d2/frame 0xddff9bb8
    mi_switch(104,0,0,0,c43ff640,c4405d4c) at mi_switch+0x122/frame 0xddff9bf0
    sleepq_switch(c43ff640,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xddff9c18
    sleepq_wait(c4405d4c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddff9c44
    _cv_wait(c4405d4c,c4405f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddff9c74
    usb_process(c4405d44,ddff9ce8,0,0,0,...) at usb_process+0xf6/frame 0xddff9ca4
    fork_exit(c0ab1850,c4405d44,ddff9ce8) at fork_exit+0xa3/frame 0xddff9cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddff9cd4
    --- trap 0, eip = 0, esp = 0xddff9d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100032 td 0xc43ff960
    sched_switch(c43ff960,0,104,0,0,...) at sched_switch+0x2d2/frame 0xddff6bb8
    mi_switch(104,0,0,0,c43ff960,c4405d1c) at mi_switch+0x122/frame 0xddff6bf0
    sleepq_switch(c43ff960,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xddff6c18
    sleepq_wait(c4405d1c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddff6c44
    _cv_wait(c4405d1c,c4405f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddff6c74
    usb_process(c4405d14,ddff6ce8,0,0,0,...) at usb_process+0xf6/frame 0xddff6ca4
    fork_exit(c0ab1850,c4405d14,ddff6ce8) at fork_exit+0xa3/frame 0xddff6cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddff6cd4
    --- trap 0, eip = 0, esp = 0xddff6d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100030 td 0xc4400000
    sched_switch(c4400000,0,104,0,0,...) at sched_switch+0x2d2/frame 0xddff0bb8
    mi_switch(104,0,0,0,c4400000,c43d7ddc) at mi_switch+0x122/frame 0xddff0bf0
    sleepq_switch(c4400000,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xddff0c18
    sleepq_wait(c43d7ddc,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddff0c44
    _cv_wait(c43d7ddc,c43d7f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddff0c74
    usb_process(c43d7dd4,ddff0ce8,0,0,0,...) at usb_process+0xf6/frame 0xddff0ca4
    fork_exit(c0ab1850,c43d7dd4,ddff0ce8) at fork_exit+0xa3/frame 0xddff0cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddff0cd4
    --- trap 0, eip = 0, esp = 0xddff0d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100029 td 0xc4400320
    sched_switch(c4400320,0,104,40002,0,...) at sched_switch+0x2d2/frame 0xddfedbb8
    mi_switch(104,0,c0aa3a12,c201c7b4,c4400320,c43d7dac) at mi_switch+0x122/frame 0xddfedbf0
    sleepq_switch(c4400320,0,c1480a56,269,100,...) at sleepq_switch+0x15b/frame 0xddfedc18
    sleepq_wait(c43d7dac,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddfedc44
    _cv_wait(c43d7dac,c43d7f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddfedc74
    usb_process(c43d7da4,ddfedce8,0,0,0,...) at usb_process+0xf6/frame 0xddfedca4
    fork_exit(c0ab1850,c43d7da4,ddfedce8) at fork_exit+0xa3/frame 0xddfedcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddfedcd4
    --- trap 0, eip = 0, esp = 0xddfedd20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100028 td 0xc4400640
    sched_switch(c4400640,0,104,0,0,...) at sched_switch+0x2d2/frame 0xddfeabb8
    mi_switch(104,0,0,0,c4400640,c43d7d7c) at mi_switch+0x122/frame 0xddfeabf0
    sleepq_switch(c4400640,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xddfeac18
    sleepq_wait(c43d7d7c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddfeac44
    _cv_wait(c43d7d7c,c43d7f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddfeac74
    usb_process(c43d7d74,ddfeace8,0,0,0,...) at usb_process+0xf6/frame 0xddfeaca4
    fork_exit(c0ab1850,c43d7d74,ddfeace8) at fork_exit+0xa3/frame 0xddfeacd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddfeacd4
    --- trap 0, eip = 0, esp = 0xddfead20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100027 td 0xc4400960
    sched_switch(c4400960,0,104,0,0,...) at sched_switch+0x2d2/frame 0xddfe7bb8
    mi_switch(104,0,0,0,c4400960,c43d7d4c) at mi_switch+0x122/frame 0xddfe7bf0
    sleepq_switch(c4400960,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xddfe7c18
    sleepq_wait(c43d7d4c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddfe7c44
    _cv_wait(c43d7d4c,c43d7f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddfe7c74
    usb_process(c43d7d44,ddfe7ce8,0,0,0,...) at usb_process+0xf6/frame 0xddfe7ca4
    fork_exit(c0ab1850,c43d7d44,ddfe7ce8) at fork_exit+0xa3/frame 0xddfe7cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddfe7cd4
    --- trap 0, eip = 0, esp = 0xddfe7d20, ebp = 0 ---
    
    Tracing command usb pid 16 tid 100026 td 0xc4298960
    sched_switch(c4298960,0,104,0,0,...) at sched_switch+0x2d2/frame 0xddfe4bb8
    mi_switch(104,0,0,0,c4298960,c43d7d1c) at mi_switch+0x122/frame 0xddfe4bf0
    sleepq_switch(c4298960,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xddfe4c18
    sleepq_wait(c43d7d1c,0,c1467e28,1,0,...) at sleepq_wait+0x3f/frame 0xddfe4c44
    _cv_wait(c43d7d1c,c43d7f48,c14490cd,73,0,...) at _cv_wait+0x192/frame 0xddfe4c74
    usb_process(c43d7d14,ddfe4ce8,0,0,0,...) at usb_process+0xf6/frame 0xddfe4ca4
    fork_exit(c0ab1850,c43d7d14,ddfe4ce8) at fork_exit+0xa3/frame 0xddfe4cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddfe4cd4
    --- trap 0, eip = 0, esp = 0xddfe4d20, ebp = 0 ---
    
    Tracing command cam pid 4 tid 100064 td 0xc4431960
    sched_switch(c4431960,0,104,0,0,...) at sched_switch+0x2d2/frame 0xe84c7b7c
    mi_switch(104,0,0,0,c4431960,c1eb7cac) at mi_switch+0x122/frame 0xe84c7bb4
    sleepq_switch(c4431960,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xe84c7bdc
    sleepq_wait(c1eb7cac,5c,c1467e28,0,0,...) at sleepq_wait+0x3f/frame 0xe84c7c08
    _sleep(c1eb7cac,c1eb7cfc,5c,c1467e28,0,...) at _sleep+0x2ae/frame 0xe84c7c50
    xpt_scanner_thread(0,e84c7ce8,0,0,0,...) at xpt_scanner_thread+0xcc/frame 0xe84c7ca4
    fork_exit(c04d0120,0,e84c7ce8) at fork_exit+0xa3/frame 0xe84c7cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84c7cd4
    --- trap 0, eip = 0, esp = 0xe84c7d20, ebp = 0 ---
    
    Tracing command cam pid 4 tid 100017 td 0xc43a7640
    sched_switch(c43a7640,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc3770b7c
    mi_switch(104,0,0,c3770c18,c43a7640,c1eb7e00) at mi_switch+0x122/frame 0xc3770bb4
    sleepq_switch(c43a7640,0,c1480a56,269,c4753000,...) at sleepq_switch+0x15b/frame 0xc3770bdc
    sleepq_wait(c1eb7e00,5c,c1467e28,0,0,...) at sleepq_wait+0x3f/frame 0xc3770c08
    _sleep(c1eb7e00,c1eb7d80,5c,c1467e28,0,...) at _sleep+0x2ae/frame 0xc3770c50
    xpt_done_td(c1eb7d80,c3770ce8,0,0,0,...) at xpt_done_td+0xce/frame 0xc3770ca4
    fork_exit(c04cfd00,c1eb7d80,c3770ce8) at fork_exit+0xa3/frame 0xc3770cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3770cd4
    --- trap 0, eip = 0, esp = 0xc3770d20, ebp = 0 ---
    
    Tracing command rand_harvestq pid 15 tid 100014 td 0xc43a8000
    sched_switch(c43a8000,0,104,0,c201d3a4,...) at sched_switch+0x2d2/frame 0xc3767b94
    mi_switch(104,0,5,c43a8000,c3767c20,c1ed4ec8) at mi_switch+0x122/frame 0xc3767bcc
    sleepq_switch(c43a8000,0,c1480a56,28c,5,...) at sleepq_switch+0x15b/frame 0xc3767bf4
    sleepq_timedwait(c1ed4ec8,0,0,0,0,...) at sleepq_timedwait+0x3f/frame 0xc3767c20
    msleep_spin_sbt(c1ed4ec8,c2014ce0,c1467e28,19999999,0,...) at msleep_spin_sbt+0x1c0/frame 0xc3767c58
    random_kthread(c0989220,c3767ce8,0,0,0,...) at random_kthread+0x2a2/frame 0xc3767ca4
    fork_exit(c09883d0,c0989220,c3767ce8) at fork_exit+0xa3/frame 0xc3767cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3767cd4
    --- trap 0, eip = 0, esp = 0xc3767d20, ebp = 0 ---
    
    Tracing command crypto returns pid 3 tid 100013 td 0xc4292000
    sched_switch(c4292000,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc3762b84
    mi_switch(104,0,0,0,c4292000,c1fa1c28) at mi_switch+0x122/frame 0xc3762bbc
    sleepq_switch(c4292000,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc3762be4
    sleepq_wait(c1fa1c28,6c,c14ac22d,0,0,...) at sleepq_wait+0x3f/frame 0xc3762c10
    _sleep(c1fa1c28,c1fa1c14,6c,c14ac22d,0,...) at _sleep+0x2ae/frame 0xc3762c58
    crypto_ret_proc(0,c3762ce8,0,0,0,...) at crypto_ret_proc+0x20a/frame 0xc3762ca4
    fork_exit(c0fd4c30,0,c3762ce8) at fork_exit+0xa3/frame 0xc3762cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3762cd4
    --- trap 0, eip = 0, esp = 0xc3762d20, ebp = 0 ---
    
    Tracing command crypto pid 2 tid 100012 td 0xc4292320
    sched_switch(c4292320,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc375fb90
    mi_switch(104,0,0,0,c4292320,c1fa1b68) at mi_switch+0x122/frame 0xc375fbc8
    sleepq_switch(c4292320,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc375fbf0
    sleepq_wait(c1fa1b68,6c,c14ac23d,0,0,...) at sleepq_wait+0x3f/frame 0xc375fc1c
    _sleep(c1fa1b68,c1fa1b50,6c,c14ac23d,0,...) at _sleep+0x2ae/frame 0xc375fc64
    crypto_proc(0,c375fce8,0,0,0,...) at crypto_proc+0x2e8/frame 0xc375fca4
    fork_exit(c0fd48e0,0,c375fce8) at fork_exit+0xa3/frame 0xc375fcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc375fcd4
    --- trap 0, eip = 0, esp = 0xc375fd20, ebp = 0 ---
    
    Tracing command geom pid 14 tid 100010 td 0xc4292960
    sched_switch(c4292960,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc3759b80
    mi_switch(104,0,c4292960,c3759bd8,c4292960,c201bdd8) at mi_switch+0x122/frame 0xc3759bb8
    sleepq_switch(c4292960,0,c1480a56,269,c4292960,...) at sleepq_switch+0x15b/frame 0xc3759be0
    sleepq_wait(c201bdd8,5c,c1467e28,0,0,...) at sleepq_wait+0x3f/frame 0xc3759c0c
    _sleep(c201bdd8,c1f75c88,25c,c1467e28,0,...) at _sleep+0x2ae/frame 0xc3759c54
    g_io_schedule_down(c4292960,5c,c146addd,6b,c3759cd4,...) at g_io_schedule_down+0x5c/frame 0xc3759c8c
    g_down_procbody(0,c3759ce8,0,0,0,...) at g_down_procbody+0x6d/frame 0xc3759ca4
    fork_exit(c0c56dd0,0,c3759ce8) at fork_exit+0xa3/frame 0xc3759cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3759cd4
    --- trap 0, eip = 0, esp = 0xc3759d20, ebp = 0 ---
    
    Tracing command geom pid 14 tid 100009 td 0xc4292c80
    sched_switch(c4292c80,0,104,c41b7e40,0,...) at sched_switch+0x2d2/frame 0xc3756b80
    mi_switch(104,0,c201bdd4,c4292c80,c4292c80,c201bdd4) at mi_switch+0x122/frame 0xc3756bb8
    sleepq_switch(c4292c80,0,c1480a56,269,0,...) at sleepq_switch+0x15b/frame 0xc3756be0
    sleepq_wait(c201bdd4,5c,c1467e28,0,0,...) at sleepq_wait+0x3f/frame 0xc3756c0c
    _sleep(c201bdd4,c1f75ca8,25c,c1467e28,0,...) at _sleep+0x2ae/frame 0xc3756c54
    g_io_schedule_up(c4292c80,5c,c146addd,5e,c3756cd4,...) at g_io_schedule_up+0xd5/frame 0xc3756c8c
    g_up_procbody(0,c3756ce8,0,0,0,...) at g_up_procbody+0x6d/frame 0xc3756ca4
    fork_exit(c0c56d60,0,c3756ce8) at fork_exit+0xa3/frame 0xc3756cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3756cd4
    --- trap 0, eip = 0, esp = 0xc3756d20, ebp = 0 ---
    
    Tracing command geom pid 14 tid 100008 td 0xc4297000
    sched_switch(c4297000,0,104,0,c4757930,...) at sched_switch+0x2d2/frame 0xc3753b94
    mi_switch(104,0,c0d1bc92,c4757640,c4297000,c201bdd0) at mi_switch+0x122/frame 0xc3753bcc
    sleepq_switch(c4297000,0,c1480a56,269,c3753c18,...) at sleepq_switch+0x15b/frame 0xc3753bf4
    sleepq_wait(c201bdd0,5c,c1467e28,0,0,...) at sleepq_wait+0x3f/frame 0xc3753c20
    _sleep(c201bdd0,c1f75c60,25c,c1467e28,0,...) at _sleep+0x2ae/frame 0xc3753c68
    g_run_events(0,c3753ce8,0,0,0,...) at g_run_events+0x62/frame 0xc3753ca4
    fork_exit(c0c56cf0,0,c3753ce8) at fork_exit+0xa3/frame 0xc3753cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3753cd4
    --- trap 0, eip = 0, esp = 0xc3753d20, ebp = 0 ---
    
    Tracing command ng_queue pid 13 tid 100007 td 0xc4297320
    sched_switch(c4297320,0,104,0,0,...) at sched_switch+0x2d2/frame 0xc3750b90
    mi_switch(104,0,c1052dad,c0e51e01,c4297320,c1e8f63c) at mi_switch+0x122/frame 0xc3750bc8
    sleepq_switch(c4297320,0,c1480a56,269,c1e8f488,...) at sleepq_switch+0x15b/frame 0xc3750bf0
    sleepq_wait(c1e8f63c,8,c1485ab5,0,0,...) at sleepq_wait+0x3f/frame 0xc3750c1c
    _sleep(c1e8f63c,c1f9b8c0,8,c1485ab5,0,...) at _sleep+0x2ae/frame 0xc3750c64
    ngthread(0,c3750ce8,0,0,0,...) at ngthread+0xbc/frame 0xc3750ca4
    fork_exit(c0e55280,0,c3750ce8) at fork_exit+0xa3/frame 0xc3750cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3750cd4
    --- trap 0, eip = 0, esp = 0xc3750d20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100062 td 0xc4495000
    fork_trampoline() at fork_trampoline
    
    Tracing command intr pid 12 tid 100060 td 0xc4495640
    sched_switch(c4495640,0,109,0,0,...) at sched_switch+0x2d2/frame 0xe84bbc30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xe84bbc68
    ithread_loop(c44a8ed0,e84bbce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xe84bbca4
    fork_exit(c0cd81d0,c44a8ed0,e84bbce8) at fork_exit+0xa3/frame 0xe84bbcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xe84bbcd4
    --- trap 0, eip = 0, esp = 0xe84bbd20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100057 td 0xc4496000
    fork_trampoline() at fork_trampoline
    
    Tracing command intr pid 12 tid 100056 td 0xc4496320
    fork_trampoline() at fork_trampoline
    
    Tracing command intr pid 12 tid 100054 td 0xc4496960
    fork_trampoline() at fork_trampoline
    
    Tracing command intr pid 12 tid 100053 td 0xc4496c80
    sched_switch(c4496c80,0,109,c1303a7a,c4295280,...) at sched_switch+0x2d2/frame 0xc3fe0c30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc3fe0c68
    ithread_loop(c4474360,c3fe0ce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc3fe0ca4
    fork_exit(c0cd81d0,c4474360,c3fe0ce8) at fork_exit+0xa3/frame 0xc3fe0cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3fe0cd4
    --- trap 0, eip = 0, esp = 0xc3fe0d20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100052 td 0xc4400c80
    sched_switch(c4400c80,0,109,c1303a7a,0,...) at sched_switch+0x2d2/frame 0xc3fbbc30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc3fbbc68
    ithread_loop(c44744d0,c3fbbce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc3fbbca4
    fork_exit(c0cd81d0,c44744d0,c3fbbce8) at fork_exit+0xa3/frame 0xc3fbbcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3fbbcd4
    --- trap 0, eip = 0, esp = 0xc3fbbd20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100049 td 0xc442f640
    fork_trampoline() at fork_trampoline
    
    Tracing command intr pid 12 tid 100043 td 0xc4430960
    sched_switch(c4430960,0,109,c1303a7a,0,...) at sched_switch+0x2d2/frame 0xc37a4c30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc37a4c68
    ithread_loop(c43cb240,c37a4ce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc37a4ca4
    fork_exit(c0cd81d0,c43cb240,c37a4ce8) at fork_exit+0xa3/frame 0xc37a4cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc37a4cd4
    --- trap 0, eip = 0, esp = 0xc37a4d20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100037 td 0xc43a8960
    sched_switch(c43a8960,0,109,c1303a7a,0,...) at sched_switch+0x2d2/frame 0xc3790c30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc3790c68
    ithread_loop(c43cb530,c3790ce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc3790ca4
    fork_exit(c0cd81d0,c43cb530,c3790ce8) at fork_exit+0xa3/frame 0xc3790cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3790cd4
    --- trap 0, eip = 0, esp = 0xc3790d20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100031 td 0xc43ffc80
    sched_switch(c43ffc80,0,109,c1303a7a,0,...) at sched_switch+0x2d2/frame 0xddff3c30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xddff3c68
    ithread_loop(c43cb790,ddff3ce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xddff3ca4
    fork_exit(c0cd81d0,c43cb790,ddff3ce8) at fork_exit+0xa3/frame 0xddff3cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xddff3cd4
    --- trap 0, eip = 0, esp = 0xddff3d20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100025 td 0xc4298c80
    fork_trampoline() at fork_trampoline
    
    Tracing command intr pid 12 tid 100024 td 0xc43a6000
    sched_switch(c43a6000,0,109,0,0,...) at sched_switch+0x2d2/frame 0xc3785c30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc3785c68
    ithread_loop(c41bbd20,c3785ce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc3785ca4
    fork_exit(c0cd81d0,c41bbd20,c3785ce8) at fork_exit+0xa3/frame 0xc3785cd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc3785cd4
    --- trap 0, eip = 0, esp = 0xc3785d20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100022 td 0xc43a6640
    sched_switch(c43a6640,0,109,0,0,...) at sched_switch+0x2d2/frame 0xc377fc30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc377fc68
    ithread_loop(c41bbd40,c377fce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc377fca4
    fork_exit(c0cd81d0,c41bbd40,c377fce8) at fork_exit+0xa3/frame 0xc377fcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc377fcd4
    --- trap 0, eip = 0, esp = 0xc377fd20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100016 td 0xc43a7960
    sched_switch(c43a7960,0,109,0,0,...) at sched_switch+0x2d2/frame 0xc376dc30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc376dc68
    ithread_loop(c41bbda0,c376dce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc376dca4
    fork_exit(c0cd81d0,c41bbda0,c376dce8) at fork_exit+0xa3/frame 0xc376dcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc376dcd4
    --- trap 0, eip = 0, esp = 0xc376dd20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100006 td 0xc4297640
    sched_switch(c4297640,0,109,c201d300,0,...) at sched_switch+0x2d2/frame 0xc374dc30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc374dc68
    ithread_loop(c41ba080,c374dce8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc374dca4
    fork_exit(c0cd81d0,c41ba080,c374dce8) at fork_exit+0xa3/frame 0xc374dcd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc374dcd4
    --- trap 0, eip = 0, esp = 0xc374dd20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100005 td 0xc4297960
    sched_switch(c4297960,0,109,c2427880,0,...) at sched_switch+0x2d2/frame 0xc374ac30
    mi_switch(109,0,c14772d3,55b,0,...) at mi_switch+0x122/frame 0xc374ac68
    ithread_loop(c41ba090,c374ace8,0,0,0,...) at ithread_loop+0x1b1/frame 0xc374aca4
    fork_exit(c0cd81d0,c41ba090,c374ace8) at fork_exit+0xa3/frame 0xc374acd4
    fork_trampoline() at fork_trampoline+0x8/frame 0xc374acd4
    --- trap 0, eip = 0, esp = 0xc374ad20, ebp = 0 ---
    
    Tracing command intr pid 12 tid 100004 td 0xc4297c80
    fork_trampoline() at fork_trampoline
    
    Tracing command idle pid 11 tid 100003 td 0xc4298000
    sched_switch(c4298000,0,608,1055f7,c1f85c80,...) at sched_switch+0x2d2/frame 0xc3743abc
    mi_switch(608,0,c147ca1c,d3,c4298000,...) at mi_switch+0x122/frame 0xc3743af4
    critical_exit(c4278a3c,0,c4298000,e,c4278a3c,...) at critical_exit+0x89/frame 0xc3743b10
    intr_event_handle(c4295300,c3743b78,3,c3743b78,c4316c00,...) at intr_event_handle+0xeb/frame 0xc3743b30
    intr_execute_handlers(c4278a3c,c3743b78,0) at intr_execute_handlers+0x42/frame 0xc3743b50
    lapic_handle_intr(38,c3743b78) at lapic_handle_intr+0x40/frame 0xc3743b68
    Xapic_isr1() at Xapic_isr1+0config.txt0600001367012734411477  7554 ustarrootwheeloptions	CONFIG_AUTOGENERATED
    ident	pfSense
    machine	i386
    cpu	I686_CPU
    cpu	I586_CPU
    cpu	I486_CPU
    makeoptions	WITH_CTF=1
    makeoptions	DEBUG=-g
    options	CONSPEED=115200
    options	MROUTING
    options	DEVICE_POLLING
    options	HIFN_RNDTEST
    options	HIFN_DEBUG
    options	MSGTQL=2048
    options	MSGSSZ=32
    options	MSGSEG=512
    options	MSGMNI=40
    options	MSGMNB=8192
    options	ALTQ_CODEL
    options	ALTQ_NOPCC
    options	ALTQ_FAIRQ
    options	ALTQ_PRIQ
    options	ALTQ_HFSC
    options	ALTQ_RIO
    options	ALTQ_RED
    options	ALTQ_CBQ
    options	ALTQ
    options	TCP_SIGNATURE
    options	IPSEC_NAT_T
    options	IPSEC
    options	NETGRAPH_PRED1
    options	NETGRAPH_DEFLATE
    options	NETGRAPH_CAR
    options	NETGRAPH_PIPE
    options	NETGRAPH_TCPMSS
    options	NETGRAPH_TEE
    options	NETGRAPH_HOLE
    options	NETGRAPH_FRAME_RELAY
    options	NETGRAPH_ASYNC
    options	NETGRAPH_ECHO
    options	NETGRAPH_CISCO
    options	NETGRAPH_BRIDGE
    options	NETGRAPH_ONE2MANY
    options	NETGRAPH_LMI
    options	NETGRAPH_KSOCKET
    options	NETGRAPH_VJC
    options	NETGRAPH_UI
    options	NETGRAPH_MPPC_ENCRYPTION
    options	NETGRAPH_TTY
    options	NETGRAPH_SOCKET
    options	NETGRAPH_RFC1490
    options	NETGRAPH_PPTPGRE
    options	NETGRAPH_PPPOE
    options	NETGRAPH_PPP
    options	NETGRAPH_EIFACE
    options	NETGRAPH_IFACE
    options	NETGRAPH_ETHER
    options	NETGRAPH_BPF
    options	NETGRAPH_L2TP
    options	NETGRAPH_VLAN
    options	NETGRAPH
    options	IPSTEALTH
    options	IPFIREWALL_VERBOSE
    options	IPFIREWALL_DEFAULT_TO_ACCEPT
    options	PPS_SYNC
    options	NULLFS
    options	UNIONFS
    options	TMPFS
    options	GEOM_BDE
    options	GEOM_ELI
    options	GEOM_UZIP
    options	GEOM_MIRROR
    options	DDB
    options	XENHVM
    options	HYPERV
    options	ATH_ENABLE_11N
    options	AH_AR5416_INTERRUPT_MITIGATION
    options	AH_SUPPORT_AR5416
    options	IEEE80211_SUPPORT_MESH
    options	IEEE80211_AMPDU_AGE
    options	SC_PIXEL_MODE
    options	VESA
    options	AHD_REG_PRETTY_PRINT
    options	AHC_REG_PRETTY_PRINT
    options	ATA_STATIC_ID
    options	SMP
    options	KDB
    options	RCTL
    options	RACCT_DEFAULT_TO_DISABLED
    options	RACCT
    options	INCLUDE_CONFIG_FILE
    options	DDB_CTF
    options	KDTRACE_HOOKS
    options	PROCDESC
    options	CAPABILITIES
    options	CAPABILITY_MODE
    options	AUDIT
    options	HWPMC_HOOKS
    options	KBD_INSTALL_CDEV
    options	PRINTF_BUFR_SIZE=128
    options	_KPOSIX_PRIORITY_SCHEDULING
    options	SYSVSEM
    options	SYSVMSG
    options	SYSVSHM
    options	STACK
    options	KTRACE
    options	SCSI_DELAY=5000
    options	GEOM_LABEL
    options	GEOM_RAID
    options	GEOM_PART_GPT
    options	PSEUDOFS
    options	PROCFS
    options	CD9660
    options	MSDOSFS
    options	NFS_ROOT
    options	NFSLOCKD
    options	NFSD
    options	NFSCL
    options	MD_ROOT
    options	QUOTA
    options	UFS_GJOURNAL
    options	UFS_DIRHASH
    options	UFS_ACL
    options	SOFTUPDATES
    options	FFS
    options	SCTP
     INCOMPLETE
    [/thread]
    


  • That's something in the VM subsystem. The time of the dump was Tue Jun 28 08:03:43 2016 (in the system's configured timezone), and appears to be on the kernel from prior to the upgrade. Was that about the time it rebooted post-upgrade?

    The closest semi-related crash I can find is this.
    https://lists.freebsd.org/pipermail/freebsd-bugs/2012-July/049368.html

    It's not the same as that, but does appear it might be md-related in your case. You using /var as a RAM disk? It something that's repeatable if you reboot the system, or upgrade again?


Log in to reply