Possibly not-pfSense related kernel panic



  • Just got a KP on a Xen VM, probably not something pfSense can do anything about, but it's somewhat relevant as it may affect future Xen+pfSense users.

    
    panic: HYPERVISOR_memory_op failed to map gnttab
    cpuid = 2
    KDB: enter: panic
    [ thread pid 26 tid 100093 ]
    Stopped at      kdb_enter+0x3b: movq    $0,kdb_why
    db:0:kdb.enter.default> textdump set
    textdump set
    db:0:kdb.enter.default>  capture on
    db:0:kdb.enter.default>  run lockinfo
    db:1:lockinfo> show locks
    No such command
    db:1:locks>  show alllocks
    No such command
    db:1:alllocks>  show lockedvnods
    Locked vnodes
    
    0xfffff800039da3b0: tag devfs, type VCHR
        usecount 1, writecount 0, refcount 71 mountedhere 0xfffff8000902bc00
        flags (VI_ACTIVE)
        v_object 0xfffff80005d94a50 ref 0 pages 530 cleanbuf 66 dirtybuf 2
        lock type devfs: EXCL by thread 0xfffff80005d9b000 (pid 26, syncer, tid 100093)
    	dev gptid/e6091acf-7471-11e7-a3b1-00163e4568fd
    db:0:kdb.enter.default>  show pcpu
    cpuid        = 2
    dynamic pcpu = 0xfffffe010d13d200
    curthread    = 0xfffff80005d9b000: pid 26 "syncer"
    curpcb       = 0xfffffe0093cacb80
    fpcurthread  = none
    idlethread   = 0xfffff80003507500: tid 100005 "idle: cpu2"
    curpmap      = 0xffffffff829e6300
    tssp         = 0xffffffff82a1ebe0
    commontssp   = 0xffffffff82a1ebe0
    rsp0         = 0xfffffe0093cacb80
    gs32p        = 0xffffffff82a25438
    ldt          = 0xffffffff82a25478
    tss          = 0xffffffff82a25468
    db:0:kdb.enter.default>  bt
    Tracing pid 26 tid 100093 td 0xfffff80005d9b000
    kdb_enter() at kdb_enter+0x3b/frame 0xfffffe0093cac3f0
    vpanic() at vpanic+0x19f/frame 0xfffffe0093cac470
    panic() at panic+0x43/frame 0xfffffe0093cac4d0
    gnttab_map() at gnttab_map+0x131/frame 0xfffffe0093cac520
    get_free_entries() at get_free_entries+0x113/frame 0xfffffe0093cac580
    gnttab_alloc_grant_references() at gnttab_alloc_grant_references+0x12/frame 0xfffffe0093cac5a0
    xbd_startio() at xbd_startio+0x1f3/frame 0xfffffe0093cac5e0
    xbd_strategy() at xbd_strategy+0xc2/frame 0xfffffe0093cac620
    g_disk_start() at g_disk_start+0x34e/frame 0xfffffe0093cac680
    g_io_request() at g_io_request+0x4a7/frame 0xfffffe0093cac6e0
    g_part_start() at g_part_start+0x2a4/frame 0xfffffe0093cac760
    g_io_request() at g_io_request+0x4a7/frame 0xfffffe0093cac7c0
    g_io_request() at g_io_request+0x4a7/frame 0xfffffe0093cac820
    bufwrite() at bufwrite+0x267/frame 0xfffffe0093cac860
    vfs_bio_awrite() at vfs_bio_awrite+0x3ca/frame 0xfffffe0093cac900
    vop_stdfsync() at vop_stdfsync+0x277/frame 0xfffffe0093cac970
    devfs_fsync() at devfs_fsync+0x26/frame 0xfffffe0093cac9a0
    VOP_FSYNC_APV() at VOP_FSYNC_APV+0x8d/frame 0xfffffe0093cac9d0
    sched_sync() at sched_sync+0x3be/frame 0xfffffe0093caca70
    fork_exit() at fork_exit+0x85/frame 0xfffffe0093cacab0
    fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0093cacab0
    --- trap 0, rip = 0, rsp = 0, rbp = 0 ---
    db:0:kdb.enter.default>  ps
      pid  ppid  pgrp   uid   state   wmesg         wchan        cmd
    96866   292   292     0  S       accept   0xfffff800094343cc php-fpm
    95134 94268 95134     0  S+      ttyin    0xfffff80003a3d0a8 sh
    94828 94560 94828     0  S+      ttyin    0xfffff800037c9ca8 sh
    94560 94182 94560     0  S+      wait     0xfffff80064987a50 sh
    94268 92794 94268     0  S+      wait     0xfffff80009139528 sh
    94182     1 94182     0  Ss+     wait     0xfffff80009139a50 login
    94028     1 94028     0  Ss+     ttyin    0xfffff80005d82ca8 getty
    93774     1 93774     0  Ss+     ttyin    0xfffff80005d830a8 getty
    93724     1 93724     0  Ss+     ttyin    0xfffff80005d834a8 getty
    93563     1 93563     0  Ss+     ttyin    0xfffff80005d838a8 getty
    93277     1 93277     0  Ss+     ttyin    0xfffff80005d83ca8 getty
    92995     1 92995     0  Ss+     ttyin    0xfffff80005d840a8 getty
    92927     1 92927     0  Ss+     ttyin    0xfffff80005d844a8 getty
    92794     1 92794     0  Ss+     wait     0xfffff80003a4e000 login
    79583 79293 79293     0  S       nanslp   0xffffffff82866b30 minicron
    79293     1 79293     0  Ss      wait     0xfffff8000967f000 minicron
    79085 78636 78636     0  S       nanslp   0xffffffff82866b32 minicron
    78636     1 78636     0  Ss      wait     0xfffff8005990c000 minicron
    78429 77978 77978     0  S       nanslp   0xffffffff82866b31 minicron
    77978     1 77978     0  Ss      wait     0xfffff80009165528 minicron
    77255 76709 77255     0  Ss      (threaded)                  sshlockout_pf
    100118                   S       piperd   0xfffff800091445f0 sshlockout_pf
    100174                   S       nanslp   0xffffffff82866b33 sshlockout_pf
    76709     1 76709     0  Ss      select   0xfffff80005dcb540 syslogd
    74485 43738    27     0  S+      nanslp   0xffffffff82866b33 sleep
    43738     1    27     0  S+      wait     0xfffff8000967f528 sh
    36608     1 36608   136  Ss      select   0xfffff80009010e40 dhcpd
    30448     1 30448     0  Ss      (threaded)                  ntpd
    100156                   S       select   0xfffff800098da2c0 ntpd
    100162                   S       usem     0xfffff800091d1600 ntpd
    30006     1 30006     0  Ss      nanslp   0xffffffff82866b32 cron
    29449 29124 29124     0  S       kqread   0xfffff80009883400 nginx
    29441 29124 29124     0  S       kqread   0xfffff8000924be00 nginx
    29124     1 29124     0  Ss      pause    0xfffff800096805d0 nginx
    22719     1 22719    59  Ss      (threaded)                  unbound
    100126                   S       kqread   0xfffff80059739c00 unbound
    100151                   S       kqread   0xfffff80009883800 unbound
    100152                   S       kqread   0xfffff800597e3700 unbound
    100153                   S       kqread   0xfffff800597e3400 unbound
    15927     1 15927     0  Ss      (threaded)                  dpinger
    100145                   S       uwait    0xfffff8000535cc00 dpinger
    100146                   S       sbwait   0xfffff80009888804 dpinger
    100147                   S       nanslp   0xffffffff82866b30 dpinger
    100148                   S       nanslp   0xffffffff82866b30 dpinger
    100149                   S       accept   0xfffff800098883cc dpinger
    13491     1 13491     0  Ss      bpf      0xfffff80009003c00 filterlog
    12932     1 12932     0  Ss      select   0xfffff80005dca540 openvpn
    11268     1 11268     0  Ss      select   0xfffff80005df0a40 openvpn
     7987     1  7987     0  Ss      (threaded)                  sshlockout_pf
    100119                   S       uwait    0xfffff80005df2400 sshlockout_pf
    100141                   S       nanslp   0xffffffff82866b32 sshlockout_pf
     7885     1  7885     0  Ss      select   0xfffff80005def0c0 sshd
      320     1   320     0  Ss      select   0xfffff80005df1240 devd
      307   306   306     0  S       kqread   0xfffff8000901f200 check_reload_status
      306     1   306     0  Ss      kqread   0xfffff80009229500 check_reload_status
      292     1   292     0  Ss      kqread   0xfffff8000901f900 php-fpm
       57     0     0     0  DL      mdwait   0xfffff80003a45000 [md0]
       26     0     0     0  RL      CPU 2                       [syncer]
       25     0     0     0  DL      vlruwt   0xfffff80003a4ea50 [vnlru]
       24     0     0     0  DL      (threaded)                  [bufdaemon]
    100091                   D       psleep   0xffffffff8295ef04 [bufdaemon]
    100107                   D       sdflush  0xfffff80005d800e8 [/ worker]
       23     0     0     0  DL      -        0xffffffff8295fbb4 [bufspacedaemon]
       22     0     0     0  DL      pgzero   0xffffffff829742e4 [pagezero]
       21     0     0     0  DL      psleep   0xffffffff8297078c [vmdaemon]
       20     0     0     0  DL      (threaded)                  [pagedaemon]
    100087                   D       psleep   0xffffffff82a1df05 [pagedaemon]
    100094                   D       umarcl   0xffffffff829700b8 [uma]
       19     0     0     0  DL      -        0xffffffff8295e714 [soaiod4]
       18     0     0     0  DL      -        0xffffffff8295e714 [soaiod3]
       17     0     0     0  DL      -        0xffffffff8295e714 [soaiod2]
        9     0     0     0  DL      -        0xffffffff8295e714 [soaiod1]
        8     0     0     0  DL      balloon  0xffffffff80a9cef0 [balloon]
        7     0     0     0  DL      -        0xffffffff8273b2f0 [rand_harvestq]
        6     0     0     0  DL      pftm     0xffffffff80f4c4a0 [pf purge]
        5     0     0     0  DL      waiting_ 0xffffffff82a0d540 [sctp_iterator]
       16     0     0     0  SL      xbread   0xfffff800feffc000 [xenstore_rcv]
       15     0     0     0  SL      waitev   0xffffffff82858dc0 [xenwatch]
        4     0     0     0  DL      (threaded)                  [cam]
    100032                   D       -        0xffffffff8260e180 [doneq0]
    100048                   D       -        0xffffffff8260dfc8 [scanner]
        3     0     0     0  DL      crypto_r 0xffffffff8296ec70 [crypto returns]
        2     0     0     0  DL      crypto_w 0xffffffff8296eb18 [crypto]
       14     0     0     0  DL      (threaded)                  [geom]
    100023                   D       -        0xffffffff829e5740 [g_event]
    100024                   D       -        0xffffffff829e5748 [g_up]
    100025                   D       -        0xffffffff829e5750 [g_down]
       13     0     0     0  DL      (threaded)                  [ng_queue]
    100019                   D       sleep    0xffffffff825cbfd0 [ng_queue0]
    100020                   D       sleep    0xffffffff825cbfd0 [ng_queue1]
    100021                   D       sleep    0xffffffff825cbfd0 [ng_queue2]
    100022                   D       sleep    0xffffffff825cbfd0 [ng_queue3]
       12     0     0     0  WL      (threaded)                  [intr]
    100007                   I                                   [swi3: vm]
    100008                   I                                   [swi4: clock (0)]
    100009                   I                                   [swi4: clock (1)]
    100010                   I                                   [swi4: clock (2)]
    100011                   I                                   [swi4: clock (3)]
    100012                   I                                   [swi1: netisr 0]
    100013                   I                                   [swi6: task queue]
    100014                   I                                   [swi6: Giant taskq]
    100017                   I                                   [swi5: fast taskq]
    100033                   I                                   [irq14: ata0]
    100034                   I                                   [irq15: ata1]
    100035                   I                                   [irq1: atkbd0]
    100036                   I                                   [irq12: psm0]
    100037                   I                                   [swi0: uart]
    100038                   I                                   [irq772: xenstore0]
    100044                   I                                   [swi1: pf send]
    100045                   I                                   [swi1: pfsync]
    100054                   I                                   [irq777: xbd0]
    100056                   I                                   [irq778: xn0]
    100058                   I                                   [irq779: xn0]
    100060                   I                                   [irq780: xn0]
    100062                   I                                   [irq781: xn0]
    100064                   I                                   [irq782: xn1]
    100066                   I                                   [irq783: xn1]
    100068                   I                                   [irq784: xn1]
    100070                   I                                   [irq785: xn1]
    100072                   I                                   [irq786: xn2]
    100074                   I                                   [irq787: xn2]
    100076                   I                                   [irq788: xn2]
    100078                   I                                   [irq789: xn2]
    100080                   I                                   [irq790: xn3]
    100082                   I                                   [irq791: xn3]
    100084                   I                                   [irq792: xn3]
    100086                   I                                   [irq793: xn3]
    100095                   I                                   [swi1: netisr 1]
    100096                   I                                   [swi1: netisr 2]
    100097                   I                                   [swi1: netisr 3]
       11     0     0     0  RL      (threaded)                  [idle]
    100003                   Run     CPU 0                       [idle: cpu0]
    100004                   Run     CPU 1                       [idle: cpu1]
    100005                   CanRun                              [idle: cpu2]
    100006                   Run     CPU 3                       [idle: cpu3]
        1     0     1     0  SLs     wait     0xfffff80003505528 [init]
       10     0     0     0  DL      audit_wo 0xffffffff82a146e0 [audit]
        0     0     0     0  DLs     (threaded)                  [kernel]
    100000                   D       swapin   0xffffffff829e5778 [swapper]
    100015                   D       -        0xfffff8000351a100 [aiod_kick taskq]
    100016                   D       -        0xfffff80003519e00 [thread]
    100018                   D       -        0xfffff80003519900 [kqueue_ctx taskq]
    100026                   D       -        0xfffff80003518a00 [firmware taskq]
    100029                   D       -        0xfffff800034f8300 [acpi_task_0]
    100030                   D       -        0xfffff800034f8300 [acpi_task_1]
    100031                   D       -        0xfffff800034f8300 [acpi_task_2]
    100041                   D       -        0xfffff800037c2c00 [mca taskq]
    100047                   D       -        0xfffff800036e6100 [CAM taskq]
    100071                   D       -        0xfffff80003eea000 [xn2 txq 0]
    100073                   D       -        0xfffff80003ee9b00 [xn2 txq 1]
    100075                   D       -        0xfffff80003ee9700 [xn2 txq 2]
    100077                   D       -        0xfffff80003ee9300 [xn2 txq 3]
    100079                   D       -        0xfffff80003ee8e00 [xn3 txq 0]
    100081                   D       -        0xfffff80003ee8a00 [xn3 txq 1]
    100083                   D       -        0xfffff80003ee8600 [xn3 txq 2]
    100085                   D       -        0xfffff80003ee8200 [xn3 txq 3]
    100098                   D       -        0xfffff80005da1900 [if_config_tqg_0]
    100099                   D       -        0xfffff80005da1700 [if_io_tqg_0]
    100100                   D       -        0xfffff80005da1500 [if_io_tqg_1]
    100101                   D       -        0xfffff80005da1300 [if_io_tqg_2]
    100102                   D       -        0xfffff80005da1100 [if_io_tqg_3]
    100132                   D       -        0xfffff80003a54100 [xn0 txq 0]
    100133                   D       -        0xfffff80003a54800 [xn0 txq 1]
    100134                   D       -        0xfffff80003a54000 [xn0 txq 2]
    100135                   D       -        0xfffff80003a54900 [xn0 txq 3]
    100136                   D       -        0xfffff80003eea400 [xn1 txq 0]
    100137                   D       -        0xfffff800039e1700 [xn1 txq 1]
    100138                   D       -        0xfffff80003eea300 [xn1 txq 2]
    100139                   D       -        0xfffff800039e1800 [xn1 txq 3]
    db:0:kdb.enter.default>  alltrace
    
    Tracing command php-fpm pid 96866 tid 100169 td 0xfffff800090ef000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093e04710
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093e04740
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093e047c0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093e047f0
    _sleep() at _sleep+0x294/frame 0xfffffe0093e04880
    kern_accept4() at kern_accept4+0x1f2/frame 0xfffffe0093e04930
    accept1() at accept1+0x60/frame 0xfffffe0093e04990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093e04ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093e04ab0
    --- syscall (30, FreeBSD ELF64, sys_accept), rip = 0x801d939ba, rsp = 0x7ffffffec848, rbp = 0x7ffffffec880 ---
    
    Tracing command sh pid 95134 tid 100173 td 0xfffff800649d8000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093e18660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093e18690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093e18710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093e18740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093e18790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093e187b0
    ttydisc_read() at ttydisc_read+0x244/frame 0xfffffe0093e18810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093e18840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093e188a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093e188f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093e18940
    sys_read() at sys_read+0x84/frame 0xfffffe0093e18990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093e18ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093e18ab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800bc61ea, rsp = 0x7fffffffe768, rbp = 0x7fffffffe870 ---
    
    Tracing command sh pid 94828 tid 100177 td 0xfffff80009981500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093e2c660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093e2c690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093e2c710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093e2c740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093e2c790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093e2c7b0
    ttydisc_read() at ttydisc_read+0x244/frame 0xfffffe0093e2c810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093e2c840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093e2c8a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093e2c8f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093e2c940
    sys_read() at sys_read+0x84/frame 0xfffffe0093e2c990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093e2cab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093e2cab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800bc61ea, rsp = 0x7fffffffe768, rbp = 0x7fffffffe870 ---
    
    Tracing command sh pid 94560 tid 100176 td 0xfffff80009981a00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093e27560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093e27590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093e27610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093e27640
    _sleep() at _sleep+0x294/frame 0xfffffe0093e276d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093e277a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093e27990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093e27ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093e27ab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800b5d75a, rsp = 0x7fffffffe818, rbp = 0x7fffffffe8d0 ---
    
    Tracing command sh pid 94268 tid 100112 td 0xfffff8000916f500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d0b560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d0b590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d0b610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d0b640
    _sleep() at _sleep+0x294/frame 0xfffffe0093d0b6d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093d0b7a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093d0b990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d0bab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d0bab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800b5d75a, rsp = 0x7fffffffe818, rbp = 0x7fffffffe8d0 ---
    
    Tracing command login pid 94182 tid 100106 td 0xfffff8000916fa00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093ced560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093ced590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093ced610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093ced640
    _sleep() at _sleep+0x294/frame 0xfffffe0093ced6d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093ced7a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093ced990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093cedab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093cedab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800d2475a, rsp = 0x7fffffffebe8, rbp = 0x7fffffffed40 ---
    
    Tracing command getty pid 94028 tid 100108 td 0xfffff800090f8500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093cf7660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093cf7690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093cf7710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093cf7740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093cf7790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093cf77b0
    ttydisc_read() at ttydisc_read+0x30a/frame 0xfffffe0093cf7810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093cf7840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093cf78a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093cf78f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093cf7940
    sys_read() at sys_read+0x84/frame 0xfffffe0093cf7990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093cf7ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093cf7ab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b811ea, rsp = 0x7fffffffec78, rbp = 0x7fffffffecc0 ---
    
    Tracing command getty pid 93774 tid 100130 td 0xfffff80009365000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d65660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d65690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d65710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d65740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093d65790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093d657b0
    ttydisc_read() at ttydisc_read+0x30a/frame 0xfffffe0093d65810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093d65840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093d658a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093d658f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093d65940
    sys_read() at sys_read+0x84/frame 0xfffffe0093d65990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d65ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d65ab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b811ea, rsp = 0x7fffffffec78, rbp = 0x7fffffffecc0 ---
    
    Tracing command getty pid 93724 tid 100175 td 0xfffff800098dd000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093e22660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093e22690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093e22710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093e22740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093e22790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093e227b0
    ttydisc_read() at ttydisc_read+0x30a/frame 0xfffffe0093e22810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093e22840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093e228a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093e228f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093e22940
    sys_read() at sys_read+0x84/frame 0xfffffe0093e22990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093e22ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093e22ab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b811ea, rsp = 0x7fffffffec78, rbp = 0x7fffffffecc0 ---
    
    Tracing command getty pid 93563 tid 100113 td 0xfffff80005df6000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d10660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d10690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d10710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d10740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093d10790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093d107b0
    ttydisc_read() at ttydisc_read+0x30a/frame 0xfffffe0093d10810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093d10840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093d108a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093d108f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093d10940
    sys_read() at sys_read+0x84/frame 0xfffffe0093d10990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d10ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d10ab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b811ea, rsp = 0x7fffffffec78, rbp = 0x7fffffffecc0 ---
    
    Tracing command getty pid 93277 tid 100125 td 0xfffff8000935ca00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d4c660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d4c690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d4c710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d4c740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093d4c790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093d4c7b0
    ttydisc_read() at ttydisc_read+0x30a/frame 0xfffffe0093d4c810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093d4c840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093d4c8a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093d4c8f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093d4c940
    sys_read() at sys_read+0x84/frame 0xfffffe0093d4c990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d4cab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d4cab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b811ea, rsp = 0x7fffffffec78, rbp = 0x7fffffffecc0 ---
    
    Tracing command getty pid 92995 tid 100122 td 0xfffff80005db8a00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d3d660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d3d690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d3d710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d3d740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093d3d790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093d3d7b0
    ttydisc_read() at ttydisc_read+0x30a/frame 0xfffffe0093d3d810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093d3d840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093d3d8a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093d3d8f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093d3d940
    sys_read() at sys_read+0x84/frame 0xfffffe0093d3d990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d3dab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d3dab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b811ea, rsp = 0x7fffffffec78, rbp = 0x7fffffffecc0 ---
    
    Tracing command getty pid 92927 tid 100105 td 0xfffff80005dce500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093ce8660
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093ce8690
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093ce8710
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093ce8740
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093ce8790
    tty_wait() at tty_wait+0x1c/frame 0xfffffe0093ce87b0
    ttydisc_read() at ttydisc_read+0x30a/frame 0xfffffe0093ce8810
    ttydev_read() at ttydev_read+0x34/frame 0xfffffe0093ce8840
    devfs_read_f() at devfs_read_f+0xe8/frame 0xfffffe0093ce88a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093ce88f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093ce8940
    sys_read() at sys_read+0x84/frame 0xfffffe0093ce8990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093ce8ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093ce8ab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b811ea, rsp = 0x7fffffffec78, rbp = 0x7fffffffecc0 ---
    
    Tracing command login pid 92794 tid 100103 td 0xfffff80005df7000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093cde560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093cde590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093cde610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093cde640
    _sleep() at _sleep+0x294/frame 0xfffffe0093cde6d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093cde7a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093cde990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093cdeab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093cdeab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800d2475a, rsp = 0x7fffffffebe8, rbp = 0x7fffffffed40 ---
    
    Tracing command minicron pid 79583 tid 100129 td 0xfffff80009365500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d60780
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d607b0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d60830
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093d60860
    _sleep() at _sleep+0x26f/frame 0xfffffe0093d608f0
    kern_nanosleep() at kern_nanosleep+0x10e/frame 0xfffffe0093d60950
    sys_nanosleep() at sys_nanosleep+0x53/frame 0xfffffe0093d60990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d60ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d60ab0
    --- syscall (240, FreeBSD ELF64, sys_nanosleep), rip = 0x8009008fa, rsp = 0x7fffffffeb88, rbp = 0x7fffffffebf0 ---
    
    Tracing command minicron pid 79293 tid 100163 td 0xfffff800598d4a00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093de2560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093de2590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093de2610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093de2640
    _sleep() at _sleep+0x294/frame 0xfffffe0093de26d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093de27a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093de2990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093de2ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093de2ab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x80090075a, rsp = 0x7fffffffebf8, rbp = 0x7fffffffec30 ---
    
    Tracing command minicron pid 79085 tid 100172 td 0xfffff800649d8500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093e13780
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093e137b0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093e13830
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093e13860
    _sleep() at _sleep+0x26f/frame 0xfffffe0093e138f0
    kern_nanosleep() at kern_nanosleep+0x10e/frame 0xfffffe0093e13950
    sys_nanosleep() at sys_nanosleep+0x53/frame 0xfffffe0093e13990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093e13ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093e13ab0
    --- syscall (240, FreeBSD ELF64, sys_nanosleep), rip = 0x8009008fa, rsp = 0x7fffffffeb98, rbp = 0x7fffffffec00 ---
    
    Tracing command minicron pid 78636 tid 100164 td 0xfffff80059813a00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093de7560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093de7590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093de7610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093de7640
    _sleep() at _sleep+0x294/frame 0xfffffe0093de76d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093de77a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093de7990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093de7ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093de7ab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x80090075a, rsp = 0x7fffffffec08, rbp = 0x7fffffffec40 ---
    
    Tracing command minicron pid 78429 tid 100117 td 0xfffff80005dcda00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d24780
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d247b0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d24830
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093d24860
    _sleep() at _sleep+0x26f/frame 0xfffffe0093d248f0
    kern_nanosleep() at kern_nanosleep+0x10e/frame 0xfffffe0093d24950
    sys_nanosleep() at sys_nanosleep+0x53/frame 0xfffffe0093d24990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d24ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d24ab0
    --- syscall (240, FreeBSD ELF64, sys_nanosleep), rip = 0x8009008fa, rsp = 0x7fffffffebb8, rbp = 0x7fffffffec20 ---
    
    Tracing command minicron pid 77978 tid 100160 td 0xfffff800098dd500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dd3560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dd3590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dd3610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093dd3640
    _sleep() at _sleep+0x294/frame 0xfffffe0093dd36d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093dd37a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093dd3990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dd3ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dd3ab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x80090075a, rsp = 0x7fffffffec28, rbp = 0x7fffffffec60 ---
    
    Tracing command sshlockout_pf pid 77255 tid 100118 td 0xfffff80005df4a00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d296d0
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d29700
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d29780
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d297b0
    _sleep() at _sleep+0x294/frame 0xfffffe0093d29840
    pipe_read() at pipe_read+0x36d/frame 0xfffffe0093d298a0
    dofileread() at dofileread+0x98/frame 0xfffffe0093d298f0
    kern_readv() at kern_readv+0x68/frame 0xfffffe0093d29940
    sys_read() at sys_read+0x84/frame 0xfffffe0093d29990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d29ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d29ab0
    --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b911ea, rsp = 0x7fffffffe7d8, rbp = 0x7fffffffe7f0 ---
    
    Tracing command sshlockout_pf pid 77255 tid 100174 td 0xfffff800598d4500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093e1d780
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093e1d7b0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093e1d830
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093e1d860
    _sleep() at _sleep+0x26f/frame 0xfffffe0093e1d8f0
    kern_nanosleep() at kern_nanosleep+0x10e/frame 0xfffffe0093e1d950
    sys_nanosleep() at sys_nanosleep+0x53/frame 0xfffffe0093e1d990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093e1dab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093e1dab0
    --- syscall (240, FreeBSD ELF64, sys_nanosleep), rip = 0x800b288fa, rsp = 0x7fffdfffdf48, rbp = 0x7fffdfffdf70 ---
    
    Tracing command syslogd pid 76709 tid 100165 td 0xfffff80059813500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dec5e0
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dec610
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dec690
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093dec6c0
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093dec710
    seltdwait() at seltdwait+0x10f/frame 0xfffffe0093dec760
    kern_select() at kern_select+0x979/frame 0xfffffe0093dec950
    sys_select() at sys_select+0x54/frame 0xfffffe0093dec990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093decab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093decab0
    --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800b851ca, rsp = 0x7fffffffdd78, rbp = 0x7fffffffec20 ---
    
    Tracing command sleep pid 74485 tid 100168 td 0xfffff800090ef500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dfb780
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dfb7b0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dfb830
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093dfb860
    _sleep() at _sleep+0x26f/frame 0xfffffe0093dfb8f0
    kern_nanosleep() at kern_nanosleep+0x10e/frame 0xfffffe0093dfb950
    sys_nanosleep() at sys_nanosleep+0x53/frame 0xfffffe0093dfb990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dfbab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dfbab0
    --- syscall (240, FreeBSD ELF64, sys_nanosleep), rip = 0x8009008fa, rsp = 0x7fffffffec58, rbp = 0x7fffffffeca0 ---
    
    Tracing command sh pid 43738 tid 100161 td 0xfffff800598ca000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dd8560
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dd8590
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dd8610
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093dd8640
    _sleep() at _sleep+0x294/frame 0xfffffe0093dd86d0
    kern_wait6() at kern_wait6+0x2d3/frame 0xfffffe0093dd87a0
    sys_wait4() at sys_wait4+0x73/frame 0xfffffe0093dd8990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dd8ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dd8ab0
    --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800b5d75a, rsp = 0x7fffffffe858, rbp = 0x7fffffffe910 ---
    
    Tracing command dhcpd pid 36608 tid 100159 td 0xfffff800598ca500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dce5c0
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dce5f0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dce670
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093dce6a0
    _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x1c4/frame 0xfffffe0093dce710
    seltdwait() at seltdwait+0xc7/frame 0xfffffe0093dce760
    kern_select() at kern_select+0x979/frame 0xfffffe0093dce950
    sys_select() at sys_select+0x54/frame 0xfffffe0093dce990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dceab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dceab0
    --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800d641ca, rsp = 0x7fffffffea28, rbp = 0x7fffffffea50 ---
    
    Tracing command ntpd pid 30448 tid 100156 td 0xfffff800597a0a00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dbf5e0
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dbf610
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dbf690
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093dbf6c0
    _cv_wait_sig() at _cv_wait_sig+0x1b2/frame 0xfffffe0093dbf710
    seltdwait() at seltdwait+0x10f/frame 0xfffffe0093dbf760
    kern_select() at kern_select+0x979/frame 0xfffffe0093dbf950
    sys_select() at sys_select+0x54/frame 0xfffffe0093dbf990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dbfab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dbfab0
    --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x8017e51ca, rsp = 0x7fffffffdc08, rbp = 0x7fffffffdc40 ---
    
    Tracing command ntpd pid 30448 tid 100162 td 0xfffff800597a0000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093ddd700
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093ddd730
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093ddd7b0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093ddd7e0
    _sleep() at _sleep+0x294/frame 0xfffffe0093ddd870
    umtxq_sleep() at umtxq_sleep+0x122/frame 0xfffffe0093ddd8d0
    do_sem2_wait() at do_sem2_wait+0x6da/frame 0xfffffe0093ddd950
    __umtx_op_sem2_wait() at __umtx_op_sem2_wait+0x6d/frame 0xfffffe0093ddd990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dddab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dddab0
    --- syscall (454, FreeBSD ELF64, sys__umtx_op), rip = 0x801716a9a, rsp = 0x7fffdfffdee8, rbp = 0x7fffdfffdf40 ---
    
    Tracing command cron pid 30006 tid 100140 td 0xfffff80009364500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d6f780
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d6f7b0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d6f830
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093d6f860
    _sleep() at _sleep+0x26f/frame 0xfffffe0093d6f8f0
    kern_nanosleep() at kern_nanosleep+0x10e/frame 0xfffffe0093d6f950
    sys_nanosleep() at sys_nanosleep+0x53/frame 0xfffffe0093d6f990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d6fab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d6fab0
    --- syscall (240, FreeBSD ELF64, sys_nanosleep), rip = 0x800d288fa, rsp = 0x7fffffffea88, rbp = 0x7fffffffecb0 ---
    
    Tracing command nginx pid 29449 tid 100155 td 0xfffff800597a1000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dba520
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dba550
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dba5d0
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093dba600
    _sleep() at _sleep+0x26f/frame 0xfffffe0093dba690
    kqueue_kevent() at kqueue_kevent+0x336/frame 0xfffffe0093dba870
    kern_kevent() at kern_kevent+0xb5/frame 0xfffffe0093dba8e0
    sys_kevent() at sys_kevent+0x11c/frame 0xfffffe0093dba990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dbaab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dbaab0
    --- syscall (363, FreeBSD ELF64, sys_kevent), rip = 0x801b6293a, rsp = 0x7fffffffe718, rbp = 0x7fffffffe760 ---
    
    Tracing command nginx pid 29441 tid 100143 td 0xfffff8000916e000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d7e520
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d7e550
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d7e5d0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d7e600
    _sleep() at _sleep+0x294/frame 0xfffffe0093d7e690
    kqueue_kevent() at kqueue_kevent+0x336/frame 0xfffffe0093d7e870
    kern_kevent() at kern_kevent+0xb5/frame 0xfffffe0093d7e8e0
    sys_kevent() at sys_kevent+0x11c/frame 0xfffffe0093d7e990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d7eab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d7eab0
    --- syscall (363, FreeBSD ELF64, sys_kevent), rip = 0x801b6293a, rsp = 0x7fffffffe718, rbp = 0x7fffffffe760 ---
    
    Tracing command nginx pid 29124 tid 100154 td 0xfffff80009364000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093db5780
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093db57b0
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093db5830
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093db5860
    _sleep() at _sleep+0x294/frame 0xfffffe0093db58f0
    kern_sigsuspend() at kern_sigsuspend+0x134/frame 0xfffffe0093db5960
    sys_sigsuspend() at sys_sigsuspend+0x31/frame 0xfffffe0093db5990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093db5ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093db5ab0
    --- syscall (4, FreeBSD ELF64, sys_write), rip = 0x801b627da, rsp = 0x7fffffffe8b8, rbp = 0x7fffffffe8f0 ---
    
    Tracing command unbound pid 22719 tid 100126 td 0xfffff8000935c500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d51520
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d51550
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d515d0
    sleepq_timedwait_sig() at sleepq_timedwait_sig+0x10/frame 0xfffffe0093d51600
    _sleep() at _sleep+0x26f/frame 0xfffffe0093d51690
    kqueue_kevent() at kqueue_kevent+0x336/frame 0xfffffe0093d51870
    kern_kevent() at kern_kevent+0xb5/frame 0xfffffe0093d518e0
    sys_kevent() at sys_kevent+0x11c/frame 0xfffffe0093d51990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d51ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d51ab0
    --- syscall (363, FreeBSD ELF64, sys_kevent), rip = 0x801b7293a, rsp = 0x7fffffffea08, rbp = 0x7fffffffea50 ---
    
    Tracing command unbound pid 22719 tid 100151 td 0xfffff8000913a000
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093da6520
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093da6550
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093da65d0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093da6600
    _sleep() at _sleep+0x294/frame 0xfffffe0093da6690
    kqueue_kevent() at kqueue_kevent+0x336/frame 0xfffffe0093da6870
    kern_kevent() at kern_kevent+0xb5/frame 0xfffffe0093da68e0
    sys_kevent() at sys_kevent+0x11c/frame 0xfffffe0093da6990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093da6ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093da6ab0
    --- syscall (363, FreeBSD ELF64, sys_kevent), rip = 0x801b7293a, rsp = 0x7fffdfffde18, rbp = 0x7fffdfffde60 ---
    
    Tracing command unbound pid 22719 tid 100152 td 0xfffff800597a1a00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093dab520
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093dab550
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093dab5d0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093dab600
    _sleep() at _sleep+0x294/frame 0xfffffe0093dab690
    kqueue_kevent() at kqueue_kevent+0x336/frame 0xfffffe0093dab870
    kern_kevent() at kern_kevent+0xb5/frame 0xfffffe0093dab8e0
    sys_kevent() at sys_kevent+0x11c/frame 0xfffffe0093dab990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093dabab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093dabab0
    --- syscall (363, FreeBSD ELF64, sys_kevent), rip = 0x801b7293a, rsp = 0x7fffdfdfce18, rbp = 0x7fffdfdfce60 ---
    
    Tracing command unbound pid 22719 tid 100153 td 0xfffff800597a1500
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093db0520
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093db0550
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093db05d0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093db0600
    _sleep() at _sleep+0x294/frame 0xfffffe0093db0690
    kqueue_kevent() at kqueue_kevent+0x336/frame 0xfffffe0093db0870
    kern_kevent() at kern_kevent+0xb5/frame 0xfffffe0093db08e0
    sys_kevent() at sys_kevent+0x11c/frame 0xfffffe0093db0990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093db0ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093db0ab0
    --- syscall (363, FreeBSD ELF64, sys_kevent), rip = 0x801b7293a, rsp = 0x7fffdfbfbe18, rbp = 0x7fffdfbfbe60 ---
    
    Tracing command dpinger pid 15927 tid 100145 td 0xfffff8000913aa00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d886f0
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d88720
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d887a0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d887d0
    _sleep() at _sleep+0x294/frame 0xfffffe0093d88860
    umtxq_sleep() at umtxq_sleep+0x122/frame 0xfffffe0093d888c0
    do_wait() at do_wait+0x419/frame 0xfffffe0093d88950
    __umtx_op_wait() at __umtx_op_wait+0x77/frame 0xfffffe0093d88990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d88ab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d88ab0
    --- syscall (454, FreeBSD ELF64, sys__umtx_op), rip = 0x80083c67c, rsp = 0x7fffffffe8f8, rbp = 0x7fffffffe980 ---
    
    Tracing command dpinger pid 15927 tid 100146 td 0xfffff8000997ea00
    sched_switch() at sched_switch+0x6cb/frame 0xfffffe0093d8d5f0
    mi_switch() at mi_switch+0xd2/frame 0xfffffe0093d8d620
    sleepq_catch_signals() at sleepq_catch_signals+0xb7/frame 0xfffffe0093d8d6a0
    sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0093d8d6d0
    _sleep() at _sleep+0x294/frame 0xfffffe0093d8d760
    sbwait() at sbwait+0x4f/frame 0xfffffe0093d8d780
    soreceive_generic() at soreceive_generic+0x1ae/frame 0xfffffe0093d8d850
    kern_recvit() at kern_recvit+0x1d6/frame 0xfffffe0093d8d920
    sys_recvfrom() at sys_recvfrom+0x86/frame 0xfffffe0093d8d990
    amd64_syscall() at amd64_syscall+0x4ce/frame 0xfffffe0093d8dab0
    Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe0093d8dab0
    --- syscall (29, FreeBSD ELF64, sys_recvfrom), rip = 0x800b3cf5a, rsp = 0x7fffdfffde98, rbp = 0x7fffdfffdee0 ---
    
    (truncated because the forum doesn't like it, ends with:)
    
    [code]
    --- trap 0, rip = 0, rsp = 0, rbp = 0 ---
    db:0:kdb.enter.default>  capture off
    db:0:kdb.enter.default>  textdump dump
    Textdump complete.
    db:0:kdb.enter.default>  reset
    cpu_reset: Restarting BSP
    cpu_reset_proxy: Stopped CPU 2
    [/code][/thread]
    

  • Rebel Alliance Developer Netgate

    Looks like a filesystem panic. Could be anything from a corrupted fs that needs fsck run to an actual hardware issue in the hypervisor, or anything in between.

    Those are hard to nail down in a virtualized environment.

    Safest bet is to wipe+reload+restore config



  • @jimp:

    Looks like a filesystem panic. Could be anything from a corrupted fs that needs fsck run to an actual hardware issue in the hypervisor, or anything in between.

    Those are hard to nail down in a virtualized environment.

    Safest bet is to wipe+reload+restore config

    Yeah, did that. The report should have been sent to your crash report recording service. Some info:

    
    					Crash report begins.  Anonymous machine information:
    
    amd64
    11.0-RELEASE-p11
    FreeBSD 11.0-RELEASE-p11 #191 de7122ed242(RELENG_2_4): Fri Jul 28 12:05:08 CDT 2017     root@buildbot2.netgate.com:/builder/ce/tmp/obj/builder/ce/tmp/FreeBSD-src/sys/pfSense
    
    Crash report details:
    
    No PHP errors found.
    
    Filename: /var/crash/bounds
    1
    
    Filename: /var/crash/info.0
    Dump header from device: /dev/gptid/e609f1ad-7471-11e7-a3b1-00163e4568fd
      Architecture: amd64
      Architecture Version: 1
      Dump Length: 69632
      Blocksize: 512
      Dumptime: Tue Aug  1 13:14:27 2017
      Hostname: firewall-1-prod.adivare.net
      Magic: FreeBSD Text Dump
      Version String: FreeBSD 11.0-RELEASE-p11 #191 de7122ed242(RELENG_2_4): Fri Jul 28 12:05:08 CDT 2017
        root@buildbot2.netgate.com:/builder/ce/tmp/obj/builder/ce/tmp/FreeBSD-src/sys/pfSense
      Panic String: HYPERVISOR_memory_op failed to map gnttab
      Dump Parity: 2668466774
      Bounds: 0
      Dump Status: good
    
    

    So it's 2.4.0-BETA (amd64) built on Fri Jul 28 11:57:59 CDT 2017, running on Xen PVHVM 4.8.1, backed by a LVM disk which is stored on a mirrored SSD (MD Array). I suspect it must have been somewhere in the virtual block device layer, either the backing end or the communication between the backing disk and the block frontend in BSD, but most likely not a BSD fault anyway. Recording it here might help someone in the future. It runs fine now, and I couldn't reproduce this.



  • More information; 2.4 / FreeBSD 11.0 reliably crashes when more than 3 virtual network interfaces are added.

    Configuration used:

    
    name = 'firewall'
    bios = "ovmf"
    uuid="11d1366c-40e2-43b2-83be-52cfe0c6542d"
    builder = 'hvm'
    memory = '2024'
    vcpus = 4
    disk = ['file:/dev/hypervisor-local/firewall-system,xvda,w']
    vif = [
    'mac=00:16:3e:e5:68:fd, bridge=wan0, script=vif-openvswitch-no-offload, type=vif',
    'mac=00:16:3e:e8:20:db, bridge=lan10, script=vif-openvswitch-no-offload, type=vif',
    #'mac=00:16:3e:ee:02:44, bridge=lan20, script=vif-openvswitch-no-offload, type=vif',
    #'mac=00:16:3e:e1:03:f1, bridge=lan30, script=vif-openvswitch-no-offload, type=vif',
    'mac=00:16:3e:e2:30:44, bridge=lan40, script=vif-openvswitch-no-offload, type=vif'
    ]
    boot = 'c'
    serial='pty'
    
    

    The VIF lines define network interfaces, when using the 3 as described in this configuration pfSense works, when adding one or more beyond that, it always crashes. With 4 it generates a panic and dump, with 5 it just panics and hangs in kdb with db>.


Log in to reply