Crash



  • We have had one of our two HA firewalls that keeps crashing. I dont know if its hardware related or if its because of some package. Are there any clues in this output?

    Crash report begins. Anonymous machine information:

    amd64
    8.1-RELEASE-p13
    FreeBSD 8.1-RELEASE-p13 #1: Fri Dec 7 23:07:32 EST 2012 root@snapshots-8_1-amd64.builders.pfsense.org:/usr/obj./usr/pfSensesrc/src/sys/pfSense_SMP.8

    Crash report details:

    Filename: /var/crash/bounds
    2

    Filename: /var/crash/info.0
    Dump header from device /dev/ad4s1b
    Architecture: amd64
    Architecture Version: 1
    Dump Length: 73216B (0 MB)
    Blocksize: 512
    Dumptime: Tue Aug 27 11:33:30 2013
    Hostname: twfw-1.transwest.com
    Magic: FreeBSD Text Dump
    Version String: FreeBSD 8.1-RELEASE-p13 #1: Fri Dec 7 23:07:32 EST 2012
    root@snapshots-8_1-amd64.builders.pfsense.org:/usr/obj./usr/pfSensesrc/src/sys/pfSense_SMP.8
    Panic String:
    Dump Parity: 2441754477
    Bounds: 0
    Dump Status: good

    Filename: /var/crash/info.1
    Dump header from device /dev/ad4s1b
    Architecture: amd64
    Architecture Version: 1
    Dump Length: 89600B (0 MB)
    Blocksize: 512
    Dumptime: Tue Sep 3 13:45:16 2013
    Hostname: twfw-1.transwest.com
    Magic: FreeBSD Text Dump
    Version String: FreeBSD 8.1-RELEASE-p13 #1: Fri Dec 7 23:07:32 EST 2012
    root@snapshots-8_1-amd64.builders.pfsense.org:/usr/obj./usr/pfSensesrc/src/sys/pfSense_SMP.8
    Panic String:
    Dump Parity: 924061037
    Bounds: 1
    Dump Status: good

    Filename: /var/crash/textdump.tar.0
    ddb.txt06000014000012207161352 7071 ustarrootwheeldb:0:kdb.enter.default> run lockinfo
    db:1:lockinfo> show locks
    No such command
    db:1:locks> show alllocks
    No such command
    db:1:alllocks> show lockedvnods
    Locked vnodes
    db:0:kdb.enter.default> show pcpu
    cpuid = 2
    dynamic pcpu = 0xffffff807ef05080
    curthread = 0xffffff000254f7c0: pid 0 "em1 que"
    curpcb = 0xffffff8000140d40
    fpcurthread = none
    idlethread = 0xffffff00022ea3e0: pid 11 "idle: cpu2"
    curpmap = 0
    tssp = 0xffffffff811dde50
    commontssp = 0xffffffff811dde50
    rsp0 = 0xffffff8000140d40
    gs32p = 0xffffffff811dcc88
    ldt = 0xffffffff811dccc8
    tss = 0xffffffff811dccb8
    db:0:kdb.enter.default> bt
    Tracing pid 0 tid 64052 td 0xffffff000254f7c0
    rn_match() at rn_match+0x25
    pfr_match_addr() at pfr_match_addr+0xcb
    pf_test_tcp() at pf_test_tcp+0xaa1
    pf_test() at pf_test+0x20fe
    pf_check_in() at pf_check_in+0x39
    pfil_run_hooks() at pfil_run_hooks+0xa2
    ip_input() at ip_input+0x34e
    netisr_dispatch_src() at netisr_dispatch_src+0x7b
    ether_demux() at ether_demux+0x169
    ether_input() at ether_input+0x174
    em_rxeof() at em_rxeof+0x1d0
    em_handle_que() at em_handle_que+0x4d
    taskqueue_run() at taskqueue_run+0x93
    taskqueue_thread_loop() at taskqueue_thread_loop+0x46
    fork_exit() at fork_exit+0x118
    fork_trampoline() at fork_trampoline+0xe
    –- trap 0, rip = 0, rsp = 0xffffff8000140d30, rbp = 0 ---
    db:0:kdb.enter.default> ps
    pid ppid pgrp uid state wmesg wchan cmd
    22685 21829 24 0 S piperd 0xffffff0002aeb888 awk
    22643 21829 24 0 S piperd 0xffffff0031d6bb60 wc
    22638 21829 24 0 S piperd 0xffffff0002b26888 awk
    22511 21829 24 0 S select 0xffffff001447ecc0 nc
    22289 21999 24 0 S nanslp 0xffffffff8115cc28 sleep
    21999 21829 24 0 S wait 0xffffff00319ba460 sh
    21829 49943 24 0 S wait 0xffffff00148c0460 sh
    4808 17301 17274 0 S accept 0xffffff0031ec5066 initial thread
    23787 17301 17274 0 S accept 0xffffff001419630e initial thread
    21900 1 21900 0 Ss select 0xffffff0014160dc0 racoon
    59471 61306 61306 0 S piperd 0xffffff001433b888 rrdtool
    5253 1705 5253 0 S+ ttyin 0xffffff00028168a8 sh
    5233 1987 5233 0 S+ ttyin 0xffffff00025a88a8 sh
    1987 1420 1987 0 S+ wait 0xffffff00029b88c0 sh
    1705 1356 1705 0 S+ wait 0xffffff0002baa460 sh
    1568 53664 1568 0 Ss (threaded) sshlockout_pf
    64213 S nanslp 0xffffffff8115cc28 sshlockout_pf
    64108 S piperd 0xffffff0002aeb5b0 initial thread
    1420 1 1420 0 Ss+ wait 0xffffff00319c4000 login
    1356 1 1356 0 Ss+ wait 0xffffff00319b98c0 login
    60689 60207 60207 0 S nanslp 0xffffffff8115cc28 minicron
    60207 1 60207 0 Ss wait 0xffffff001424b460 minicron
    59981 59633 59633 0 S nanslp 0xffffffff8115cc28 minicron
    59633 1 59633 0 Ss wait 0xffffff0002aac460 minicron
    59430 59116 59116 0 S nanslp 0xffffffff8115cc28 minicron
    59116 1 59116 0 Ss wait 0xffffff00319c58c0 minicron
    11428 1 11428 0 Ss select 0xffffff0031849640 ntpd
    55011 1 55011 0 Ss nanslp 0xffffffff8115cc28 cron
    49943 1 24 0 S+ piperd 0xffffff0002b242d8 sh
    30909 1 30722 65534 S select 0xffffff00319d89c0 dnsmasq
    27545 1 27545 1002 Ss select 0xffffff00028f7bc0 dhcpd
    17301 1 17274 0 S kqread 0xffffff000294d600 lighttpd
    61306 1 61306 0 Ss select 0xffffff0031849a40 apinger
    55048 1 55048 0 Ss select 0xffffff0002972240 inetd
    54426 1 54426 0 Ss select 0xffffff0002a899c0 openvpn
    53986 1 24 0 S+ piperd 0xffffff0002966b60 logger
    53846 1 24 0 S+ bpf 0xffffff00319be000 tcpdump
    53664 1 53664 0 Ss select 0xffffff00319d8040 syslogd
    7603 1 7603 0 Ss select 0xffffff00028f7040 sshd
    260 1 260 0 Ss select 0xffffff00028aeb40 devd
    250 248 248 0 S kqread 0xffffff0002905600 check_reload_status
    248 1 248 0 Ss kqread 0xffffff00028b7100 check_reload_status
    39 0 0 0 SL mdwait 0xffffff0002a46800 [md0]
    23 0 0 0 SL sdflush 0xffffffff811a3738 [softdepflush]
    22 0 0 0 SL vlruwt 0xffffff00028b8000 [vnlru]
    21 0 0 0 SL syncer 0xffffffff81180e40 [syncer]
    20 0 0 0 SL psleep 0xffffffff81180968 [bufdaemon]
    19 0 0 0 SL pollid 0xffffffff8115ba48 [idlepoll]
    18 0 0 0 SL pgzero 0xffffffff811a51cc [pagezero]
    17 0 0 0 SL psleep 0xffffffff811a4568 [vmdaemon]
    16 0 0 0 SL psleep 0xffffffff811a452c [pagedaemon]
    9 0 0 0 SL ccb_scan 0xffffffff811229e0 [xpt_thrd]
    8 0 0 0 SL pftm 0xffffffff80204dd0 [pfpurge]
    7 0 0 0 SL waiting_ 0xffffffff8118ce00 [sctp_iterator]
    15 0 0 0 SL (threaded) usb
    64079 D - 0xffffff8000484dd0 [usbus6]
    64078 D - 0xffffff8000484d78 [usbus6]
    64077 D - 0xffffff8000484d20 [usbus6]
    64076 D - 0xffffff8000484cc8 [usbus6]
    64075 D - 0xffffff800047bef0 [usbus5]
    64074 D - 0xffffff800047be98 [usbus5]
    64073 D - 0xffffff800047be40 [usbus5]
    64072 D - 0xffffff800047bde8 [usbus5]
    64071 D - 0xffffff8000472ef0 [usbus4]
    64070 D - 0xffffff8000472e98 [usbus4]
    64069 D - 0xffffff8000472e40 [usbus4]
    64068 D - 0xffffff8000472de8 [usbus4]
    64066 D - 0xffffff8000469ef0 [usbus3]
    64065 D - 0xffffff8000469e98 [usbus3]
    64064 D - 0xffffff8000469e40 [usbus3]
    64063 D - 0xffffff8000469de8 [usbus3]
    64049 D - 0xffffff80003c4dd0 [usbus2]
    64048 D - 0xffffff80003c4d78 [usbus2]
    64047 D - 0xffffff80003c4d20 [usbus2]
    64046 D - 0xffffff80003c4cc8 [usbus2]
    64044 D - 0xffffff80003bbef0 [usbus1]
    64043 D - 0xffffff80003bbe98 [usbus1]
    64042 D - 0xffffff80003bbe40 [usbus1]
    64041 D - 0xffffff80003bbde8 [usbus1]
    64039 D - 0xffffff80003b2ef0 [usbus0]
    64038 D - 0xffffff80003b2e98 [usbus0]
    64037 D - 0xffffff80003b2e40 [usbus0]
    64036 D - 0xffffff80003b2de8 [usbus0]
    14 0 0 0 SL - 0xffffffff8115c904 [yarrow]
    6 0 0 0 SL crypto_r 0xffffffff811a2660 [crypto returns]
    5 0 0 0 SL crypto_w 0xffffffff811a2620 [crypto]
    4 0 0 0 SL - 0xffffffff811586e8 [g_down]
    3 0 0 0 SL - 0xffffffff811586e0 [g_up]
    2 0 0 0 SL - 0xffffffff811586d0 [g_event]
    13 0 0 0 SL (threaded) ng_queue
    64017 D sleep 0xffffffff810d19d0 [ng_queue3]
    64016 D sleep 0xffffffff810d19d0 [ng_queue2]
    64015 D sleep 0xffffffff810d19d0 [ng_queue1]
    64014 D sleep 0xffffffff810d19d0 [ng_queue0]
    12 0 0 0 RL (threaded) intr
    64096 I [swi1: netisr 1]
    64095 I [swi1: netisr 2]
    64094 I [swi1: netisr 3]
    64082 I [irq1: atkbd0]
    64081 I [swi0: uart uart]
    64080 I [irq14: ata0]
    64067 I [irq19: uhci3]
    64062 I [irq23: uhci2 ehci1]
    64045 I [irq18: ehci0 uhci4+]
    64040 I [irq21: uhci1]
    64035 I [irq16: uhci0]
    64034 I [irq9: acpi0]
    64032 I [swi5: +]
    64030 I [swi2: cambio]
    64026 I [swi6: task queue]
    64025 I [swi6: Giant taskq]
    64013 I [swi3: vm]
    64012 RunQ [swi4: clock]
    64011 I [swi4: clock]
    64010 RunQ [swi4: clock]
    64009 RunQ [swi4: clock]
    64008 I [swi1: netisr 0]
    11 0 0 0 RL (threaded) idle
    64007 CanRun [idle: cpu0]
    64006 CanRun [idle: cpu1]
    64005 CanRun [idle: cpu2]
    64004 CanRun [idle: cpu3]
    1 0 1 0 SLs wait 0xffffff00022e78c0 [init]
    10 0 0 0 SL audit_wo 0xffffffff811a2b90 [audit]
    0 0 0 0 RLs (threaded) kernel
    64061 D - 0xffffff000259d180 [em5 txq]
    64060 D - 0xffffff000259d200 [em5 que]
    64059 D - 0xffffff0002596300 [em4 txq]
    64058 D - 0xffffff000257f580 [em4 que]
    64057 D - 0xffffff000257e980 [em3 txq]
    64056 Run CPU 1 [em3 que]
    64055 D - 0xffffff000256d380 [em2 txq]
    64054 Run CPU 3 [em2 que]
    64053 D - 0xffffff0002560a80 [em1 txq]
    64052 Run CPU 2 [em1 que]
    64051 D - 0xffffff0002555080 [em0 txq]
    64050 Run CPU 0 [em0 que]
    64033 D - 0xffffff000242f880


Log in to reply