Upgrade to 2.3 crashing every 2 minutes.
-
After my upgrade my firewall has been crashing and rebooting every 2 min.
I've been submitting the crash report if i can before it crashes again, my live system is unworkable. Luckily I just had an older pFsense lying around. What a savior!
How can i distinguish if this is package related and therefore remove the problematic package.
The box is a supermicro with 4MB ram / intel atom 1.8 d2500 (i think) with Intel NIC cards and an intel SSD
-
Do you have anything in /boot/loader.conf or /boot/loader.conf.local that is trying to "tune" your NICs?
Is this part of an HA cluster?
The panic has quite a colossal backtrace that isn't familiar, but mostly lies in packet processing, and I see limiters in there as well (and pppoe, and IPsec, and …)
db:0:kdb.enter.default> bt Tracing pid 0 tid 100065 td 0xc8949640 kdb_enter(c147bc56,c147bc56,c1642e07,c1fb691c,0,...) at kdb_enter+0x3d/frame 0xc1fb68c8 vpanic(c1642e07,c1fb691c,c1fb691c,c1fb6934,c12e718b,...) at vpanic+0x13b/frame 0xc1fb68fc panic(c1642e07,0,0,0,f0f47050,...) at panic+0x1b/frame 0xc1fb6910 dblfault_handler() at dblfault_handler+0xab/frame 0xc1fb6910 --- trap 0x17, eip = 0xc0d3c962, esp = 0xf0f46ff0, ebp = 0xf0f47050 --- cpu_search_lowest(c1f95ed8,f0f47088,8,7fffffff,0,...) at cpu_search_lowest+0x172/frame 0xf0f47050 cpu_search_lowest(c1f95ec4,f0f470e0,0,8,c9307320,...) at cpu_search_lowest+0x17e/frame 0xf0f470b8 sched_pickcpu(c1f95eec,f0f47148,ffffffff,0,0,...) at sched_pickcpu+0x230/frame 0xf0f47110 sched_add(c9307320,0,c0d3dcb0,53cd41,2710) at sched_add+0x69/frame 0xf0f4714c sched_wakeup(c9307320,0,c9307320,f0f471b4,c0d60611,...) at sched_wakeup+0x161/frame 0xf0f47174 setrunnable(c9307320,0,f,0,0,...) at setrunnable+0x42/frame 0xf0f47188 sleepq_resume_thread(0,0,c148023a,344,1,...) at sleepq_resume_thread+0x101/frame 0xf0f471b4 sleepq_signal(c9352000,0,0,0,c8949640,...) at sleepq_signal+0x99/frame 0xf0f471e0 wakeup_one(c9352000) at wakeup_one+0x33/frame 0xf0f47200 taskqueue_thread_enqueue(c98ed918,c1ead400,c0d100be,f0f9007c,f0f47238,...) at taskqueue_thread_enqueue+0x11/frame 0xf0f4720c taskqueue_enqueue(c9352000,c98ed91c,f0f47258,c98e3cdb,c9352000,...) at taskqueue_enqueue+0x13e/frame 0xf0f47238 taskqueue_enqueue_fast(c9352000,c98ed91c,f0f472e4,c0d29032,0,...) at taskqueue_enqueue_fast+0x18/frame 0xf0f47248 dummynet(0,6e1675c,0,0,0,...) at dummynet+0x1b/frame 0xf0f47258 softclock_call_cc(1,f0f47358,c0cb2fd2,1,0,...) at softclock_call_cc+0x192/frame 0xf0f472e4 callout_process(881801f,1575,f0f47388,c0d1ba4d,c9352000,...) at callout_process+0x26e/frame 0xf0f47358 handleevents(0,1,f0f473d4,c0d3f380,1,...) at handleevents+0x270/frame 0xf0f473a8 timercb(c88e1064,0,0,0,c9f2a100,...) at timercb+0x346/frame 0xf0f47404 hpet_intr_single(c88e1064,c88b66c0,f0f474b0,0,f0f47468,...) at hpet_intr_single+0x190/frame 0xf0f47430 hpet_intr(c88e1000,0,c8949640,14,c86f4f14,...) at hpet_intr+0x6d/frame 0xf0f47448 intr_event_handle(c8714100,f0f474b0,570dbe50,f0f474b0,6e00,...) at intr_event_handle+0x85/frame 0xf0f47468 intr_execute_handlers(c86f4f14,f0f474b0,0) at intr_execute_handlers+0x42/frame 0xf0f47488 lapic_handle_intr(33,f0f474b0) at lapic_handle_intr+0x40/frame 0xf0f474a0 Xapic_isr1() at Xapic_isr1+0x43/frame 0xf0f474a0 --- interrupt, eip = 0xc0f9d6e2, esp = 0xf0f474f0, ebp = 0xf0f477b8 --- pf_test_rule(1,c8f1ed00,c9f2a100,14,f0f47960,...) at pf_test_rule+0xd52/frame 0xf0f477b8 pf_test(1,c89eec00,f0f47a64,0,1,...) at pf_test+0x810/frame 0xf0f47a20 pf_check_in(0,f0f47a64,c89eec00,1,0,...) at pf_check_in+0x29/frame 0xf0f47a40 pfil_run_hooks(c2020b7c,f0f47ae4,c89eec00,1,0,...) at pfil_run_hooks+0x88/frame 0xf0f47a98 ipsec_filter(f0f47ae4,1,102,102,4,...) at ipsec_filter+0xe8/frame 0xf0f47ac0 ipsec4_common_input_cb(c9f2a100,c9f8c200,14,9,f0f47d1c,...) at ipsec4_common_input_cb+0x5c5/frame 0xf0f47b38 esp_input_cb(ca204bf4,ca03906c,ca204bf4,f0f47c78,c0fdc549,...) at esp_input_cb+0x7a7/frame 0xf0f47c40 crypto_done(ca204bf4,ca03906c,8,f0f47cf0,f0f48258,...) at crypto_done+0x9e/frame 0xf0f47c78 swcr_process(c85c8580,ca204bf4,0,c268d000,0,...) at swcr_process+0xcaf/frame 0xf0f48340 crypto_invoke(0,c9924bb8,c,c9924bb8,ca204bf4,...) at crypto_invoke+0x79/frame 0xf0f48374 crypto_dispatch(ca204bf4,c14a6a48,1b2,c9924bb8,c0d887a8,...) at crypto_dispatch+0x64/frame 0xf0f48398 esp_input(c9f2a100,c9f8c200,14,9,d7,...) at esp_input+0x74c/frame 0xf0f48480 ipsec_common_input(c9f2a100,14,9,2,32) at ipsec_common_input+0x597/frame 0xf0f4851c esp4_input(c9f2a100,14,c88aa000,1,0,...) at esp4_input+0x30/frame 0xf0f48538 ip_input(c9f2a100,f0f485d8,c1f95eec,f0f485e0,0,...) at ip_input+0x2d2/frame 0xf0f485a0 netisr_dispatch_src(1,0,c9f2a100) at netisr_dispatch_src+0x8b/frame 0xf0f485e8 netisr_dispatch(1,c9f2a100,c0d3c96e,c1f95ed8,f0f48640,...) at netisr_dispatch+0x20/frame 0xf0f485fc ng_iface_rcvdata(c9106800,c9fba8c0,0,0,c0,...) at ng_iface_rcvdata+0xea/frame 0xf0f48624 ng_apply_item(0,f0f486b8,ffffffff,c26609f0,0,...) at ng_apply_item+0x22d/frame 0xf0f48680 ng_snd_item(c9fba8c0,0,c9106900,0,8000002,...) at ng_snd_item+0x19d/frame 0xf0f486b4 ng_tcpmss_rcvdata(c9922c00,c9fba8c0,0,0,0,...) at ng_tcpmss_rcvdata+0xa8/frame 0xf0f486fc ng_apply_item(0,2,0,2,ffffffff,...) at ng_apply_item+0x22d/frame 0xf0f48758 ng_snd_item(c9fba8c0,0,c9352100,0,c9fba8c0,...) at ng_snd_item+0x19d/frame 0xf0f4878c ng_ppp_comp_recv(21,0,1,c9f2a100,c8fd6000,...) at ng_ppp_comp_recv+0x158/frame 0xf0f487b8 ng_ppp_crypt_recv(21,0,100,c0d3e1f1,c9307320,...) at ng_ppp_crypt_recv+0x70/frame 0xf0f487d4 ng_ppp_rcvdata(c9033a80,c9fba8c0,0,1c0,1,...) at ng_ppp_rcvdata+0x2db/frame 0xf0f48830 ng_apply_item(0,c1f95ec4,46,1,0,...) at ng_apply_item+0x22d/frame 0xf0f4888c ng_snd_item(c9fba8c0,0,c9033a00,0,c8711320,...) at ng_snd_item+0x19d/frame 0xf0f488c0 ng_tee_rcvdata(c9352280,c9fba8c0,c2676b40,0,f0f48940,...) at ng_tee_rcvdata+0x156/frame 0xf0f488e8 ng_apply_item(0,ca039008,6,ca08da00,f0f48978,...) at ng_apply_item+0x22d/frame 0xf0f48944 ng_snd_item(c9fba8c0,0,c9352500,0,53c417d8,...) at ng_snd_item+0x19d/frame 0xf0f48978 ng_pppoe_rcvdata_ether(c9352300,c9fba8c0,c0d3c96e,0,f0f48a38,...) at ng_pppoe_rcvdata_ether+0x2a3/frame 0xf0f489f4 ng_apply_item(0,c8992000,c9f2a100,0,f0f48a84,...) at ng_apply_item+0x22d/frame 0xf0f48a50 ng_snd_item(c9fba8c0,0,c9352380,0,c8992000,...) at ng_snd_item+0x19d/frame 0xf0f48a84 ng_ether_input_orphan(c8992000,c9f2a100,fffffff1,156d,c9003fc8,...) at ng_ether_input_orphan+0x66/frame 0xf0f48aa4 ether_demux(c8992000,c9f2a100,6,7fe,ca029802,...) at ether_demux+0x1e7/frame 0xf0f48ad0 ether_nh_input(c9f2a100,39,c8931000,f0f48b88,c078c1db,...) at ether_nh_input+0x36b/frame 0xf0f48b1c netisr_dispatch_src(9,0,c9f2a100) at netisr_dispatch_src+0x8b/frame 0xf0f48b64 netisr_dispatch(9,c9f2a100) at netisr_dispatch+0x20/frame 0xf0f48b78 ether_input(c8992000,c9f2a100,f0f48be8,c8949640,6097cc7d,...) at ether_input+0x19/frame 0xf0f48b88 lem_rxeof(0,0,f0f48c34,c0d5f7cf,c8949640,...) at lem_rxeof+0x4a2/frame 0xf0f48bf8 lem_handle_rxtx(c8931000,1,c0d1b843,c898ec80,0,...) at lem_handle_rxtx+0x30/frame 0xf0f48c28 taskqueue_run_locked(c898ec80,c898ec98,c1467538,0,0,...) at taskqueue_run_locked+0xee/frame 0xf0f48c6c taskqueue_thread_loop(c89335dc,f0f48ce8,effdee93,e9fffcd1,6febff37,...) at taskqueue_thread_loop+0xc7/frame 0xf0f48ca4 fork_exit(c0d638e0,c89335dc,f0f48ce8) at fork_exit+0xa3/frame 0xf0f48cd4 fork_trampoline() at fork_trampoline+0x8/frame 0xf0f48cd4
Fatal double fault: eip = 0xc0d3c962 esp = 0xf0f46ff0 ebp = 0xf0f47050 cpuid = 0; apic id = 00 panic: double fault cpuid = 0 KDB: enter: panic
-
Thank You for you response, the crash log you have displayed is this from my machine?
it does look familiar, I'm asking because I'm surprised you located it.
I have not checked the loader.config files I will setup the unit on a secondary network and examine the files. The pfsense is not part of a cluster.
Do you suggest a clean install?
-
That is from your crash report submission, yes.
A clean install may help, but really there isn't enough information to say for sure. If you did have some customizations held over from older versions on the filesystem, it's possible one of those was hurting things, which is why I asked about /boot/loader.conf.local. I haven't seen this specific crash before, but we have seen other issues from leftover "tuning" that is no longer necessary.
-
I've had the exact same issue, with crashing quite frequently.
I tracked this down to the use of IPv6, where disabling the interface that maps to my HE.NET tunnel stopped the frequent crashing.
I assumed it was related to IPv6 because of the presence of related stack frames.
It started after upgrading to 2.3, tested again with 2.3.1-RELEASE-p5 (i386), crashes similarly.
Thanks.
– Here's my crash stack:
Tracing pid 12 tid 100029 td 0xc706e320 kdb_enter(c147c596,c147c596,c1643747,c1fb7b9c,0,...) at kdb_enter+0x3d/frame 0xc1fb7b48 vpanic(c1643747,c1fb7b9c,c1fb7b9c,c1fb7bb4,c12e79ab,...) at vpanic+0x13b/frame 0xc1fb7b7c panic(c1643747,0,0,0,c6bd1030,...) at panic+0x1b/frame 0xc1fb7b90 dblfault_handler() at dblfault_handler+0xab/frame 0xc1fb7b90 --- trap 0x17, eip = 0xc0d3c828, esp = 0xc6bd0fd0, ebp = 0xc6bd1030 --- cpu_search_lowest(c1f97158,c6bd1068,2,56f000,0,...) at cpu_search_lowest+0x18/frame 0xc6bd1030 cpu_search_lowest(c1f97144,c6bd10c0,0,54,c720b320,...) at cpu_search_lowest+0x17e/frame 0xc6bd1098 sched_pickcpu(c6bd1120,c1f97158,c6bd1128,0,2c0,...) at sched_pickcpu+0x230/frame 0xc6bd10ec sched_add(c720b320,0,1,d2ff05,2710) at sched_add+0x69/frame 0xc6bd1128 sched_wakeup(c720b320,c130ddd8,c720b320,c6bd1190,c0d60631,...) at sched_wakeup+0x161/frame 0xc6bd1150 setrunnable(c720b320,c6bd1198,0,c6bd1200,c0d10056,...) at setrunnable+0x42/frame 0xc6bd1164 sleepq_resume_thread(0,0,c1480b7a,37e,c706e320,...) at sleepq_resume_thread+0x101/frame 0xc6bd1190 sleepq_timeout(c720b320,c354e56e,c7091000,0,0,...) at sleepq_timeout+0x93/frame 0xc6bd11b8 softclock_call_cc(1,c6bd12b8,c0cb2ff2,1,0,...) at softclock_call_cc+0x192/frame 0xc6bd1244 callout_process(f192ad29,3604,c6bd1330,c0d3c98e,c1f97158,...) at callout_process+0x26e/frame 0xc6bd12b8 handleevents(0,c2,0,0,0,...) at handleevents+0x270/frame 0xc6bd1308 timercb(c7091064,0,1,c6f10c80,700,...) at timercb+0x346/frame 0xc6bd1364 hpet_intr_single(c7091064,c7097d40,c6bd1410,0,c6bd13c8,...) at hpet_intr_single+0x190/frame 0xc6bd1390 hpet_intr(c7091000,0,c706e320,14,c6ef4f14,...) at hpet_intr+0x6d/frame 0xc6bd13a8 intr_event_handle(c6f14100,c6bd1410,2710,c6bd1410,c1e907fc,...) at intr_event_handle+0x85/frame 0xc6bd13c8 intr_execute_handlers(c6ef4f14,c6bd1410,0) at intr_execute_handlers+0x42/frame 0xc6bd13e8 lapic_handle_intr(32,c6bd1410) at lapic_handle_intr+0x40/frame 0xc6bd1400 Xapic_isr1() at Xapic_isr1+0x43/frame 0xc6bd1400 --- interrupt, eip = 0xc12d6087, esp = 0xc6bd1450, ebp = 0xc6bd1454 --- spinlock_exit(c1e907fc,0,0,0) at spinlock_exit+0x27/frame 0xc6bd1454 wakeup_one(c1e907fc,c8304a00,5d,0,6f7c249) at wakeup_one+0x3d/frame 0xc6bd1474 ng_worklist_add(50939500,338,28,1,10000000,...) at ng_worklist_add+0xc0/frame 0xc6bd149c ng_queue_rw(0,c7646800,c841f900,0,c6bd14fc,...) at ng_queue_rw+0xa5/frame 0xc6bd14c8 ng_snd_item(c7f259c0,0,c769c180,0,5c,...) at ng_snd_item+0xde/frame 0xc6bd14fc ng_iface_send(2,1,0,c8304e00,1,...) at ng_iface_send+0xe4/frame 0xc6bd152c ng_iface_start(c7646800,c841f900,0,f5ae2a93,f19140c1,...) at ng_iface_start+0x42/frame 0xc6bd1560 if_transmit(c7646800,c841f900,4,c841f900,0,...) at if_transmit+0x154/frame 0xc6bd1598 ng_iface_output(c7646800,c841f900,c6bd17f8,0,c6bd1770,...) at ng_iface_output+0x1ca/frame 0xc6bd15c8 pf_test(2,c7646800,c6bd1874,0,0,...) at pf_test+0x2a5e/frame 0xc6bd1830 pf_check_out(0,c6bd1874,c7646800,2,0,...) at pf_check_out+0x29/frame 0xc6bd1850 pfil_run_hooks(c2021dfc,c6bd1944,c7646800,2,0,...) at pfil_run_hooks+0x88/frame 0xc6bd18a8 ip_output(c841f900,0,0,0,0,...) at ip_output+0xa3f/frame 0xc6bd1968 in_gif_output(c7207400,c841f900,29,0,c7582c80,...) at in_gif_output+0x135/frame 0xc6bd19bc gif_transmit(c7207400,c841f900,7,1c) at gif_transmit+0x37a/frame 0xc6bd19f0 gif_output(c7207400,c841f900,c6bd1ce8,0,be74,fd011080,fc0ac741) at gif_output+0x4b/frame 0xc6bd1a10 nd6_output_ifp(c841f900,c6bd1ce8,0,0,0,...) at nd6_output_ifp+0xe3/frame 0xc6bd1a40 nd6_output(c7207400,c7207400,c841f900,c6bd1ce8,0,...) at nd6_output+0x1aa/frame 0xc6bd1ac0 pf_test6(2,c7207400,c6bd1d6c,c8e24ccc,c6bd1fd8,...) at pf_test6+0x27c1/frame 0xc6bd1d28 pf_check6_out(0,c6bd1d6c,c7207400,2,c8e24ccc,...) at pf_check6_out+0x29/frame 0xc6bd1d48 pfil_run_hooks(c20237e4,c6bd1efc,c7207400,2,c8e24ccc,...) at pfil_run_hooks+0x88/frame 0xc6bd1da0 ip6_output(c841f900,0,c6bd1fd8,0,0,...) at ip6_output+0x1464/frame 0xc6bd1f28 tcp_output(c7f3b8e8,41c70ac9,41c70afc,0,c20226ec,...) at tcp_output+0x1d8b/frame 0xc6bd2098 tcp_do_segment(c78af9f0,c7f3b8e8,48,33,0,...) at tcp_do_segment+0x3d7a/frame 0xc6bd2150 tcp_input(c8438a00,28,0,c6bd22e0,0,...) at tcp_input+0x12e7/frame 0xc6bd2258 tcp6_input(c6bd236c,c6bd2368,6,0,0,...) at tcp6_input+0x1b9/frame 0xc6bd2298 ip6_input(c8438a00,c7646800,0,c0e85166,c1e85240,...) at ip6_input+0xca2/frame 0xc6bd2448 netisr_dispatch_src(a,0,c8438a00) at netisr_dispatch_src+0x8b/frame 0xc6bd2490 netisr_dispatch(a,c8438a00,7b,0,c0e8eff6,...) at netisr_dispatch+0x20/frame 0xc6bd24a4 gif_input(c8438a00,c7207400,29,0,0,...) at gif_input+0x335/frame 0xc6bd24d8 in_gif_input10(c8438a00,14,4,1,14,...) at in_gif_input10+0x57/frame 0xc6bd2500 encap4_input(c8438a00,14,c7646800,1,0,...) at encap4_input+0x210/frame 0xc6bd2560 ip_input(c8438a00,c12d6070,0,c6bd260c,c0d6074a,...) at ip_input+0x2d2/frame 0xc6bd25c8 netisr_dispatch_src(1,0,c8438a00) at netisr_dispatch_src+0x8b/frame 0xc6bd2610 netisr_dispatch(1,c8438a00,4,c8438a00,2,...) at netisr_dispatch+0x20/frame 0xc6bd2624 ng_iface_rcvdata(c769c180,c7f259c0,c784cbec,c784cb98,c6bd2688,...) at ng_iface_rcvdata+0xea/frame 0xc6bd264c ng_apply_item(0,c1f96860,c6bd26dc,c0cf4203,c6bd26dc,...) at ng_apply_item+0x22d/frame 0xc6bd26a8 ng_snd_item(c7f259c0,0,c76e2380,0,0,...) at ng_snd_item+0x19d/frame 0xc6bd26dc ng_tcpmss_rcvdata(c76e2500,c7f259c0,0,0,c76e0680,...) at ng_tcpmss_rcvdata+0xa8/frame 0xc6bd2724 ng_apply_item(0,c80fc640,c841e900,0,c7f28b40,...) at ng_apply_item+0x22d/frame 0xc6bd2780 ng_snd_item(c7f259c0,0,c76e0480,0,c7f259c0,...) at ng_snd_item+0x19d/frame 0xc6bd27b4 ng_ppp_comp_recv(21,0,1,c8438a00,c7724000,...) at ng_ppp_comp_recv+0x158/frame 0xc6bd27e0 ng_ppp_crypt_recv(21,0,0,c6bd282c,c0d889ed,...) at ng_ppp_crypt_recv+0x70/frame 0xc6bd27fc ng_ppp_rcvdata(c76e2400,c7f259c0,c7ead01a,c81d67c0,c706e320,...) at ng_ppp_rcvdata+0x2db/frame 0xc6bd2858 ng_apply_item(0,101a8c0,c8e2b1f8,c841e900,0,...) at ng_apply_item+0x22d/frame 0xc6bd28b4 ng_snd_item(c7f259c0,0,c76e0e80,0,c6bd2a08,...) at ng_snd_item+0x19d/frame 0xc6bd28e8 ng_tee_rcvdata(c76e0100,c7f259c0,264,c841e900,0,...) at ng_tee_rcvdata+0x156/frame 0xc6bd2910 ng_apply_item(0,c8480006,6,400000c0,c841e900,...) at ng_apply_item+0x22d/frame 0xc6bd296c ng_snd_item(c7f259c0,0,c76e0180,0,b275d85b,...) at ng_snd_item+0x19d/frame 0xc6bd29a0 ng_pppoe_rcvdata_ether(c76e0500,c7f259c0,0,c6bd2a88,c70dae40,...) at ng_pppoe_rcvdata_ether+0x2a3/frame 0xc6bd2a1c ng_apply_item(0,c70cd800,c8438a00,0,c6bd2aac,...) at ng_apply_item+0x22d/frame 0xc6bd2a78 ng_snd_item(c7f259c0,0,c76e0300,0,c70cd800,...) at ng_snd_item+0x19d/frame 0xc6bd2aac ng_ether_input_orphan(c70cd800,c8438a00,c8e20fc8,c6bd2b20,c1052dad,...) at ng_ether_input_orphan+0x66/frame 0xc6bd2acc ether_demux(c70cd800,c8438a00,6,c70d2240,c6bd2b90,...) at ether_demux+0x1e7/frame 0xc6bd2af8 ether_nh_input(c8438a00,c709d9bc,c709d000,c838a900,c6bd2bb4,...) at ether_nh_input+0x36b/frame 0xc6bd2b48 netisr_dispatch_src(9,0,c8438a00) at netisr_dispatch_src+0x8b/frame 0xc6bd2b90 netisr_dispatch(9,c8438a00) at netisr_dispatch+0x20/frame 0xc6bd2ba4 ether_input(c70cd800,c8438a00,c6bd2bf8,c6f16000,0,...) at ether_input+0x19/frame 0xc6bd2bb4 re_rxeof(c1f86e80,c6f16000,0,0,0,...) at re_rxeof+0x4a4/frame 0xc6bd2c08 re_intr_msi(c709d000,0,246,0,a525361c,...) at re_intr_msi+0x107/frame 0xc6bd2c3c intr_event_execute_handlers(109,c6f15880,c147739b,55b,ffffffff,...) at intr_event_execute_handlers+0xaa/frame 0xc6bd2c68 ithread_loop(c70cadc0,c6bd2ce8,ffffffff,ffffffff,fffbffff,...) at ithread_loop+0x80/frame 0xc6bd2ca4 fork_exit(c0cd81d0,c70cadc0,c6bd2ce8) at fork_exit+0xa3/frame 0xc6bd2cd4 fork_trampoline() at fork_trampoline+0x8/frame 0xc6bd2cd4