Seems multi threading not working properly on pppoe:(
#PPPOE interface not using all queues:
[2.4.5-RELEASE][root@router.kilsol.com]/root: sysctl -a | grep '.ix.1..*x_pack'
dev.ix.1.queue3.rx_packets: 0
dev.ix.1.queue3.tx_packets: 5535356
dev.ix.1.queue2.rx_packets: 0
dev.ix.1.queue2.tx_packets: 6279749
dev.ix.1.queue1.rx_packets: 0
dev.ix.1.queue1.tx_packets: 6031232
dev.ix.1.queue0.rx_packets: 51346277
dev.ix.1.queue0.tx_packets: 9096401
#Non PPPOE interface is using all queues:
[2.4.5-RELEASE][root@router.kilsol.com]/root: sysctl -a | grep '.ix.0..*x_pack'
dev.ix.0.queue3.rx_packets: 5396172
dev.ix.0.queue3.tx_packets: 81242
dev.ix.0.queue2.rx_packets: 6706346
dev.ix.0.queue2.tx_packets: 618190
dev.ix.0.queue1.rx_packets: 6020980
dev.ix.0.queue1.tx_packets: 174660
dev.ix.0.queue0.rx_packets: 7843425
dev.ix.0.queue0.tx_packets: 48490796
[2.4.5-RELEASE][root@router.kilsol.com]/root:
#Tunables do indeed match # of cores
[2.4.5-RELEASE][root@router.kilsol.com]/root: sysctl net.isr.maxthreads
net.isr.maxthreads: 4
[2.4.5-RELEASE][root@router.kilsol.com]/root: sysctl net.isr.numthreads
net.isr.numthreads: 4
#net.isr.dispatch was set correctly
[2.4.5-RELEASE][root@router.kilsol.com]/root: cat /boot/loader.conf.local
net.isr.dispatch=deferred