How to use non-legacy virtio networking with libvirt?

nazar-pc

I just noticed all network interfaces are recognized by pfSense 2.8.0 as "VirtIO PCI (legacy) Network adapter".

I then wondered why are they legacy and tried to make them non-legacy by adding -global virtio-blk-pci.disable-legacy=on -global virtio-blk-pci.disable-modern=false to QEMU CLI, but it didn't seem to make any difference.

Am I missing something else?

Performance of virtio network interface is quite bad right now, can't even saturate 2Gbps on 3.3 GHz CPU due to it being seemingly single-threaded (https://redmine.pfsense.org/issues/14802 / https://forum.netgate.com/post/807755).

nazar-pc

Experimented some with this. Looks like system need to boot with UEFI. With UEFI network interface is detected as "VirtIO PCI (modern) Network adapter".

nazar-pc

Small update: I had to re-create PCIe devices, including disk and network. Not sure why, but they remained "legacy" no matter what, so I copied and tweaked config from another VM that was recognized as "modern". Only bus and slot were different, yet it somehow made a big difference.

So make sure VM is running as UEFI and re-create devices if necessary, then they'll be recognized as "modern".

wickeren

@nazar-pc

Did you see a big performance improvement because of this? In the past I have seen bad performance and/or high CPU-usage, mainly IRQ.
Is UEFI boot enough or did you still need the directives for the QEMU CLI mentioned in your first post?

nazar-pc

@wickeren Honestly, at least peak performance of the interface appears to be about the same.
I was not using any explicit CLI options, including those above, just reconfigured all PCIe devices in libvirt's XML definition of the domain.
It is still atrociously slow comparing to Linux performance, there I have no problem getting well over 10 Gbps on virtio network interfaces.

wickeren

@nazar-pc

Did you enable multi-queue? "modern" should support that, "legacy" not. That will spread the load among multiple CPU's

I'm on Proxmox which is KVM too, but despite any setting I've used the Virtio adapter is always recognized as "legacy". It's UEFI and Q35, but I'm unable to to force it to "modern"...

nazar-pc

@wickeren I actually had it enabled with legacy version (but I didn't make a difference), while switching to modern I removed it.
Probably should add back and see if there is a difference, however as mentioned in the links in the first post, I don't think pfSense has corresponding support enabled in the kernel anyway

There must be something equivalent in Proxmox as well, it probably designs PCIe architecture in a way that produces legacy devices just like it was in my case originally.
I'm still puzzled as to why that was the case, but glad it is resolved.

Here is the full QEMU command that libvirt generates for the VM in case it is helpful:

/usr/bin/qemu-system-x86_64 -name guest=pfSense,debug-threads=on -S -object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-26-pfSense/master-key.aes"} -blockdev {"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE_4M.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"} -blockdev {"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/pfSense_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"} -machine pc-q35-8.2,usb=off,dump-guest-core=off,memory-backend=pc.ram,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,hpet=off,acpi=on -accel kvm -cpu host,migratable=on -m size=2097152k -object {"qom-type":"memory-backend-ram","id":"pc.ram","size":2147483648} -overcommit mem-lock=off -smp 8,sockets=1,dies=1,cores=8,threads=1 -uuid REDACTED -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=38,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot menu=off,strict=on -device {"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x2"} -device {"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x2.0x1"} -device {"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x2.0x2"} -device {"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x2.0x3"} -device {"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x2.0x4"} -device {"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x2.0x5"} -device {"driver":"ich9-usb-ehci1","id":"usb","bus":"pcie.0","addr":"0x1d.0x7"} -device {"driver":"ich9-usb-uhci1","masterbus":"usb.0","firstport":0,"bus":"pcie.0","multifunction":true,"addr":"0x1d"} -device {"driver":"ich9-usb-uhci2","masterbus":"usb.0","firstport":2,"bus":"pcie.0","addr":"0x1d.0x1"} -device {"driver":"ich9-usb-uhci3","masterbus":"usb.0","firstport":4,"bus":"pcie.0","addr":"0x1d.0x2"} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/pfSense.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2","file":"libvirt-1-storage","backing":null} -device {"driver":"virtio-blk-pci","bus":"pci.3","addr":"0x0","drive":"libvirt-1-format","id":"virtio-disk0","bootindex":1} -netdev {"type":"tap","fd":"39","vhost":true,"vhostfd":"44","id":"hostnet0"} -device {"driver":"virtio-net-pci","netdev":"hostnet0","id":"net0","mac":"REDACTED","bus":"pci.1","addr":"0x0"} -netdev {"type":"tap","fd":"45","vhost":true,"vhostfd":"46","id":"hostnet1"} -device {"driver":"virtio-net-pci","netdev":"hostnet1","id":"net1","mac":"REDACTED","bus":"pci.2","addr":"0x0"} -netdev {"type":"tap","fd":"47","vhost":true,"vhostfd":"48","id":"hostnet2"} -device {"driver":"virtio-net-pci","netdev":"hostnet2","id":"net2","mac":"REDACTED","bus":"pci.5","addr":"0x0"} -netdev {"type":"tap","fd":"49","vhost":true,"vhostfd":"50","id":"hostnet3"} -device {"driver":"virtio-net-pci","netdev":"hostnet3","id":"net3","mac":"REDACTED","bus":"pci.6","addr":"0x0"} -chardev pty,id=charserial0 -device {"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0} -audiodev {"id":"audio1","driver":"spice"} -spice port=5901,addr=127.0.0.1,disable-ticketing=on,seamless-migration=on -device {"driver":"qxl-vga","id":"video0","max_outputs":1,"ram_size":67108864,"vram_size":67108864,"vram64_size_mb":0,"vgamem_mb":16,"bus":"pcie.0","addr":"0x1"} -global ICH9-LPC.noreboot=off -watchdog-action reset -device {"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.4","addr":"0x0"} -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

And this is XML domain config it was generated from:

<domain type="kvm">
  <name>pfSense</name>
  <uuid>REDACTED</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://freebsd.org/freebsd/14.0"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit="KiB">2097152</memory>
  <currentMemory unit="KiB">2097152</currentMemory>
  <vcpu placement="static" cpuset="8-11,24-27">8</vcpu>
  <os firmware="efi">
    <type arch="x86_64" machine="pc-q35-8.2">hvm</type>
    <firmware>
      <feature enabled="no" name="enrolled-keys"/>
      <feature enabled="no" name="secure-boot"/>
    </firmware>
    <loader readonly="yes" secure="no" type="pflash">/usr/share/OVMF/OVMF_CODE_4M.fd</loader>
    <nvram template="/usr/share/OVMF/OVMF_VARS_4M.fd">/var/lib/libvirt/qemu/nvram/pfSense_VARS.fd</nvram>
    <boot dev="hd"/>
    <bootmenu enable="no"/>
  </os>
  <features>
    <acpi/>
    <apic/>
  </features>
  <cpu mode="host-passthrough" check="none" migratable="on">
    <topology sockets="1" dies="1" cores="8" threads="1"/>
  </cpu>
  <clock offset="utc">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
    <timer name="hpet" present="no"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type="file" device="disk">
      <driver name="qemu" type="qcow2"/>
      <source file="/var/lib/libvirt/images/pfSense.qcow2"/>
      <target dev="vda" bus="virtio"/>
      <address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
    </disk>
    <controller type="sata" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pcie-root"/>
    <controller type="pci" index="1" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="1" port="0x10"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="2" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="2" port="0x11"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
    </controller>
    <controller type="pci" index="3" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="3" port="0x12"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
    </controller>
    <controller type="pci" index="4" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="4" port="0x13"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
    </controller>
    <controller type="pci" index="5" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="5" port="0x14"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
    </controller>
    <controller type="pci" index="6" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="6" port="0x15"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
    </controller>
    <controller type="pci" index="7" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="7" port="0x16"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x6"/>
    </controller>
    <controller type="usb" index="0" model="qemu-xhci" ports="15">
      <address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
    </controller>
    <interface type="bridge">
      <mac address="REDACTED"/>
      <source bridge="wan"/>
      <target dev="pfsense-wan"/>
      <model type="virtio"/>
      <address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
    </interface>
    <interface type="bridge">
      <mac address="REDACTED"/>
      <source bridge="wan2"/>
      <target dev="pfsense-wan2"/>
      <model type="virtio"/>
      <address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
    </interface>
    <interface type="bridge">
      <mac address="REDACTED"/>
      <source bridge="lan"/>
      <target dev="pfsense-lan"/>
      <model type="virtio"/>
      <address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
    </interface>
    <interface type="bridge">
      <mac address="REDACTED"/>
      <source bridge="guest"/>
      <target dev="pfsense-guest"/>
      <model type="virtio"/>
      <address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
    </interface>
    <serial type="pty">
      <target type="isa-serial" port="0">
        <model name="isa-serial"/>
      </target>
    </serial>
    <console type="pty">
      <target type="serial" port="0"/>
    </console>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <graphics type="spice" autoport="yes">
      <listen type="address"/>
    </graphics>
    <audio id="1" type="spice"/>
    <video>
      <model type="virtio" heads="1" primary="yes"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"/>
    </video>
    <watchdog model="itco" action="reset"/>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
    </memballoon>
  </devices>
</domain>