PCI passthrough Tesla K80 to KVM

Hello all,

I’m trying to get K80 passthrough to a KVM. The VM isn’t starting at all.

Passthrough is working fine with mellanox IB card.

Getting the error in libvirtd qemu logs:

char device redirected to /dev/pts/1
assigned_dev_register_regions: Error: Couldn't mmap 0x0!
2017-12-01T22:43:28.524367Z qemu-kvm: -device pci-assign,host=86:00.0,id=hostdev0,configfd=23,bus=pci.0,addr=0x5: Device 'pci-assign' could not be initialized

Have intel_iommu enabled:

[root@comet-31-20 ~]# cat /proc/cmdline
ro root=UUID=31413545-96ee-483d-9fae-828669546718 rd_NO_LUKS rd_NO_LVM LANG=en_US.UTF-8 rd_NO_MD SYSFONT=latarcyrheb-sun16  KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM rhgb quiet intel_iommu=on pci=realloc rdblacklist=nvidia

The device is detached from the host:

# readlink /sys/bus/pci/devices/0000\:86\:00.0/driver
../../../../../../bus/pci/drivers/pci-stub

My VM XML:

<domain type='kvm'>
  <name>vm-sc17k8s-00</name>
  <uuid>c6fb015d-2283-7d73-8ef3-4add97bf9e4e</uuid>
  <memory unit='KiB'>112640000</memory>
  <currentMemory unit='KiB'>112640000</currentMemory>
  <memoryBacking>
    <nosharepages/>
  </memoryBacking>
  <vcpu placement='static'>24</vcpu>
  <numatune>
    <memory mode='strict' nodeset='0-1'/>
  </numatune>
  <os>
    <type arch='x86_64' machine='rhel6.6.0'>hvm</type>
    <boot dev='network'/>
    <boot dev='hd'/>
    <bootmenu enable='yes'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pae/>
  </features>
  <cpu mode='host-passthrough'>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw'/>
      <source dev='/dev/sdb'/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </disk>
    <disk type='file' device='cdrom'>
      <driver name='qemu' type='raw'/>
      <target dev='hda' bus='ide'/>
      <readonly/>
      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
    </disk>
    <controller type='usb' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='ide' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <serial type='pty'>
      <target port='0'/>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <input type='tablet' bus='usb'/>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' keymap='en-us' passwd='55zPqpiCfa0FXtC3'/>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x86' slot='0x00' function='0x0'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </hostdev>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </memballoon>
  </devices>
</domain>

Starting command:

LC_ALL=C PATH=/sbin:/usr/sbin:/bin:/usr/bin QEMU_AUDIO_DRV=none /usr/libexec/qemu-kvm -name vm-sc17k8s-00 -S -M rhel6.6.0 -cpu host -enable-kvm -m 110000 -redhat-disable-KSM -realtime mlock=off -smp 24,sockets=24,cores=1,threads=1 -uuid c6fb015d-2283-7d73-8ef3-4add97bf9e4e -nodefconfig -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/vm-sc17k8s-00.monitor,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot order=nc,menu=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive file=/dev/sdb,if=none,id=drive-virtio-disk0,format=raw -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x3,drive=drive-virtio-disk0,id=virtio-disk0 -drive if=none,media=cdrom,id=drive-ide0-0-0,readonly=on,format=raw -device ide-drive,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -device usb-tablet,id=input0 -vnc 127.0.0.1:0,password -k en-us -vga cirrus -device pci-assign,host=86:00.0,id=hostdev0,configfd=23,bus=pci.0,addr=0x5 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x4 -msg timestamp=on

Kernel 2.6.32-696.10.3.el6.x86_64, CentOS 6.4, qemu-kvm 2:0.12.1.2-2.503.el6_9.3

These instructions may be of interest:

https://github.com/dholt/kvm-gpu

Thanks for the tip!
But getting the same error

assigned_dev_register_regions: Error: Couldn’t mmap 0x0!
2017-12-02T01:22:56.237888Z qemu-kvm: -device pci-assign,host=85:00.0,id=hostdev0,configfd=26,bus=pci.0,addr=0x5: Device ‘pci-assign’ could not be initialized