Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

virtio-fs produces corrupted data on copy/move #764

Closed
ChristophSchmidpeter opened this issue May 4, 2022 · 5 comments
Closed

virtio-fs produces corrupted data on copy/move #764

ChristophSchmidpeter opened this issue May 4, 2022 · 5 comments
Assignees

Comments

@ChristophSchmidpeter
Copy link

Describe the bug
When I try to copy/move folders with content, some resulting files/folders have corrupted names and their content is unreadable.

To Reproduce

  • Copy or move folders with content several times.
  • Notice the corrupted file name (see screenshot)

I can reproduce it around 1/5 times with folders having some dozens to hundreds of MB content. This happens despite the queue="1024" workaround for the known >2MB copy issue.

Expected behavior
Copy/move of folders does not result in corrupted data.

Screenshots

ScreenshotOfCopiedFolder

Host:

  • Disto: Arch Linux

  • Kernel version: 5.17.1-zen1-1-zen

  • QEMU version: 6.2.0-4

  • QEMU command line
    /usr/bin/qemu-system-x86_64 -name guest=Win11,debug-threads=on -S -object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-37-Win11/master-key.aes"} -blockdev {"driver":"file","filename":"/usr/share/edk2-ovmf/x64/OVMF_CODE.secboot.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"} -blockdev {"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/Win11_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"} -machine pc-q35-6.2,usb=off,vmport=off,smm=on,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram -accel kvm -cpu host,migratable=off,topoext=on,hv-time=on,hv-passthrough=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,host-cache-info=on,l3-cache=off -global driver=cfi.pflash01,property=secure,value=on -m 8192 -object {"qom-type":"memory-backend-file","id":"pc.ram","mem-path":"/dev/hugepages/libvirt/qemu/37-Win11","share":true,"x-use-canonical-path-for-ramblock-id":false,"prealloc":true,"size":8589934592} -overcommit mem-lock=off -smp 12,sockets=1,dies=1,cores=6,threads=2 -uuid 004bfcd5-b6a2-4ff9-984c-d074c362b459 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=35,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=localtime,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on -device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=22,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=23,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 -device pcie-root-port,port=24,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x3 -device pcie-pci-bridge,id=pci.10,bus=pci.1,addr=0x0 -device pcie-root-port,port=25,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x1 -device pcie-root-port,port=26,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x2 -device pcie-root-port,port=27,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x3 -device pcie-root-port,port=28,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x4 -device pcie-root-port,port=29,chassis=15,id=pci.15,bus=pcie.0,addr=0x3.0x5 -device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.4,addr=0x0 -device virtio-scsi-pci,id=scsi0,bus=pci.5,addr=0x0 -device virtio-serial-pci,id=virtio-serial0,bus=pci.6,addr=0x0 -device usb-ccid,id=ccid0,bus=usb.0,port=1 -device ide-cd,bus=ide.0,id=sata0-0-0 -device ide-cd,bus=ide.1,id=sata0-0-1 -blockdev {"driver":"host_device","filename":"/dev/zvol/ZfsRoot/vm/Win11/hdOs","aio":"native","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"} -device virtio-blk-pci,bus=pci.3,addr=0x0,drive=libvirt-2-format,id=virtio-disk0,bootindex=1,write-cache=on -blockdev {"driver":"host_device","filename":"/dev/zvol/ZfsRoot/vm/Win11/hdData","aio":"native","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"} -device virtio-blk-pci,bus=pci.7,addr=0x0,drive=libvirt-1-format,id=virtio-disk1,write-cache=on -chardev socket,id=chr-vu-fs0,path=/var/lib/libvirt/qemu/domain-37-Win11/fs0-fs.sock -device vhost-user-fs-pci,id=fs0,chardev=chr-vu-fs0,queue-size=1024,tag=Shared,bus=pci.14,addr=0x0 -netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:7d:ab:61,bus=pci.2,addr=0x0 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0,index=0 -chardev spicevmc,id=charchannel0,name=vdagent -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 -chardev spiceport,id=charchannel1,name=org.spice-space.webdav.0 -device virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=org.spice-space.webdav.0 -chardev socket,id=charchannel2,fd=34,server=on,wait=off -device virtserialport,bus=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel2,name=org.qemu.guest_agent.0 -device virtio-keyboard-pci,id=input2,bus=pci.11,addr=0x0 -device virtio-tablet-pci,id=input3,bus=pci.12,addr=0x0 -audiodev {"id":"audio1","driver":"spice"} -spice port=0,disable-ticketing=on,image-compression=off,gl=on,rendernode=/dev/dri/renderD128,seamless-migration=on -device qxl-vga,id=video0,max_outputs=1,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,bus=pcie.0,addr=0x1 -device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1 -device virtio-balloon-pci,id=balloon0,bus=pci.8,addr=0x0 -object {"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"} -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.9,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -device vhost-vsock-pci,id=vsock0,guest-cid=3,vhostfd=31,bus=pci.13,addr=0x0 -msg timestamp=on

  • libvirt version: 8.2.0-7

  • libvirt XML file:

<domain type='kvm' id='37'>
 <name>Win11</name>
 <uuid>004bfcd5-b6a2-4ff9-984c-d074c362b459</uuid>
 <metadata>
   <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
     <libosinfo:os id="http://microsoft.com/win/10"/>
   </libosinfo:libosinfo>
 </metadata>
 <memory unit='KiB'>8388608</memory>
 <currentMemory unit='KiB'>8388608</currentMemory>
 <memoryBacking>
   <hugepages>
     <page size='2048' unit='KiB'/>
   </hugepages>
   <access mode='shared'/>
 </memoryBacking>
 <vcpu placement='static'>12</vcpu>
 <resource>
   <partition>/machine</partition>
 </resource>
 <os>
   <type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
   <loader readonly='yes' secure='yes' type='pflash'>/usr/share/edk2-ovmf/x64/OVMF_CODE.secboot.fd</loader>
   <nvram template='/usr/share/edk2-ovmf/x64/OVMF_VARS.fd'>/var/lib/libvirt/qemu/nvram/Win11_VARS.fd</nvram>
   <boot dev='hd'/>
 </os>
 <features>
   <acpi/>
   <apic/>
   <hyperv mode='passthrough'>
     <relaxed state='on'/>
     <vapic state='on'/>
     <spinlocks state='on' retries='8191'/>
   </hyperv>
   <vmport state='off'/>
   <smm state='on'/>
 </features>
 <cpu mode='host-passthrough' check='none' migratable='off'>
   <topology sockets='1' dies='1' cores='6' threads='2'/>
   <cache mode='passthrough'/>
   <feature policy='require' name='topoext'/>
 </cpu>
 <clock offset='localtime'>
   <timer name='rtc' tickpolicy='catchup'/>
   <timer name='pit' tickpolicy='delay'/>
   <timer name='hpet' present='no'/>
   <timer name='hypervclock' present='yes'/>
 </clock>
 <on_poweroff>destroy</on_poweroff>
 <on_reboot>restart</on_reboot>
 <on_crash>destroy</on_crash>
 <pm>
   <suspend-to-mem enabled='no'/>
   <suspend-to-disk enabled='no'/>
 </pm>
 <devices>
   <emulator>/usr/bin/qemu-system-x86_64</emulator>
   <disk type='file' device='cdrom'>
     <driver name='qemu'/>
     <target dev='sda' bus='sata'/>
     <readonly/>
     <alias name='sata0-0-0'/>
     <address type='drive' controller='0' bus='0' target='0' unit='0'/>
   </disk>
   <disk type='file' device='cdrom'>
     <driver name='qemu'/>
     <target dev='sdb' bus='sata'/>
     <readonly/>
     <alias name='sata0-0-1'/>
     <address type='drive' controller='0' bus='0' target='0' unit='1'/>
   </disk>
   <disk type='block' device='disk'>
     <driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
     <source dev='/dev/zvol/ZfsRoot/vm/Win11/hdOs' index='2'/>
     <backingStore/>
     <target dev='vda' bus='virtio'/>
     <alias name='virtio-disk0'/>
     <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
   </disk>
   <disk type='block' device='disk'>
     <driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
     <source dev='/dev/zvol/ZfsRoot/vm/Win11/hdData' index='1'/>
     <backingStore/>
     <target dev='vdb' bus='virtio'/>
     <alias name='virtio-disk1'/>
     <address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
   </disk>
   <controller type='usb' index='0' model='qemu-xhci' ports='15'>
     <alias name='usb'/>
     <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
   </controller>
   <controller type='scsi' index='0' model='virtio-scsi'>
     <alias name='scsi0'/>
     <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
   </controller>
   <controller type='virtio-serial' index='0'>
     <alias name='virtio-serial0'/>
     <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
   </controller>
   <controller type='sata' index='0'>
     <alias name='ide'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
   </controller>
   <controller type='pci' index='0' model='pcie-root'>
     <alias name='pcie.0'/>
   </controller>
   <controller type='pci' index='1' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='1' port='0x10'/>
     <alias name='pci.1'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
   </controller>
   <controller type='pci' index='2' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='2' port='0x11'/>
     <alias name='pci.2'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
   </controller>
   <controller type='pci' index='3' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='3' port='0x12'/>
     <alias name='pci.3'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
   </controller>
   <controller type='pci' index='4' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='4' port='0x13'/>
     <alias name='pci.4'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
   </controller>
   <controller type='pci' index='5' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='5' port='0x14'/>
     <alias name='pci.5'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
   </controller>
   <controller type='pci' index='6' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='6' port='0x15'/>
     <alias name='pci.6'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
   </controller>
   <controller type='pci' index='7' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='7' port='0x16'/>
     <alias name='pci.7'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
   </controller>
   <controller type='pci' index='8' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='8' port='0x17'/>
     <alias name='pci.8'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
   </controller>
   <controller type='pci' index='9' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='9' port='0x18'/>
     <alias name='pci.9'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
   </controller>
   <controller type='pci' index='10' model='pcie-to-pci-bridge'>
     <model name='pcie-pci-bridge'/>
     <alias name='pci.10'/>
     <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
   </controller>
   <controller type='pci' index='11' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='11' port='0x19'/>
     <alias name='pci.11'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
   </controller>
   <controller type='pci' index='12' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='12' port='0x1a'/>
     <alias name='pci.12'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
   </controller>
   <controller type='pci' index='13' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='13' port='0x1b'/>
     <alias name='pci.13'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/>
   </controller>
   <controller type='pci' index='14' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='14' port='0x1c'/>
     <alias name='pci.14'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x4'/>
   </controller>
   <controller type='pci' index='15' model='pcie-root-port'>
     <model name='pcie-root-port'/>
     <target chassis='15' port='0x1d'/>
     <alias name='pci.15'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x5'/>
   </controller>
   <controller type='ccid' index='0'>
     <alias name='ccid0'/>
     <address type='usb' bus='0' port='1'/>
   </controller>
   <filesystem type='mount' accessmode='passthrough'>
     <driver type='virtiofs' queue='1024'/>
     <binary path='/usr/lib/qemu/virtiofsd'/>
     <source dir='/home/vbm/Shared'/>
     <target dir='Shared'/>
     <alias name='fs0'/>
     <address type='pci' domain='0x0000' bus='0x0e' slot='0x00' function='0x0'/>
   </filesystem>
   <interface type='network'>
     <mac address='52:54:00:7d:ab:61'/>
     <source network='Vpn' portid='eb529e04-6c26-4a66-b48a-b0fa1d59331a' bridge='VirtVpn0'/>
     <target dev='vnet34'/>
     <model type='virtio'/>
     <link state='up'/>
     <alias name='net0'/>
     <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
   </interface>
   <serial type='pty'>
     <source path='/dev/pts/6'/>
     <target type='isa-serial' port='0'>
       <model name='isa-serial'/>
     </target>
     <alias name='serial0'/>
   </serial>
   <console type='pty' tty='/dev/pts/6'>
     <source path='/dev/pts/6'/>
     <target type='serial' port='0'/>
     <alias name='serial0'/>
   </console>
   <channel type='spicevmc'>
     <target type='virtio' name='com.redhat.spice.0' state='connected'/>
     <alias name='channel0'/>
     <address type='virtio-serial' controller='0' bus='0' port='1'/>
   </channel>
   <channel type='spiceport'>
     <source channel='org.spice-space.webdav.0'/>
     <target type='virtio' name='org.spice-space.webdav.0' state='disconnected'/>
     <alias name='channel1'/>
     <address type='virtio-serial' controller='0' bus='0' port='2'/>
   </channel>
   <channel type='unix'>
     <source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-37-Win11/org.qemu.guest_agent.0'/>
     <target type='virtio' name='org.qemu.guest_agent.0' state='connected'/>
     <alias name='channel2'/>
     <address type='virtio-serial' controller='0' bus='0' port='3'/>
   </channel>
   <input type='mouse' bus='ps2'>
     <alias name='input0'/>
   </input>
   <input type='keyboard' bus='ps2'>
     <alias name='input1'/>
   </input>
   <input type='keyboard' bus='virtio'>
     <alias name='input2'/>
     <address type='pci' domain='0x0000' bus='0x0b' slot='0x00' function='0x0'/>
   </input>
   <input type='tablet' bus='virtio'>
     <alias name='input3'/>
     <address type='pci' domain='0x0000' bus='0x0c' slot='0x00' function='0x0'/>
   </input>
   <graphics type='spice'>
     <listen type='none'/>
     <image compression='off'/>
     <gl enable='yes' rendernode='/dev/dri/renderD128'/>
   </graphics>
   <sound model='ich9'>
     <alias name='sound0'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
   </sound>
   <audio id='1' type='spice'/>
   <video>
     <model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/>
     <alias name='video0'/>
     <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
   </video>
   <memballoon model='virtio'>
     <alias name='balloon0'/>
     <address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
   </memballoon>
   <rng model='virtio'>
     <backend model='random'>/dev/urandom</backend>
     <alias name='rng0'/>
     <address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
   </rng>
   <vsock model='virtio'>
     <cid auto='yes' address='3'/>
     <alias name='vsock0'/>
     <address type='pci' domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
   </vsock>
 </devices>
 <seclabel type='dynamic' model='dac' relabel='yes'>
   <label>+1000:+985</label>
   <imagelabel>+1000:+985</imagelabel>
 </seclabel>
</domain>

VM:

  • Windows version: 11
  • Which driver has a problem: virtio-fs
  • Driver version or commit hash that was used to build the driver: 0.1.217

Additional context
queue="1024"

@xiagao
Copy link

xiagao commented Jun 7, 2022

I use qemu-kvm-7.0.0-2.el9.x86_64 and virtiofsd pkg, copying folder works.

rpm -qf /usr/libexec/virtiofsd

virtiofsd-1.1.0-4.el9_0.x86_64

You can have a try with virtiofsd.

@ChristophSchmidpeter
Copy link
Author

ChristophSchmidpeter commented Jul 24, 2022

@xiagao

It only fails sometimes. How many times and how large folders have you tried to copy?

virtiofsd-1.1.0-4.el9_0.x86_64

From the version number I am guessing you are using the Rust virtiofsd implementation? Or the normal ANSI C one?
The Rust one I cannot get to start via Libvirt.

@xiagao
Copy link

xiagao commented Jul 25, 2022

@xiagao

It only fails sometimes. How many times and how large folders have you tried to copy?

Hi, I tried this test scenario again for 5 times, copying about 10G files , it still works well.

virtiofsd-1.1.0-4.el9_0.x86_64

From the version number I am guessing you are using the Rust virtiofsd implementation? Or the normal ANSI C one? The Rust one I cannot get to start via Libvirt.

Yes, it's virtiofsd rust version.

@ammgws
Copy link

ammgws commented Aug 23, 2022

Yes, it's virtiofsd rust version.

@xiagao Sorry slightly offtopic but is there a reason you're using the Rust version?

@xiagao
Copy link

xiagao commented Aug 24, 2022

Hi,
As far as I know, rust version is more stable and upstream suggest to use it. Here is some description in upstream of virtiofsd.
+There is a new Rust implementation of virtiofs at
+[https://gitlab.com/virtio-fs/virtiofsd](https://gitlab.com/virtio-fs/virtiofsd);
+since this is now marked stable, new development should be done on that
+rather than the existing C version in the QEMU tree.
+The C version will still accept fixes and patches that
+are already in development for the moment.

Regards,
Xiaoling

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Development

No branches or pull requests

5 participants