Hi all:
my step as follow:
1, create raid0 : # mdadm -C -v /dev/md0 -l 0 -n 2 /dev/nvme1n1 /dev/nvme2n1
2, mkfs.ex4 : # mkfs.ext4 /dev/md0
3, mount : # mount -o data=ordered /dev/md0 /test
4, run gdsio test:
root@dpu:/usr/local/cuda-12.1/gds/tools# ./gdsio -D /test -d 0 -w 4 -s 1G -i 4096k -x 0 -I 1 -T 60
file register error: GPUDirect Storage not supported on current file filename :/test/gdsio.0
root@dpu:/usr/local/cuda-12.1/gds/tools#
dmesg have not error.
cufile.log errors as follow:
26-04-2023 03:26:08:220 [pid=3159 tid=3159] ERROR cufio-topo-nvfs:78 pci device not present in topology device attribute table: 0000:45:00.0
26-04-2023 03:26:08:221 [pid=3159 tid=3159] ERROR cufio-topo-nvfs:78 pci device not present in topology device attribute table: 0000:45:00.0
26-04-2023 03:26:08:234 [pid=3159 tid=3159] ERROR cufio-fs:87 cannot verify RAID members, RAID group size empty
26-04-2023 03:26:08:234 [pid=3159 tid=3159] ERROR cufio-fs:807 RAID member not supported by cuFile in RAID group : /dev/md0
26-04-2023 03:26:08:234 [pid=3159 tid=3159] NOTICE cufio-fs:419 dumping volume attributes: DEVNAME:/dev/md0,ID_FS_TYPE:ext4,ID_FS_USAGE:filesystem,MD_LEVEL:raid0,ext4_journal_mode:ordered,fsid:6da0da84a17568420x,queue/logical_block_size:4096,
26-04-2023 03:26:08:234 [pid=3159 tid=3159] ERROR cufio:1550 cuFileHandleRegister error, file checks failed
26-04-2023 03:26:08:234 [pid=3159 tid=3159] ERROR cufio:1592 cuFileHandleRegister error: GPUDirect Storage not supported on current file
26-04-2023 03:26:08:235 [pid=3159 tid=3159] ERROR 0:106 cuDeviceGet failed with error 4
when I run the local nvme disk , the test is ok.
root@dpu:/usr/local/cuda-12.1/gds/tools# mount | grep /test
/dev/nvme1n1 on /test type ext4 (rw,relatime,stripe=32,data=ordered)
root@dpu:/usr/local/cuda-12.1/gds/tools#
root@dpu:/usr/local/cuda-12.1/gds/tools# ./gdsio -D /test -d 0 -w 4 -s 1G -i 4096k -x 0 -I 1 -T 60
IoType: WRITE XferType: GPUD Threads: 4 DataSetSize: 125804544/4194304(KiB) IOSize: 4096(KiB) Throughput: 1.943322 GiB/sec, Avg_Latency: 8040.564767 usecs ops: 30714 total_time 61.737861 secs
root@dpu:/usr/local/cuda-12.1/gds/tools#