Jetson AGX zero-copy implementation: Using the function cudaImportExternalMemory to import reserved physical memory results in an error

Robotics & Edge Computing Jetson & Embedded Systems Jetson AGX Orin
1

We are hoping that Jetson AGX GPU core can directly access data in reserved contiguous physical memory space.

Step 1: Modify dtb file, set the reserved memory base address to 0xb0000000, and the length of the physical memory to 0x10000000. Then successfuly reboot the system.
Step 2: Open /dev/mem.
Step 3: Call mmap to map reserved physical memory.
Step 4: Call ‘cudaImportExternalMemory’ to import the fd of ‘/dev/mem’.
This step(Step 4) results in consistent error: Failed to import external memory: invalid arguement.

code snippet:

include
include <unistd.h>
include <fcntl.h>
include <sys/mman.h>
include <cuda_runtime.h>

const size_t PHYSICAL_MEM_BASE = 0xb0000000; // 外部物理内存的基地址
const size_t PHYSICAL_MEM_SIZE = 0x10000000; // 外部物理内存的大小

int main() {
int fd = open(“/dev/mem”, O_RDWR | O_SYNC);
if (fd == -1) {
std::cerr << “Failed to open /dev/mem\n”;
return 1;
}

// 映射外部物理内存到用户空间
void* physical_mem = mmap(0, PHYSICAL_MEM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, PHYSICAL_MEM_BASE);
if (physical_mem == MAP_FAILED) {
    std::cerr << "Failed to map physical memory\n";
    close(fd);
    return 1;
}

// 导入外部物理内存到 CUDA
cudaExternalMemory_t extMemObj;
cudaExternalMemoryHandleDesc handleDesc;

// 设置外部内存句柄类型和值（示例中使用外部物理内存的地址）
handleDesc.type = cudaExternalMemoryHandleTypeOpaqueFd;
handleDesc.handle.fd = fd;

cudaError_t err = cudaImportExternalMemory(&extMemObj, &handleDesc);
if (err != cudaSuccess) {
    std::cerr << "Failed to import external memory: " << cudaGetErrorString(err) << std::endl;
    munmap(physical_mem, PHYSICAL_MEM_SIZE);
    close(fd);
    return 1;
}

// 获取设备指针并在 CUDA 中使用
void* d_extMemPtr;
err = cudaExternalMemoryGetMappedBuffer(&d_extMemPtr, extMemObj, 0);
if (err != cudaSuccess) {
    std::cerr << "Failed to get mapped buffer: " << cudaGetErrorString(err) << std::endl;
    cudaDestroyExternalMemory(extMemObj);
    munmap(physical_mem, PHYSICAL_MEM_SIZE);
    close(fd);
    return 1;
}

// 在这里可以使用 d_extMemPtr 来执行 CUDA 计算

// Use d_extMemPtr to execute Cuda computing

// 解除内存映射和关闭文件描述符
cudaDestroyExternalMemory(extMemObj);
munmap(physical_mem, PHYSICAL_MEM_SIZE);
close(fd);

return 0;

}

4

Hi,

Have you tried it with cudaExternalMemoryHandleTypeNvSciBuf?
Usually, we expect users to use NvSci on Jetson for memory handling.

Thanks.

5

Hi,

Thank you for replying. Could you provide us an example?

Thanks,
Yezheng

6

@AastaLLL I have the same problem. In the Jetson Agx orin system, when modifying the dts file, reserving physical memory, open /dev/mem file to obtain fd, and using mmap function, then using the cudaImportExternalMemory function always reports “Failed to import external memory: invalid argument”. Can’t we use this function cudaImportExternalMemory，

7

Hi, both

Please find the NvSci example in the below link:

Thanks.

8

thanks！
@AastaLLL
Jetson AGX dtb is the following code, Could this mem region be used by NvSCI， Does this memory area require special labeling to be used by nvsci ？

“/reserved-memory {
#address-cells = <2>;
#size-cells = <2>;
ranges;

reserved_memory: memory@80000000 {
    reg = <0x0 0x80000000 0x0 0x10000000>; 
    no-map;
};

};

9

If there are multiple reserved memory regions, how does nvSCI know which one to use ，thanks for your guidance

@AastaLLL

10

Hi,

You will need to allocate the memory with NvSCI API.
Exporting a pre-allocated memory is not supported.

Thanks.

12

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.