How to get physical address from the dmabuf_fd?

we need DMA cross the different multiple Jetson modules.

after decode, we hope get the physical address from the dmabuf_fd

  1. driver code (ignore error handler)

int mca_get_sg_from_fd(u32 dmabuf_fd)
{
        struct dma_buf_attachment *attachment;
        struct sg_table *sgt;
        struct scatterlist *sg;
        unsigned int si;
        struct device *dev;
        struct dma_buf *dmabuf;
        int err;

        dmabuf = dma_buf_get(dmabuf_fd);
        if (IS_ERR(dmabuf)) {
                err = PTR_ERR(dmabuf);
                mca_info_fl("got dmabuf fail %d\n", err);
                goto err_buf_get;
        }

       

        mca_info_fl("got dmabuf %p %s\n", dmabuf, dmabuf->exp_name);
                
        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
        dev_set_name(dev, "try_get_sg");
                                
        attachment = dma_buf_attach(dmabuf, dev);
        if (IS_ERR(attachment)) {
                err = PTR_ERR(attachment);
                mca_info_fl("Failed to attach %d\n", err);
                goto err_buf_attach;
        }

        sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
        if (IS_ERR(sgt)) {
                err = PTR_ERR(sgt);
                mca_info_fl("Failed to attachement %d\n", err);
                goto err_buf_attachment;
        }

        mca_info_fl("got sg->nents %d\n", sgt->nents);
        mca_info_fl("got sg->orig_nents %d dma_buf->size=%ld/%lx\n",
                sgt->orig_nents, dmabuf->size, dmabuf->size);

        for_each_sg(sgt->sgl, sg, sgt->orig_nents, si) {
                dma_addr_t sg_addr = sg_dma_address(sg);
                u32 sg_len = sg_dma_len(sg);
                mca_info_fl("sgt[%d] sg_addr = 0x%lx, sg_len = %d/%#x sg_phys= %#llx\n",
                                si,
                                (unsigned long)sg_addr, sg_len, sg_len,
                                (unsigned long long)sg_phys(sg));
        }

        dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
err_buf_attachment:
        dma_buf_detach(dmabuf, attachment);

err_buf_attach:
        dma_buf_put(dmabuf);

err_buf_get:

        kfree(dev);
        return err;
}

  1. user app code, code from the demo video_decode_main.cpp
        while (1)
        {    
            struct v4l2_buffer v4l2_buf;
            struct v4l2_plane planes[MAX_PLANES];

            memset(&v4l2_buf, 0, sizeof(v4l2_buf));
            memset(planes, 0, sizeof(planes));
            v4l2_buf.m.planes = planes;

            /* Dequeue a filled buffer. */
            if (dec->capture_plane.dqBuffer(v4l2_buf, &dec_buffer, NULL, 0))
            {    
                if (errno == EAGAIN)
                {    
                    usleep(1000);
                }    
                else 
                {    
                    abort(ctx);
                    cerr << "Error while calling dequeue at capture plane" <<
                        endl;
                }    
                break;
            }    

            cout << "current v4l2_buf.index " << v4l2_buf.index << endl;
            cout << "current fd " << ctx->dmabuff_fd[v4l2_buf.index] << endl;

            **McaDmabufFd(ctx->dmabuff_fd[v4l2_buf.index]);**  /* call driver:  mca_get_sg_from_fd*/
  1. ./video_decode H264 -f 2 --disable-rendering ./1080P_H264_4m.264

kernel output by mca_get_sg_from_fd

[ 66.711521] [mca_info] ep_mca_open(), line 297
[ 66.711990] [mca_info] DMABUF_FD 1065
[ 66.711999] [mca_info] mca_get_sg_from_fd(), line 377 got dmabuf ffffffc7d5e71200 nvmap_dmabuf
[ 66.712258] [mca_info] mca_get_sg_from_fd(), line 397 got sg->nents 2
[ 66.712265] [mca_info] mca_get_sg_from_fd(), line 399 got sg->orig_nents 2 dma_buf->size=3538944/360000
[ 66.712271] [mca_info] mca_get_sg_from_fd(), line 407 sgt[0] sg_addr = 0x80e8d0000, sg_len = 3342336/0x330000 sg_phys= 0x80e8d0000
[ 66.712275] [mca_info] mca_get_sg_from_fd(), line 407 sgt[1] sg_addr = 0x80e400000, sg_len = 196608/0x30000 sg_phys= 0x80e400000
[ 66.713386] [mca_info] DMABUF_FD 1066
[ 66.713396] [mca_info] mca_get_sg_from_fd(), line 377 got dmabuf ffffffc7d5e72600 nvmap_dmabuf
[ 66.713433] [mca_info] mca_get_sg_from_fd(), line 397 got sg->nents 1
[ 66.713439] [mca_info] mca_get_sg_from_fd(), line 399 got sg->orig_nents 1 dma_buf->size=3538944/360000
[ 66.713444] [mca_info] mca_get_sg_from_fd(), line 407 sgt[0] sg_addr = 0x80e430000, sg_len = 3538944/0x360000 sg_phys= 0x80e430000
[ 66.715904] [mca_info] DMABUF_FD 1067
[ 66.715912] [mca_info] mca_get_sg_from_fd(), line 377 got dmabuf ffffffc7d5e71600 nvmap_dmabuf
[ 66.715955] [mca_info] mca_get_sg_from_fd(), line 397 got sg->nents 2
[ 66.715960] [mca_info] mca_get_sg_from_fd(), line 399 got sg->orig_nents 2 dma_buf->size=3538944/360000
[ 66.715964] [mca_info] mca_get_sg_from_fd(), line 407 sgt[0] sg_addr = 0x80e790000, sg_len = 458752/0x70000 sg_phys= 0x80e790000
[ 66.715968] [mca_info] mca_get_sg_from_fd(), line 407 sgt[1] sg_addr = 0x80e000000, sg_len = 3080192/0x2f0000 sg_phys= 0x80e000000
[ 66.717807] [mca_info] DMABUF_FD 1068
[ 66.717816] [mca_info] mca_get_sg_from_fd(), line 377 got dmabuf ffffffc7d5e70a00 nvmap_dmabuf
[ 66.717849] [mca_info] mca_get_sg_from_fd(), line 397 got sg->nents 2
[ 66.717856] [mca_info] mca_get_sg_from_fd(), line 399 got sg->orig_nents 2 dma_buf->size=3538944/360000
[ 66.717861] [mca_info] mca_get_sg_from_fd(), line 407 sgt[0] sg_addr = 0x80e2f0000, sg_len = 1114112/0x110000 sg_phys= 0x80e2f0000
[ 66.717866] [mca_info] mca_get_sg_from_fd(), line 407 sgt[1] sg_addr = 0x80dc00000, sg_len = 2424832/0x250000 sg_phys= 0x80dc00000
[ 66.719633] [mca_info] DMABUF_FD 1069
[ 66.719641] [mca_info] mca_get_sg_from_fd(), line 377 got dmabuf ffffffc7d5e70c00 nvmap_dmabuf
[ 66.719695] [mca_info] mca_get_sg_from_fd(), line 397 got sg->nents 2
[ 66.719700] [mca_info] mca_get_sg_from_fd(), line 399 got sg->orig_nents 2 dma_buf->size=3538944/360000
[ 66.719704] [mca_info] mca_get_sg_from_fd(), line 407 sgt[0] sg_addr = 0x80de50000, sg_len = 1769472/0x1b0000 sg_phys= 0x80de50000
[ 66.719708] [mca_info] mca_get_sg_from_fd(), line 407 sgt[1] sg_addr = 0x80d800000, sg_len = 1769472/0x1b0000 sg_phys= 0x80d800000
[ 66.721463] [mca_info] DMABUF_FD 1070

Q1: Did the output get the y or uv plane data?
Q2: how to get physical address from the dmabuf_fd?
we got the sg_table by dma_buf_map_attachment, but how to explain for_each_sg(),
the sg_phys(sg) is what we need?
Q3: there is one scatterlist or two scatterlists, why?

besides, we do not need IOMMU for the DMA transfer.

thanks.

Hi,
Would need your help to share your use-case for information. We now only have NvBuffer APIs in userspace. Not sure why you need to access the buffers in kernel.

ok, let’s brief the scenario.

There are more than 1 Jetson AGX Xavier as the PCIe EPs, let them as EP1, EP2, …
when one of EPs finish decode, we need DMA the decoded video stream to the other EP over the PCIe.
for example, EP1 decode, then send to EP2, EP2 continue to process decoded video stream.

so we need the physical address for the DMA transfer.
But the NvBuffer API only expose the dmabuf_fd rather than physical address.

out test case is the jetson_multimedia_api/samples/000_video_decode

more logs are available, thanks.

mca_get_sg_from_fd.dmesg (201.7 KB)
video_decode.log (70.5 KB)