EGLstream DMA Consumer solutions

Hi guys,
There are two solution for DMA Consumer in EGLstream at Tegra devices

  1. using nveglstreamsrc plugin which is close source and I’m not sure if it is zero copy or it has HW copy and other problem is it needs Gstreamer.

using Argus Interfaces

 Argus::Status status;
    Argus::UniqueObj<EGLStream::FrameConsumer> consumer(EGLStream::FrameConsumer::create(eglDisplay ,egl_str,&status));
    iFrameConsumer = Argus::interface_cast<EGLStream::IFrameConsumer>(consumer);
    Argus::UniqueObj< EGLStream::Frame > frameLeft{ iFrameConsumer->acquireFrame() };
    EGLStream::IFrame * iFrameLeft{ interface_cast< EGLStream::IFrame >( frameLeft ) }; 
    
    EGLStream::NV::IImageNativeBuffer * iNativeBufferLeft{ interface_cast< EGLStream::NV::IImageNativeBuffer >( iFrameLeft->getImage() ) };
    static int fdLeft =-1;
    if(fdLeft==-1)
    {
        fdLeft = iNativeBufferLeft->createNvBuffer( Argus::Size2D<uint32_t>(1920,1080), NvBufferColorFormat_ABGR32, NvBufferLayout_Pitch );
    }
    else
    {
        iNativeBufferLeft->copyToNvBuffer(fdLeft);
    }

problem about this code it clearly say “copyToNvBuffer” it means it is not zero copy and other problem is I don’t use ARGUS as capture source I use nvv4l2camerasrc. In samples in “/usr/src/jetson_multimedia_api/argus/samples” all samples use ARGUS as capture source and use static FrameConsumer* create(Argus::OutputStream* outputStream, Argus::Status* status = NULL);
as constuctor.
Is there any example that use DMA buffer as source like nvv4l2camerasrc or nvv4l2decoder.
my final goals is produce GL buffer as GL producer in EGL stream and get DMA buffer of that through EGLstream. Right know my codes works through this Gstreamer pipeline

nveglstreamsrc name=egl_src ! video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080, framerate=30/1 ! appsink name=testsink sync=false

so it uses Gstreamer I need low level API and zero copy.
Thanks so much

Hi,
createNvBuffer() and copyToNvBuffer() are done on hardware engine(VIC engine) so it does not take CPU usage. It can be treated as zero copy. You can use NvBuffer APIs to map the buffer to EGLImage or CPU pointer.

Do you use Argus camera source in your use-case? Would like to get more detail about your use-case so that we can check and suggest next.

Thanks for your attention
I need an example of EGLstream that use secound constructor

 Argus::UniqueObj<EGLStream::FrameConsumer> consumer(EGLStream::FrameConsumer::create(eglDisplay ,egl_str,&status));

not first one

   static FrameConsumer* create(Argus::OutputStream* outputStream,
                                 Argus::Status* status = NULL);

I think all examples in “usr/src/jetson_multimedia_api” are base on Argus, please a complete example with producer and consumer with secound constructor.
Actually I wrote it myself, It is my source code

  int fdLeft =-1;
    EGLStream::IFrameConsumer * iFrameConsumer =nullptr;
  Argus::Status status;
    Argus::UniqueObj<EGLStream::FrameConsumer> consumer(EGLStream::FrameConsumer::create(eglDisplay ,egl_str,&status));
    iFrameConsumer = Argus::interface_cast<EGLStream::IFrameConsumer>(consumer);
   Argus::Status status;
    Argus::UniqueObj< EGLStream::Frame > frameLeft{ iFrameConsumer->acquireFrame(Argus::TIMEOUT_INFINITE ,& status) };
    EGLStream::IFrame * iFrameLeft{ Argus::interface_cast< EGLStream::IFrame >( frameLeft ) };

    EGLStream::NV::IImageNativeBuffer * iNativeBufferLeft{ Argus::interface_cast< EGLStream::NV::IImageNativeBuffer >( iFrameLeft->getImage() ) };
      fdLeft =-1;
    if(fdLeft==-1)
    {
        fdLeft = iNativeBufferLeft->createNvBuffer( Argus::Size2D<uint32_t>(1920,1080), NvBufferColorFormat_ABGR32, NvBufferLayout_Pitch );

            if(this->callBack != nullptr)
            {
                callBack(this->m_pBackward, fdLeft);
            }

    }
    else
    {
        iNativeBufferLeft->copyToNvBuffer(fdLeft);
      
    }

and producer side is

 GstElement* appsrcVideo = nullptr;
  gst_init (NULL, NULL);

    GMainLoop *main_loop;
    main_loop = g_main_loop_new (NULL, FALSE);
    GstPipeline *gst_pipeline = nullptr;
    int ret = 0;
    GError *error = nullptr;
    char parseLaunch[1000]={'\0',};





    gst_pipeline  = (GstPipeline*) gst_parse_launch("nvv4l2camerasrc name=testsrc ! video/x-raw(memory:NVMM), format=RGBA, width=1920, height=1080  ! nvvidconv ! video/x-raw(memory:NVMM), format=I420 ! nvvideosink name=egl_src ", &error);


    if (gst_pipeline == nullptr) {
        g_print( "Failed to parse launch: %s\n", error->message);

    }
    if(error) g_error_free(error);

    appsrcVideo = gst_bin_get_by_name (GST_BIN(gst_pipeline), "testsrc");
   


    GstElement *videoSource = gst_bin_get_by_name(GST_BIN(gst_pipeline), "egl_src");
    if(!GST_IS_ELEMENT(videoSource)) {
        g_print( "Failed to get source from pipeline \n");

    }
    g_object_set(G_OBJECT(videoSource), "display", eglDisplay, NULL);
    g_object_set(G_OBJECT(videoSource), "stream", egl_str, NULL);






    gst_element_set_state((GstElement*)gst_pipeline, GST_STATE_PLAYING);

    //    sleep(10);
    g_main_loop_run (main_loop);

    gst_element_set_state((GstElement*)gst_pipeline, GST_STATE_NULL);
    gst_object_unref(GST_OBJECT(gst_pipeline));
    g_main_loop_unref(main_loop);



    g_print("last line");

and errors are here

(Argus) Error InvalidState:  (propagating from src/eglstream/FrameConsumerImpl.cpp, function streamEventThread(), line 135)
(Argus) Error BadParameter:  (propagating from src/eglstream/FrameConsumerImpl.cpp, function acquireFrame(), line 247)
(Argus) Error InvalidState:  (propagating from src/eglstream/FrameConsumerImpl.cpp, function streamEventThreadStatic(), line 177)

  1. can all producers like GL, CUDA , or DMA producer (nvvideosink) connect to dma consumer ?
  Argus::Status status;
    Argus::UniqueObj<EGLStream::FrameConsumer> consumer(EGLStream::FrameConsumer::create(eglDisplay ,egl_str,&status));

if yes an example please?
Thanks so much.

Hi,
Do you use Bayer camera sensor as the source? Would like to confirm this.

Hi
Thanks for your attention,
we don’t use Bayer sensor. we capture through v4l2src and nvv4l2camerasrc.
because of that

  static FrameConsumer* create(Argus::OutputStream* outputStream,
                                 Argus::Status* status = NULL);

constructor is not appropriate for me and I must use secound one

 Argus::Status status;
    Argus::UniqueObj<EGLStream::FrameConsumer> consumer(EGLStream::FrameConsumer::create(eglDisplay ,egl_str,&status));

?
Does source code of “nveglstreamsrc” contain Argus code ?
why there is not clear zero copy low level solution for DMA producer and DMA consumer in tegra tx2 in Eglstream ?

In Nvidia document it is written
" EGLStream Producer

The EGLStream producer is the entity that posts EGL image frames into the EGLStream. In the NvMedia domain there are two types of producers:

• Video producer: Posts NvMedia video surfaces as EGL image frames. Video surfaces can be generated by an NvMedia video decoder, by NvMedia video capture, etc. The supported video surface formats include:

• NvMediaSurfaceType_R8G8B8A8

• NvMediaSurfaceType_Video_420

• Image producer: Posts NvMedia image surfaces as EGL image frames. Image surfaces can be generated by an NvMedia image JPEG decoder, by NvMedia image capture, etc. The supported image surface formats include:

• NvMediaSurfaceType_Image_RGBA

• NvMediaSurfaceType_Image_YUV_420

Other types of producers outside the NvMedia domain are:

• CUDA producer: Posts a CUDA array or CUDA pointer as EGL image frames to the EGLStream.

• GL producer: Posts graphic surfaces as EGL image frames to the EGLStream.

EGLStream Consumer

The EGLStream consumer is the entity that retrieves or acquires EGL image frames from the EGLStream. In the NvMedia domain, there are two types of consumers:

• Video consumer: Acquires EGL image frames and fills the frame information in NvMedia video surfaces. Video surfaces acquired can then be consumed by NvMedia video encoder, or NvMedia video display, etc.

• Image consumer: Acquires EGL image frames and fills the frame information in NvMedia image surfaces (NvMediaImage). Image surfaces acquired can then be consumed by NvMedia image encoder, or NvMedia image display, etc.

Other types of consumers outside the NvMedia domain are:

• CUDA consumer: Retrieves EGL image frames and fills the frame information as a CUDA array or CUDA pointer. The CUDA frames can then be processed in the CUDA domain.

• GL consumer: Retrieves EGL image frames that can then be used in graphic rendering.

• EGL output consumer (egldevice window interface only): Retrieves EGL image frames and renders them directly to EGL output. This consumer is valid when EGL output is used on the egldevice window interface.

"

so we expect that easily and zero copy use Eglstream for GL producer, Cuda producer and DMA producer as well as CUDA consumer, GL consumer, DMA consumer and EGL out put (drm)conusmer.
is all type supported in Tegra? for example
GL producer DMA consumer I did it through “nveglstreamsrc” it works but I don’t want relay on Gstreamer. Argus solution didn’t work and errors occurred.
How about GL producer and CUDA consumer , I did it I acquire frame in CUDA but pitch value of frame is zero and frame is not valid.
base on https://forums.developer.nvidia.com/t/how-to-acquire-frames-out-of-eglstreamkhr-and-map-to-nvbuffer-for-use-in-nvvideoencoder/112794 this post they want use CUDA producer and CUDA consumer in EGLstream and after that convert to DMA buffer for HW encoding.
solution that is suggested using DMA -CUDA interoperablity and CUDA-copy consumer frame to DMA frame.
In my use-case I need GL producer -CUDA consumer doesn’t work for me.
Thanks a lot

Hi,
A possible solution is to use NvBuffer APIs to share NvBuffer between processes. Pleas refer to the method in
How to share the buffer in process context? - #11 by DaneLLL

For capturing frame data into NvBuffer, you can try 12_camera_v4l2_cuda sample.

Thanks for your attention,
your suggestion didn’t work for me, As I previously mentioned I want a method to convert GL buffer to DMA buffer zero copy. Beacause EGLstream supports commiunication between interfaces like DMA,GL,CUDA,EGLOutput, I wanted a EGLstream example that producer is GL and Consumer is DMA.
Two solutions were suggested by me, first one base on “nveglstreamsrc” and secound one Argus Interface.
nveglstreamsrc is not low level and depends on Gstreamer.
Argus Interface didn’t work for me. error occurred.
so both solution have problems.

Finally I found solution , base on GL - CUDA Interoperablity

 EGLSurface eglSurface = EGL_NO_SURFACE;
    EGLContext eglContext = EGL_NO_CONTEXT;
    EGLConfig egl_cfg;
    EGLDisplay eglDisplay = EGL_NO_DISPLAY;
    int xsurfsize = 1920, ysurfsize = 1080;
  cudaArray* srcArray;
    int dma_CUDA_GL;
  CUgraphicsResource pResource = 0;
    EGLImageKHR egl_image = 0;
    void* cudaPtr = 0;
cudaGraphicsResource *cudapbo;
    GLuint texture;
    GLuint  framebuffer;
 EGLint num_config;

    eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
  

    eglInitialize(eglDisplay, nullptr, nullptr);

    static const EGLint configAttribs[] = {
        EGL_SURFACE_TYPE, EGL_PBUFFER_BIT,
        EGL_BLUE_SIZE, 8,
        EGL_GREEN_SIZE, 8,
        EGL_RED_SIZE, 8,
        EGL_DEPTH_SIZE, 8,
        EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT,
        EGL_NONE
    };
    eglChooseConfig(eglDisplay, configAttribs, &config, 1, &num_config);
  
    eglBindAPI(EGL_OPENGL_API);

    context = eglCreateContext(eglDisplay, config, EGL_NO_CONTEXT, NULL);
  
    const EGLint srfPbufferAttr[] = {
        EGL_WIDTH, WIDTH,
        EGL_HEIGHT, HEIGHT,
        EGL_NONE
    };

    eglSurface = eglCreatePbufferSurface(eglDisplay, config, srfPbufferAttr);

  NvBufferCreateParams outPut_params  = {0};



    outPut_params.payloadType = NvBufferPayload_SurfArray;
    outPut_params.layout = NvBufferLayout_Pitch;
    outPut_params.width = xsurfsize  ;
    outPut_params.height = ysurfsize;
    outPut_params.colorFormat = NvBufferColorFormat_ABGR32;
    outPut_params.nvbuf_tag = NvBufferTag_NONE;
    NvBufferCreateEx(&dma_CUDA_GL, &outPut_params);


 int ret;
    CUresult status;
    CUeglFrame eglFrame;

    //    // Create EGLImage from dmabuf fd
    egl_image = NvEGLImageFromFd(eglDisplay,  dma_CUDA_GL);
    if (egl_image == NULL)
    {
        printf("\n    if (egl_image == NULL) \n");
    }
    if(egl_image != NULL)
    {
        cudaFree(0);
        ret = cuGraphicsEGLRegisterImage(& pResource,
                                         egl_image,
                                         CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
        if (ret != CUDA_SUCCESS) {
            printf ("\n Failed to register EGLImage in cuda : %d \n",ret);
        }
        else
            if (cuGraphicsResourceGetMappedEglFrame (&eglFrame,
                                                     pResource, 0, 0) != CUDA_SUCCESS) {
                printf ("\n Failed to get mapped EGL Frame\n");
            }
            else
            {
                status = cuCtxSynchronize();
                if (status != CUDA_SUCCESS) {
                    printf ("cuCtxSynchronize failed \n");
                }

                printf("\n capture width %d \n",eglFrame.width);
                printf("\n capture height %d \n",eglFrame.height);
                printf("\n capture depth %d \n",eglFrame.depth);
                printf("\n capture pitch %d \n",eglFrame.pitch);
                printf("\n capture planeCount %d \n",eglFrame.planeCount);
                printf("\n capture numChannels %d \n",eglFrame.numChannels);
                printf("\n capture frameType %d \n",eglFrame.frameType);
                printf("\n capture eglColorFormat %d \n",eglFrame.eglColorFormat);
                printf("\n capture cuFormat %d \n",eglFrame.cuFormat);
                //                sleep(5);

                cudaPtr = ( unsigned char*)(CUdeviceptr) eglFrame.frame.pPitch[0];
                status = cuCtxSynchronize();
                if (status != CUDA_SUCCESS) {
                    printf ("cuCtxSynchronize failed after memcpy \n");
                }
            }



    
    }






  glGenFramebuffers( 1, &framebuffer );
    glBindFramebuffer( GL_FRAMEBUFFER, framebuffer );

    glGenTextures( 1, &texture );
    glBindTexture( GL_TEXTURE_2D, texture );

    glFramebufferTexture2D( GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture, 0 );
    // set basic parameters
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);

    // Create texture data (4-component unsigned byte)
    glTexImage2D( GL_TEXTURE_2D, 0, GL_RGBA, xsurfsize, ysurfsize, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL );

    cudaError_t code;
    code = cudaGraphicsGLRegisterImage(&cudapbo, texture,GL_TEXTURE_2D, cudaGraphicsRegisterFlagsReadOnly);
    if (code != cudaSuccess)
    {
        fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code),__FILE__, __LINE__);
        if (abort)
        {
            exit(code);
        }
    }

    code = cudaGraphicsMapResources(1, &cudapbo, NULL);
    if (code != cudaSuccess)
    {
        fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code),__FILE__, __LINE__);
        if (abort)
        {
            exit(code);
        }
    }
    cudaArray* srcArray;
    //        gpuErrchk(cudaGraphicsResourceGetMappedPointer((void**)&a_d, NULL, cudapbo));
    code =  cudaGraphicsSubResourceGetMappedArray( &srcArray, cudapbo, 0, 0 ) ;
    if (code != cudaSuccess)
    {
        fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code),__FILE__, __LINE__);
        if (abort)
        {
            exit(code);
        }
    }

 

    eglMakeCurrent(display, eglSurface, eglSurface, context);

    while (true)
    {


         glEnable(GL_SCISSOR_TEST);
    glEnable(GL_BLEND);
    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
    glScissor(0, 0, 100, 100);
    glClearColor(0.9,0.9,0.6,1);
    glClear(GL_COLOR_BUFFER_BIT);
    // Remember to disable scissor test, or, perhaps reset the scissor rectangle:
    glDisable(GL_SCISSOR_TEST);
    glDisable(GL_BLEND);

        if (glGetError() != GL_NO_ERROR)
        {
            printf(" error  %s",glGetError());
            //return -1;
        }

        CUresult status;


        status = cuCtxSynchronize();
        if (status != CUDA_SUCCESS) {
            printf ("cuCtxSynchronize failed after memcpy \n");
        }

        cudaError_t error;
        //          error= cudaMemcpyFromArray(cudaPtr,srcArray,0,0,xsurfsize*4*ysurfsize,cudaMemcpyDeviceToDevice);
        error = cudaMemcpy2DFromArray(cudaPtr,xsurfsize*4,srcArray,0,0,xsurfsize * 4,ysurfsize,cudaMemcpyDeviceToDevice);

        if(error != cudaSuccess)
            printf( " \n erroer %d \n ",error);



        status = cuCtxSynchronize();
        if (status != CUDA_SUCCESS) {
            printf ("cuCtxSynchronize failed after memcpy \n");
        }
}



I created a DMA buffer and Mapped it to CUDA, after that through GL-CUDA Interoperablity I transfer GL-texture color to CUDA and copy GL-CUDA to my DMA-cuda. so I transfer GL-buffer to DMA.
problem here is one device to device copy

 cudaMemcpy2DFromArray(cudaPtr,xsurfsize*4,srcArray,0,0,xsurfsize * 4,ysurfsize,cudaMemcpyDeviceToDevice);

so Any better solution without need to copy DeviceToDevice.
Does “nveglstreamsrc” have a device-to-device or VIC copy ?
Can we omit this copy?
Thanks so much.

Hi,
It is not supported to convert GL buffer to DMA buffer. The solution is to create NvBuffer and then map to EGLImage. If you are not able to use the method, there may not be solution for you use-case. It would be great if you can consider the working method.

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.