dwImageNvMedia dwImageStreamer dwImageCUDA

Hi

I want to get dwImageCUDA as input to TensorRT Model, and i refer to ./sample_camera_gmsl.But encounter some problem.

the code below is runNvMedia_pipeline() in ./sample_camera_gmsl

void runNvMedia_pipeline(WindowBase *window, dwRendererHandle_t renderer, dwContextHandle_t sdk,
                         dwSensorHandle_t cameraSensor, int cameraWidth, int cameraHeight)
{
    // RGBA image pool for conversion from YUV camera output
    NvMediaDevice *nvmedia;
    dwContext_getNvMediaDevice(&nvmedia, sdk);

    std::vector<dwImageNvMedia *> rgbaImagePool;
    for (int i = 0; i < 4; ++i) {
        NVM_SURF_FMT_DEFINE_ATTR(surfFormatAttrs);
        NVM_SURF_FMT_SET_ATTR_RGBA(surfFormatAttrs, RGBA, UINT, 8, PL);
        NvMediaSurfaceType type = NvMediaSurfaceFormatGetType(surfFormatAttrs,
                                                        NVM_SURF_FMT_ATTR_MAX);

        NvMediaSurfAllocAttr surfAllocAttrs[8];
        uint32_t numSurfAllocAttrs = 0;
        surfAllocAttrs[0].type = NVM_SURF_ATTR_WIDTH;
        surfAllocAttrs[0].value = cameraWidth;
        surfAllocAttrs[1].type = NVM_SURF_ATTR_HEIGHT;
        surfAllocAttrs[1].value = cameraHeight;
        surfAllocAttrs[2].type = NVM_SURF_ATTR_CPU_ACCESS;
        surfAllocAttrs[2].value = NVM_SURF_ATTR_CPU_ACCESS_CACHED;
        surfAllocAttrs[3].type = NVM_SURF_ATTR_ALLOC_TYPE;
        surfAllocAttrs[3].value = NVM_SURF_ATTR_ALLOC_ISOCHRONOUS;
        numSurfAllocAttrs = 4;

        dwImageNvMedia *rgbaImage = new dwImageNvMedia();
        NvMediaImage *rgbaNvMediaImage;

        rgbaNvMediaImage = NvMediaImageCreateNew(nvmedia, type, surfAllocAttrs,
                                              numSurfAllocAttrs, 0);

        dwImageNvMedia_setFromImage(rgbaImage, rgbaNvMediaImage);
        rgbaImagePool.push_back(rgbaImage);
    }

    // format converter
    dwImageProperties cameraImageProperties;
    dwSensorCamera_getImageProperties(&cameraImageProperties, DW_CAMERA_PROCESSED_IMAGE, cameraSensor);
    dwImageProperties displayImageProperties = cameraImageProperties;
    displayImageProperties.pxlFormat = DW_IMAGE_RGBA;
    displayImageProperties.planeCount = 1;

    dwImageFormatConverterHandle_t yuv2rgba = DW_NULL_HANDLE;
    dwStatus status                         = dwImageFormatConverter_initialize(&yuv2rgba,
                                                        cameraImageProperties.type,
                                                        sdk);
    if (status != DW_SUCCESS) {
        std::cerr << "Cannot initialize pixel format converter" << std::endl;
        exit(1);
    }

    // image API translator
    dwImageStreamerHandle_t nvm2gl = DW_NULL_HANDLE;
    dwImageStreamer_initialize(&nvm2gl, &displayImageProperties, DW_IMAGE_GL, sdk);

dwImageStreamerHandle_t nvm2cuda = DW_NULL_HANDLE;
    status = dwImageStreamer_initialize(&nvm2cuda, &cameraImageProperties, DW_IMAGE_CUDA, sdk);
    if (status != DW_SUCCESS) {
        std::cerr << "\n ERROR dwImageStreamer_initialize: " <<dwGetStatusName(status)<<std::endl;
    }

    // Start Sensor and Processing
    dwCameraProperties cameraProperties;
    dwSensorCamera_getSensorProperties(&cameraProperties, cameraSensor);

    g_run = dwSensor_start(cameraSensor) == DW_SUCCESS;

    // Message msg;
    while (g_run && !window->shouldClose()) {
        dwCameraFrameHandle_t frameHandle;
        dwImageNvMedia *frame = nullptr;
        uint32_t camera = 0u;
        dwStatus status = dwSensorCamera_readFrame(&frameHandle, camera, 1000000, cameraSensor);
        if (status != DW_SUCCESS) {
            std::cout << "\n ERROR readFrame: " << dwGetStatusName(status) << std::endl;
            continue;
        }

        if( cameraProperties.outputTypes & DW_CAMERA_PROCESSED_IMAGE) {
            status = dwSensorCamera_getImageNvMedia(&frame, DW_CAMERA_PROCESSED_IMAGE, frameHandle);
            if( status != DW_SUCCESS ) {
                std::cout << "\n ERROR getImageNvMedia " << dwGetStatusName(status) << std::endl;
            }
        }

        // get embedded lines
        if( cameraProperties.outputTypes & DW_CAMERA_DATALINES) {
            const dwImageDataLines* dataLines = nullptr;
            status = dwSensorCamera_getDataLines(&dataLines, frameHandle);
            // parse the data
            if( status == DW_SUCCESS ) {
                status = dwSensorCamera_parseDataNvMedia(&sensorData, dataLines, cameraSensor);
                if( status == DW_SUCCESS ) {
#if (VIBRANTE_PDK_MAJOR == 5)
#else
                    std::cout << "Exposure Time (s): " << sensorData.exposureMidpointTime << "\r";// std::endl;
#endif
                } else {
                    std::cout << "Could not parse embedded data: " << dwGetStatusName(status) << "\r"; //std::endl;
                }
            } else {
                std::cout << "Error getting datalines: " << dwGetStatusName(status) << "\r"; //std::endl;
            }
        }

        // Convert from YUV to RGBA
        if (frame && rgbaImagePool.size() > 0) {
            dwImageNvMedia *rgbaImage = rgbaImagePool.back();
            rgbaImagePool.pop_back();

            //std::cout << " CONVERSION YUV->RGBA\n";
            status = dwImageFormatConverter_copyConvertNvMedia(rgbaImage, frame, yuv2rgba);
            if (status != DW_SUCCESS) {
                std::cout << "\n ERROR copyConvert: " << dwGetStatusName(status) << std::endl;
                rgbaImagePool.push_back(rgbaImage);

            } else {
                // Send via ImageStreamer to get GL image back
                status = dwImageStreamer_postNvMedia(rgbaImage, nvm2gl);
                if (status != DW_SUCCESS) {
                    std::cout << "\n ERROR postNvMedia: " << dwGetStatusName(status) << std::endl;
                } else {
                    dwImageGL *frameGL = nullptr;
                    status             = dwImageStreamer_receiveGL(&frameGL, 60000, nvm2gl);

                    if (status == DW_SUCCESS && frameGL) {
                        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

                        // render received texture
                        dwRenderer_renderTexture(frameGL->tex, frameGL->target, renderer);

                        dwImageStreamer_returnReceivedGL(frameGL, nvm2gl);
                    }
                }
                dwImageCUDA *cudaImage = nullptr;
                status                 = dwImageStreamer_receiveCUDA( &cudaImage, 600000, nvm2cuda);
                if (status == DW_SUCCESS && cudaImage) {
                    dwImageStreamer_returnReceivedCUDA(cudaImage, nvm2cuda);
                    
                    ///////////////   insert cudaImage to TensorRT Model    ///////////////  

                }
                else{
                    std::cout << "\n ERROR dwImageStreamer_receiveCUDA: " << dwGetStatusName(status) << std::endl;
                    dwImageStreamer_returnReceivedCUDA(cudaImage, nvm2cuda);
                }
                // any image returned back, we put back into the pool
                dwImageNvMedia *retimg = nullptr;
                dwImageStreamer_waitPostedNvMedia(&retimg, 33000, nvm2gl);

                if (retimg)
                    rgbaImagePool.push_back(retimg);
            }
        }

        dwSensorCamera_returnFrame(&frameHandle);

        if (window)
            window->swapBuffers();
    }
    dwSensor_stop(cameraSensor);
    dwImageStreamer_release(&nvm2gl);

    for (auto frame : rgbaImagePool) {
        NvMediaImageDestroy(frame->img);
        delete frame;
    }

    dwImageFormatConverter_release(&yuv2rgba);
}

I try to get dwImageCUDA by insert some code at line 58~62 and line 137~148 . But i got error as below. Thank u ~~~~

ERROR dwImageStreamer_reveiveCUDA: DW_TIME_OUT
ImageStreamer: cannot receive, EGLstream busy

when i add

status = dwImageStreamer_postNvMedia(rgbaImage, nvm2cuda);

before dwImageStreamer_receiveCUDA(), it always show error type incompatible.

Dear allenchen8210,
I will check reproducing the same error on my side and get back to you.

Dear allenchen8210,
In my code inspection, I can see that you are sending frames only on nvm2gl ImageStreamer and trying to receive frames on nvm2cuda ImageStreamer which is actually empty.
As I understand, you want to send same NvMedia frame to both GL and CUDA. If so, Please check sample_object_detector code on how it is making use of ImageStreamer calls for this purpose.
It looks like you are using DW 0.6. If possible please upgrade to latest PDK version(https://developer.nvidia.com/nvidia-drive-downloads).

Hi SivaRamaKrishna

Thank u for your reply.

What i want is to get dwImageCUDA and insert it into TensorRT model.

I edit code at line 56~63 and line 141~158.

void runNvMedia_pipeline(WindowBase *window, dwRendererHandle_t renderer, dwContextHandle_t sdk,
                         dwSensorHandle_t cameraSensor, int cameraWidth, int cameraHeight)
{
    // RGBA image pool for conversion from YUV camera output
    NvMediaDevice *nvmedia;
    dwContext_getNvMediaDevice(&nvmedia, sdk);

    std::vector<dwImageNvMedia *> rgbaImagePool;
    for (int i = 0; i < 4; ++i) {
        NVM_SURF_FMT_DEFINE_ATTR(surfFormatAttrs);
        NVM_SURF_FMT_SET_ATTR_RGBA(surfFormatAttrs, RGBA, UINT, 8, PL);
        NvMediaSurfaceType type = NvMediaSurfaceFormatGetType(surfFormatAttrs,
                                                        NVM_SURF_FMT_ATTR_MAX);

        NvMediaSurfAllocAttr surfAllocAttrs[8];
        uint32_t numSurfAllocAttrs = 0;
        surfAllocAttrs[0].type = NVM_SURF_ATTR_WIDTH;
        surfAllocAttrs[0].value = cameraWidth;
        surfAllocAttrs[1].type = NVM_SURF_ATTR_HEIGHT;
        surfAllocAttrs[1].value = cameraHeight;
        surfAllocAttrs[2].type = NVM_SURF_ATTR_CPU_ACCESS;
        surfAllocAttrs[2].value = NVM_SURF_ATTR_CPU_ACCESS_CACHED;
        surfAllocAttrs[3].type = NVM_SURF_ATTR_ALLOC_TYPE;
        surfAllocAttrs[3].value = NVM_SURF_ATTR_ALLOC_ISOCHRONOUS;
        numSurfAllocAttrs = 4;

        dwImageNvMedia *rgbaImage = new dwImageNvMedia();
        NvMediaImage *rgbaNvMediaImage;

        rgbaNvMediaImage = NvMediaImageCreateNew(nvmedia, type, surfAllocAttrs,
                                              numSurfAllocAttrs, 0);

        dwImageNvMedia_setFromImage(rgbaImage, rgbaNvMediaImage);
        rgbaImagePool.push_back(rgbaImage);
    }

    // format converter
    dwImageProperties cameraImageProperties;
    dwSensorCamera_getImageProperties(&cameraImageProperties, DW_CAMERA_PROCESSED_IMAGE, cameraSensor);
    dwImageProperties displayImageProperties = cameraImageProperties;
    displayImageProperties.pxlFormat = DW_IMAGE_RGBA;
    displayImageProperties.planeCount = 1;
    dwImageFormatConverterHandle_t yuv2rgba = DW_NULL_HANDLE;
    dwStatus status                         = dwImageFormatConverter_initialize(&yuv2rgba,
                                                        cameraImageProperties.type,
                                                        sdk);
    if (status != DW_SUCCESS) {
        std::cerr << "Cannot initialize pixel format converter" << std::endl;
        exit(1);
    }

    // image API translator
    dwImageStreamerHandle_t nvm2gl = DW_NULL_HANDLE;
    dwImageStreamer_initialize(&nvm2gl, &displayImageProperties, DW_IMAGE_GL, sdk);

    dwImageStreamerHandle_t nvm2cuda = DW_NULL_HANDLE;
    cameraImageProperties.type = DW_IMAGE_NVMEDIA;

    status = dwImageStreamer_initialize(&nvm2cuda, &cameraImageProperties, DW_IMAGE_CUDA, sdk);
    if (status != DW_SUCCESS) {
        std::cerr << "\n ERROR dwImageStreamer_initialize: " <<dwGetStatusName(status)<<std::endl;
        exit(1);
    }

  
    // Start Sensor and Processing
    dwCameraProperties cameraProperties;
    dwSensorCamera_getSensorProperties(&cameraProperties, cameraSensor);
    g_run = dwSensor_start(cameraSensor) == DW_SUCCESS;

    // Message msg;
    while (g_run && !window->shouldClose()) {
        dwCameraFrameHandle_t frameHandle;
        dwImageNvMedia *frame = nullptr;
        uint32_t camera = 0u;
        dwStatus status = dwSensorCamera_readFrame(&frameHandle, camera, 1000000, cameraSensor);
        if (status != DW_SUCCESS) {
            std::cout << "\n ERROR readFrame: " << dwGetStatusName(status) << std::endl;
            continue;
        }

        if( cameraProperties.outputTypes & DW_CAMERA_PROCESSED_IMAGE) {
            status = dwSensorCamera_getImageNvMedia(&frame, DW_CAMERA_PROCESSED_IMAGE, frameHandle);
            if( status != DW_SUCCESS ) {
                std::cout << "\n ERROR getImageNvMedia " << dwGetStatusName(status) << std::endl;
            }
        }

        // get embedded lines
        if( cameraProperties.outputTypes & DW_CAMERA_DATALINES) {
            const dwImageDataLines* dataLines = nullptr;
            status = dwSensorCamera_getDataLines(&dataLines, frameHandle);
            // parse the data
            if( status == DW_SUCCESS ) {
                status = dwSensorCamera_parseDataNvMedia(&sensorData, dataLines, cameraSensor);
                if( status == DW_SUCCESS ) {
#if (VIBRANTE_PDK_MAJOR == 5)
#else
                    std::cout << "Exposure Time (s): " << sensorData.exposureMidpointTime << "\r";// std::endl;
#endif
                } else {
                    std::cout << "Could not parse embedded data: " << dwGetStatusName(status) << "\r"; //std::endl;
                }
            } else {
                std::cout << "Error getting datalines: " << dwGetStatusName(status) << "\r"; //std::endl;
            }
        }

        // Convert from YUV to RGBA
        if (frame && rgbaImagePool.size() > 0) {
            dwImageNvMedia *rgbaImage = rgbaImagePool.back();
            rgbaImagePool.pop_back();

            //std::cout << " CONVERSION YUV->RGBA\n";
            status = dwImageFormatConverter_copyConvertNvMedia(rgbaImage, frame, yuv2rgba);
            if (status != DW_SUCCESS) {
                std::cout << "\n ERROR copyConvert: " << dwGetStatusName(status) << std::endl;
                rgbaImagePool.push_back(rgbaImage);

            } else {
                // Send via ImageStreamer to get GL image back
                status = dwImageStreamer_postNvMedia(rgbaImage, nvm2gl);
                if (status != DW_SUCCESS) {
                    std::cout << "\n ERROR postNvMedia: " << dwGetStatusName(status) << std::endl;
                } else {
                    dwImageGL *frameGL = nullptr;
                    status             = dwImageStreamer_receiveGL(&frameGL, 60000, nvm2gl);

                    if (status == DW_SUCCESS && frameGL) {
                        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

                        // render received texture
                        dwRenderer_renderTexture(frameGL->tex, frameGL->target, renderer);

                        dwImageStreamer_returnReceivedGL(frameGL, nvm2gl);
                    }
                }

                ///////////////////////////////////////////////////////////////////////////////////////
                
                status = dwImageStreamer_postNvMedia( frame, nvm2cuda);
                if (status != DW_SUCCESS){
                    std::cout << "\n ERROR postNvMedia: " <<dwGetStatusName(status) <<std::endl;
                }
                else{
                    std::cout << "\n SUCCESS postNvMedia" << std::endl;
                    dwImageCUDA *cudaImage = nullptr;
                    status                 = dwImageStreamer_receiveCUDA( &cudaImage, 60000, nvm2cuda);
                    if (status == DW_SUCCESS && cudaImage) {

                        std::cout << "\n SUCCESS dwImageStreamer_receiveCUDA: " << std::endl;
                        dwImageStreamer_returnReceivedCUDA(cudaImage, nvm2cuda);
                    }
                    else{
                        std::cout << "\n ERROR dwImageStreamer_receiveCUDA: " << dwGetStatusName(status) << std::endl;
                        
                    }
                }             
                // any image returned back, we put back into the pool
                dwImageNvMedia *retimg = nullptr;
                dwImageStreamer_waitPostedNvMedia(&retimg, 33000, nvm2gl);

                if (retimg)
                    rgbaImagePool.push_back(retimg);
            }
        }

        dwSensorCamera_returnFrame(&frameHandle);

        if (window)
            window->swapBuffers();
    }
    dwSensor_stop(cameraSensor);
    dwImageStreamer_release(&nvm2gl);
    //////////////////////////////////////////////////////////////////////////
    dwImageStreamer_release( &nvm2cuda);
    //////////////////////////////////////////////////////////////////////////
    for (auto frame : rgbaImagePool) {
        NvMediaImageDestroy(frame->img);
        delete frame;
    }

    dwImageFormatConverter_release(&yuv2rgba);
}

Finally i success get dwImageCUDA, but just few frames, not all. I encountered error as below.

SUCCESS postNvMedia

 SUCCESS dwImageStreamer_receiveCUDA: 

 SUCCESS postNvMedia

 SUCCESS dwImageStreamer_receiveCUDA: 

 SUCCESS postNvMedia

 SUCCESS dwImageStreamer_receiveCUDA: 

 SUCCESS postNvMedia

 SUCCESS dwImageStreamer_receiveCUDA: 
ImageStreamer: cannot post, no available slot

 ERROR postNvMedia: DW_BUSY_WAITING
ImageStreamer: cannot post, no available slot

 ERROR postNvMedia: DW_BUSY_WAITING
ImageStreamer: cannot post, no available slot

 ERROR postNvMedia: DW_BUSY_WAITING
ImageStreamer: cannot post, no available slot

Really thank u for your help…

Dear allenchen8210,
This error indicate your producer speed is more than consumer speed and ImageStreamer queue is full. Did you get a chance to look at object detector sample on how it handles two image streamers with out these issues.

The question has been settled.