Argus PIXEL_FMT_YCbCr_420_888 NvBufSurface To RGB cuda

[satchelwu]

1. Create camera, key code below:

            CaptureSession *captureSession = iCameraProvider->createCaptureSession(devices[i]);
            ICaptureSession *iCaptureSession = interface_cast<ICaptureSession>(captureSession);
            IEventProvider *iEventProvider = interface_cast<IEventProvider>(captureSession);
            if (!iCaptureSession || !iEventProvider)
            {
                return false;
            }
            /* Create the OutputStream */
            UniqueObj<OutputStreamSettings> streamSettings(
                iCaptureSession->createOutputStreamSettings(STREAM_TYPE_EGL));
            IOutputStreamSettings *iStreamSettings = interface_cast<IOutputStreamSettings>(streamSettings);

            IEGLOutputStreamSettings *iEglStreamSettings =
                interface_cast<IEGLOutputStreamSettings>(streamSettings);
            if (!iEglStreamSettings)
            {
                return false;
            }

            iEglStreamSettings->setPixelFormat(PIXEL_FMT_YCbCr_420_888);

2. In comsumer:

 bool ConsumerThread::detectKeyPointsFromRGB(NvBufSurface **bufSurfaceArray, int index)
    {
        NvBufSurface *bufSurface = bufSurfaceArray[index];
        if (bufSurface)
        {
            if (0 == NvBufSurfaceMapEglImage(bufSurface, 0) && bufSurface->surfaceList[0].mappedAddr.eglImage)
            {
                int width = bufSurface->surfaceList[0].width;
                int height = bufSurface->surfaceList[0].height;
                EGLImageKHR image = bufSurface->surfaceList[0].mappedAddr.eglImage;
                CUresult result = cuGraphicsEGLRegisterImage(&m_cudaFrameResources[index], image, 0);
                result = cuGraphicsResourceGetMappedEglFrame(&m_cuEglFrames[index], m_cudaFrameResources[index], 0, 0);
                result = cuCtxSynchronize();



                result = cuGraphicsUnregisterResource(m_cudaFrameResources[index]);
                int errorCode = NvBufSurfaceUnMapEglImage(bufSurface, 0);
                return true;
            }
        }
        return false;
    }

I tried nppiYUV420ToBGR_8u_P3C3R and custom cuda kernel to convert the color format，but not work. I wonder is there something i missed? Thanks

[carolyuu]

Hi,

For the camera basic functionality first needs to check the device and driver configuration.
You can reference to below program guide for the detailed information of device tree and driver implementation.
https://docs.nvidia.com/jetson/archives/r36.3/DeveloperGuide/SD/CameraDevelopment/SensorSoftwareDriverProgramming.html?highlight=programing#sensor-software-driver-programming

Please refer to Applications Using V4L2 IOCTL Directly by using V4L2 IOCTL to verify basic camera functionality.
https://docs.nvidia.com/jetson/archives/r36.3/DeveloperGuide/SD/CameraDevelopment/SensorSoftwareDriverProgramming.html?highlight=programing#to-run-a-v4l2-ctl-test

Once confirm the configure and still failed below link help to get log and some information and some tips for debug.
https://elinux.org/Jetson/l4t/Camera_BringUp#Steps_to_enable_more_debug_messages

Thanks!

[satchelwu]

Render works ok, I want to convert the yuv420 to rgb to do some image processing, but argus lib did not directly output the rgb format, so I have to convert the color format.

[JerryChang]

hello satchelwu,

you may run gst pipeline by calling videoconvert to convert the formats as BGRx.
for instance,
$ gst-launch-1.0 nvarguscamerasrc ! 'video/x-raw(memory:NVMM), width=(int)1920, height=(int)1080,format=(string)NV12, framerate=(fraction)30/1' ! nvvidconv ! 'video/x-raw, format=(string)BGRx' ! videoconvert ! xvimagesink

[satchelwu]

m_dmabufs[i] = iNativeBuffer->createNvBuffer(iEglOutputStreams[i]->getResolution(),
                                                                 NVBUF_COLOR_FORMAT_YUV420,
                                                                 NVBUF_LAYOUT_PITCH);

  if (0 == NvBufSurfaceMapEglImage(bufSurface, 0) && bufSurface->surfaceList[0].mappedAddr.eglImage)
  {
      int width = bufSurface->surfaceList[0].width;
      int height = bufSurface->surfaceList[0].height;
      EGLImageKHR image = bufSurface->surfaceList[0].mappedAddr.eglImage;
      CUresult result = cuGraphicsEGLRegisterImage(&m_cudaFrameResources[index], image, 0);
      result = cuGraphicsResourceGetMappedEglFrame(&m_cuEglFrames[index], m_cudaFrameResources[index], 0, 0);
      result = cuCtxSynchronize();
      NvBufSurfaceParams surfaceParams0 = bufSurface->surfaceList[0];
      const Npp8u *pSrc[3] = {(unsigned char *)(m_cuEglFrames[index].frame.pArray[0]), (unsigned char *)(m_cuEglFrames[index].frame.pArray[1]), (unsigned char *)(m_cuEglFrames[index].frame.pArray[2])};
      int nSrcStep[3] = {(int)surfaceParams0.planeParams.pitch[0], (int)surfaceParams0.planeParams.pitch[1], (int)surfaceParams0.planeParams.pitch[2]}; 
      int nDstStep = width * 3;
      NppiSize oSizeROI;
      oSizeROI.width = width;
      oSizeROI.height = height;
      int pStepSize = width * 3;
      NppStatus status = nppiYUV420ToBGR_8u_P3C3R(
          pSrc, nSrcStep,
          m_cudaImageDatas[index], pStepSize,
          oSizeROI);
      result = cuGraphicsUnregisterResource(m_cudaFrameResources[index]);
      int errorCode = NvBufSurfaceUnMapEglImage(bufSurface, 0);
      return true;
  }

Method above works, Thanks!