Parameters for conversion from RGB to YUV with NvMedia

Hi

I’d like to read RGB image data via OpenCV and convert it to YUV444 with NvMedia2DBlitEx on Drive AGX.
I implemented code referring to img_producer.c in ipp_yuv,
however I got the following error message at NvMediaImageCreateNew for output.

NvMediaSurfaceType 1018 is deprecated.
Use NvMediaSurfaceFormatGetType function to get new surface type.
NvMediaImageLock: Failed Params check

My code is as follows.
Could you give me some advice to fix the problem?

typedef struct {
NvMediaDevice* device;
char* inputImages;
uint32_t width;
uint32_t height;
uint32_t ippNum;
uint32_t frameCount;
NvMediaSurfaceType surfaceType;
NvMedia2D* i2d;
NvMediaRect dstRect;
NvMediaRect srcRect;
NvMedia2DBlitParameters* blitParams;
} ImageProducerCtx;

void NvMediaBGR2YUV(const cv::Mat& bgr) {
NvMediaStatus status;
ImageProducerCtx* client = new ImageProducerCtx;
if (!client) {
printf("%s:: failed to alloc memory\n", func);
return;
}
memset(client, 0, sizeof(ImageProducerCtx));
client->device = NvMediaDeviceCreate();
client->i2d = NvMedia2DCreate(client->device);
if (!client->i2d) {
printf("%s: Failed to create NvMedia 2D i2d\n", func);
return;
}
NvMediaSurfAllocAttr surfAllocAttrs[8];
surfAllocAttrs[0].type = NVM_SURF_ATTR_WIDTH;
surfAllocAttrs[0].value = 1920 * 3;
surfAllocAttrs[1].type = NVM_SURF_ATTR_HEIGHT;
NvMediaImageSurfaceMap inputSurfaceMap;
status = NvMediaImageLock(input, NVMEDIA_IMAGE_ACCESS_READ, &inputSurfaceMap);
status = NvMediaImagePutBits(input, &dstRect, (void**)&bgr.data, &dstPitch);
NvMediaImageUnlock(input);
status = NvMedia2DImageRegister(client->i2d, input, NVMEDIA_ACCESS_MODE_READ);
status = NvMedia2DImageRegister(client->i2d, output,
NVMEDIA_ACCESS_MODE_READ_WRITE);
NVM_SURF_FMT_DEFINE_ATTR(output_surface_format_attr);
NVM_SURF_FMT_SET_ATTR_YUV(output_surface_format_attr, YUV, 444, PLANAR, UINT,
8, PL);
NvMediaSurfaceType output_type =
NvMediaSurfaceFormatGetType(output_surface_format_attr, 7);

NvMediaImage* input =
NvMediaImageCreateNew(client->device, input_type, surfAllocAttrs, 2, 0);
NvMediaImage* output =
NvMediaImageCreateNew(client->device, output_type, surfAllocAttrs, 2, 0);
input->type = NvMediaSurfaceType_Image_RGBA;
input->width = 1920;
input->height = 1080;
input->imageCount = 1;
input->colorStd = NVMEDIA_2D_BLIT_PARAMS_COLOR_STD;
output->type = NvMediaSurfaceType_Image_YUV_444;
output->width = 1920;
output->height = 1080;
output->imageCount = 1;
output->colorStd = NVMEDIA_2D_BLIT_PARAMS_COLOR_STD;
NvMediaRect dstRect = {0, 0, 1920, 1080};
const uint32_t dstPitch = 1920 * 3;
NvMediaImageSurfaceMap inputSurfaceMap;
status = NvMediaImageLock(input, NVMEDIA_IMAGE_ACCESS_READ, &inputSurfaceMap);
status = NvMediaImagePutBits(input, &dstRect, (void**)&bgr.data, &dstPitch);
NvMediaImageUnlock(input);
status = NvMedia2DImageRegister(client->i2d, input, NVMEDIA_ACCESS_MODE_READ);
status = NvMedia2DImageRegister(client->i2d, output,
NVMEDIA_ACCESS_MODE_READ_WRITE);

client->blitParams = new NvMedia2DBlitParameters;
memset(client->blitParams, 0, sizeof(NvMedia2DBlitParameters));
if (!client->blitParams) {
printf(“Out of memory\n”);
return;
}
status = NvMedia2DBlitEx(client->i2d, output, NULL, input, NULL,
client->blitParams, NULL);
cv::Mat dst(1080, 1920, CV_8UC3);
status = NvMediaImageGetBits(output, &dstRect, (void**)&bgr.data, &dstPitch);
cv::imwrite("./yuv_nvmedia.png", dst);

status = NvMedia2DImageUnRegister(client->i2d, input);
status = NvMedia2DImageUnRegister(client->i2d, output);
NvMedia2DDestroy(client->i2d);
NvMediaDeviceDestroy(client->device);
delete client->blitParams;
delete output;
delete input;
delete client;
}

Hardware Platform: [DRIVE AGX Xavier™ Developer Kit]
Software Version: [DRIVE Software 10]
Host Machine Version: [Example: native Ubuntu 18.04]
SDK Manager Version: [Example: 1.0.1.5538]

Hi @eri.kasamatsu,

NvMediaSurfaceType_Image_RGBA is Obsolete RGBA image type per the document. You cannot use it. Pleas refer to our samples to use NVM_SURF_FMT_SET_ATTR_RGBA macro defined in nvmedia_surface.h.

Thanks!

Hi @VickNV

Thank you for your advice.
I could execute my program but I got unexpected result.

I guess some of my parameters are still wrong,

 void NvMediaBGR2YUV(const cv::Mat& bgr) {
  NvMediaStatus status;
  ImageProducerCtx* client = new ImageProducerCtx;
  if (!client) {
    printf("%s:: failed to alloc memory\n", __func__);
    return;
  }
  memset(client, 0, sizeof(ImageProducerCtx));
  client->device = NvMediaDeviceCreate();
  client->i2d = NvMedia2DCreate(client->device);
  if (!client->i2d) {
    printf("%s: Failed to create NvMedia 2D i2d\n", __func__);
    return;
  }
  NvMediaSurfAllocAttr surfAllocAttrs[8];
  surfAllocAttrs[0].type = NVM_SURF_ATTR_WIDTH;
  surfAllocAttrs[0].value = 1920 * 3;
  surfAllocAttrs[1].type = NVM_SURF_ATTR_HEIGHT;
  surfAllocAttrs[1].value = 1080;
  // surfAllocAttrs[2].type = NVM_SURF_ATTR_CPU_ACCESS;
  // surfAllocAttrs[2].value = NVM_SURF_ATTR_CPU_ACCESS_UNCACHED;
  NVM_SURF_FMT_DEFINE_ATTR(intput_surface_format_attr);
  NVM_SURF_FMT_SET_ATTR_RGBA(intput_surface_format_attr, RGBA, UINT, 8, PL);
  NvMediaSurfaceType input_type =
      NvMediaSurfaceFormatGetType(intput_surface_format_attr, 7);

  NvMediaSurfAllocAttr outSurfAllocAttrs[8];
  outSurfAllocAttrs[0].type = NVM_SURF_ATTR_WIDTH;
  outSurfAllocAttrs[0].value = 1920 * 3;
  outSurfAllocAttrs[1].type = NVM_SURF_ATTR_HEIGHT;
  outSurfAllocAttrs[1].value = 1080;
  NVM_SURF_FMT_DEFINE_ATTR(output_surface_format_attr);
  NVM_SURF_FMT_SET_ATTR_YUV(output_surface_format_attr, YUV, 444, PLANAR, UINT, 8, PL);
  NvMediaSurfaceType output_type =
      NvMediaSurfaceFormatGetType(output_surface_format_attr, 7);

  NvMediaImage* input =
      NvMediaImageCreateNew(client->device, input_type, surfAllocAttrs, 2, 0);
  NvMediaImage* output = NvMediaImageCreateNew(client->device, output_type, outSurfAllocAttrs, 2, 0);
  input->type = input_type;
  input->width = 1920;
  input->height = 1080;
  input->imageCount = 1;
  input->colorStd = NVMEDIA_2D_BLIT_PARAMS_COLOR_STD;
  output->type = output_type;
  output->width = 1920;
  output->height = 1080;
  output->imageCount = 1;
  output->colorStd = NVMEDIA_2D_BLIT_PARAMS_COLOR_STD;
  const uint32_t input_dstPitch = 1920 * 3;
  NvMediaImageSurfaceMap inputSurfaceMap;
  status =
      NvMediaImageLock(input, NVMEDIA_IMAGE_ACCESS_WRITE, &inputSurfaceMap);
  status = NvMediaImagePutBits(input, NULL, (void**)&bgr.data, &input_dstPitch);
  NvMediaImageUnlock(input);

  client->blitParams = new NvMedia2DBlitParameters;
  memset(client->blitParams, 0, sizeof(NvMedia2DBlitParameters));
  if (!client->blitParams) {
    printf("%s: Out of memory\n", __func__);
    return;
  }
  client->blitParams->filter = NVMEDIA_2D_STRETCH_FILTER_OFF;
  status =
      NvMediaImageLock(input, NVMEDIA_IMAGE_ACCESS_WRITE, &inputSurfaceMap);
  NvMediaImageUnlock(input);
  status = NvMedia2DBlitEx(client->i2d, output, NULL, input, NULL,
                           client->blitParams, NULL);

  NvMediaImageSurfaceMap outputSurfaceMap;
  status =
      NvMediaImageLock(output, NVMEDIA_IMAGE_ACCESS_READ, &outputSurfaceMap);
  NvMediaImageUnlock(output);
  cv::Mat dst(1080, 1920, CV_8UC3);
  uint32_t output_dstPitch[3] = {1920, 1920, 1920};
  status =
      NvMediaImageLock(output, NVMEDIA_IMAGE_ACCESS_READ, &outputSurfaceMap);
  status =
      NvMediaImageGetBits(output, NULL, (void**)&dst.data, output_dstPitch);
  cv::imwrite("./yuv_nvmedia.png", dst);
  NvMediaImageUnlock(output);
}

This is expected one.

sorry, i forgot that rgb order of opencv is BGR and i had to use RGBA.

then i modified my code and still got unexpected result.

void NvMediaBGR2YUV(const cv::Mat& bgr) {
  cv::Mat rgb;
  cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGBA);
  int ch = rgb.channels();
  NvMediaStatus status;
  ImageProducerCtx* client = new ImageProducerCtx;
  if (!client) {
    printf("%s:: failed to alloc memory\n", __func__);
    return;
  }
  memset(client, 0, sizeof(ImageProducerCtx));
  client->device = NvMediaDeviceCreate();
  client->i2d = NvMedia2DCreate(client->device);
  if (!client->i2d) {
    printf("%s: Failed to create NvMedia 2D i2d\n", __func__);
    return;
  }
  NvMediaSurfAllocAttr surfAllocAttrs[8];
  surfAllocAttrs[0].type = NVM_SURF_ATTR_WIDTH;
  surfAllocAttrs[0].value = 1920;
  surfAllocAttrs[1].type = NVM_SURF_ATTR_HEIGHT;
  surfAllocAttrs[1].value = 1080;
  NVM_SURF_FMT_DEFINE_ATTR(intput_surface_format_attr);
  NVM_SURF_FMT_SET_ATTR_RGBA(intput_surface_format_attr, RGBA, UINT, 8, PL);
  NvMediaSurfaceType input_type =
      NvMediaSurfaceFormatGetType(intput_surface_format_attr, 7);

  NvMediaSurfAllocAttr outSurfAllocAttrs[8];
  outSurfAllocAttrs[0].type = NVM_SURF_ATTR_WIDTH;
  outSurfAllocAttrs[0].value = 1920;
  outSurfAllocAttrs[1].type = NVM_SURF_ATTR_HEIGHT;
  outSurfAllocAttrs[1].value = 1080;
  NVM_SURF_FMT_DEFINE_ATTR(output_surface_format_attr);
  NVM_SURF_FMT_SET_ATTR_YUV(output_surface_format_attr, YUV, 444, PLANAR, UINT,
                            8, PL);
  NvMediaSurfaceType output_type =
      NvMediaSurfaceFormatGetType(output_surface_format_attr, 7);

  NvMediaImage* input =
      NvMediaImageCreateNew(client->device, input_type, surfAllocAttrs, 2, 0);
  NvMediaImage* output = NvMediaImageCreateNew(client->device, output_type,
                                               outSurfAllocAttrs, 2, 0);
  input->type = input_type;
  input->width = 1920;
  input->height = 1080;
  input->imageCount = 1;
  input->colorStd = NVMEDIA_2D_BLIT_PARAMS_COLOR_STD;
  output->type = output_type;
  output->width = 1920;
  output->height = 1080;
  output->imageCount = 1;
  output->colorStd = NVMEDIA_2D_BLIT_PARAMS_COLOR_STD;
  const uint32_t input_dstPitch = 1920 * 4;
  NvMediaImageSurfaceMap inputSurfaceMap;
  status =
      NvMediaImageLock(input, NVMEDIA_IMAGE_ACCESS_WRITE, &inputSurfaceMap);
  status = NvMediaImagePutBits(input, NULL, (void**)&rgb.data, &input_dstPitch);
  NvMediaImageUnlock(input);

  client->blitParams = new NvMedia2DBlitParameters;
  memset(client->blitParams, 0, sizeof(NvMedia2DBlitParameters));
  if (!client->blitParams) {
    printf("%s: Out of memory\n", __func__);
    return;
  }
  client->blitParams->filter = NVMEDIA_2D_STRETCH_FILTER_OFF;
  status =
      NvMediaImageLock(input, NVMEDIA_IMAGE_ACCESS_WRITE, &inputSurfaceMap);
  NvMediaImageUnlock(input);
  status = NvMedia2DBlitEx(client->i2d, output, NULL, input, NULL,
                           client->blitParams, NULL);

  NvMediaImageSurfaceMap outputSurfaceMap;
  status =
      NvMediaImageLock(output, NVMEDIA_IMAGE_ACCESS_READ, &outputSurfaceMap);
  NvMediaImageUnlock(output);
  uint32_t output_dstPitch[3] = {1920, 1920, 1920};
  status =
      NvMediaImageLock(output, NVMEDIA_IMAGE_ACCESS_READ, &outputSurfaceMap);

  cv::Mat dst(1080, 1920, CV_8UC3);
  status =
      NvMediaImageGetBits(output, NULL, (void**)&dst.data, output_dstPitch);
  cv::imwrite("./yuv_nvmedia.png", dst);
  NvMediaImageUnlock(output);
}

I’m thinking about it may be something to do with your surface format and the opencv function.
You can refer to WriteImageNew() in ~/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_DDPX/DRIVEOS/drive-t186ref-linux/samples/nvmedia/utils/surf_utils.c to understand how we save image files and can also just call it. Thanks!

Hi @VickNV

Thank you for your help.
I could get expected result.

Let me ask you some questions.

  1. NvMedia2DBlitEx does not support RGB format so we have to convert from rgb to rgba before we use NvMedia2DBlitEx. Is my understanding correct?

  2. Can NvMediaImagePutBits and NvMediaImageGetBits be a bottleneck of the program when we use NvMedia2DBlitEx just to convert RGBA->YUV as a part of our program?
    It seems that NvMediaImagePutBits and NvMediaImageGetBits take too much time than NvMedia2DBlitEx.

  3. Is copy processing of NvMediaImagePutBits and NvMediaImageGetBits by CPU, or by DMA?

Could you share how you actually solve your problem?

Yes

Per “Image Read and Write by Client” document, it’s not for production as below.

For better performance, please check NvSciBuf in below document and sample:

  • NvStreams document
  • ~/nvidia/nvidia_sdk/DRIVE_Software_10.0_Linux_OS_DDPX/DRIVEOS/drive-t186ref-linux/samples/nvsci

Could you share how you actually solve your problem?

I had to set each buffer pointers, as below.

constexpr int kDstCh = 3;
uint32_t dstPitches[kDstCh] = {bgr.cols, bgr.cols, bgr.cols};
int size = bgr.cols * bgr.rows;
int imageSize = size * kDstCh;
std::vector<cv::Mat> buffers(kDstCh);
uint8_t* buffer_ptrs[kDstCh];
for (int i = 0; i < kDstCh; i++) {
  buffers[i] = cv::Mat(bgr.rows, bgr.cols, CV_8UC1);
  buffer_ptrs[i] = buffers[i].ptr();
}

Preformatted textstatus = NvMediaImageGetBits(output, NULL, (void**)buffer_ptrs, dstPitches);

I’m trying to use NvSciBuf and got the following error at NvSciBufAttrListReconcileAndObjAlloc.
What I want to do is that

  1. Create test image data(cv::Mat ) on CPU.
  2. Transfer it to NvMediaImage using NvSciBuf.
  3. Get a pointer to the destination surface plane.
  4. Visualize data to confirm that it is equal to test image data.

[ERROR: NvSciBufAttrListCommonSetAttr]: Invalid operation on key
[ERROR: NvSciBufAttrListCommonSetAttr]: key: NvSciBufGeneralAttrKey_Types, readonly: 0, status: 0x765130
[ERROR: NvSciBufAttrListSlotSetAttrs]: Failed to set Buffer type
[NvMediaImageFillNvSciBufAttrs:779] NvMNvSciBufAttrListSetAttrs failed (error code = 256)

void NvMediaBGR2YUVwithNvSciBuf(const cv::Mat& bgr) {
        NvSciBufObj bufobj_in;
        NvSciBufObj bufobj_out;
      NvMediaImage* nvmimg_in;
      NvMediaImage* nvmimg_out;
      NvMediaDevice* nvmdevice = NvMediaDeviceCreate();
      NvMediaImageNvSciBufInit();
      NvMedia2D* nvm2d = NvMedia2DCreate(nvmdevice);
      NvMediaStatus status;
      NvSciError sci_ret;
      NvSciBufModule bufModule;
      sci_ret = NvSciBufModuleOpen(&bufModule);

  // for input
  NvMediaSurfAllocAttr surfAllocAttrs[8];
  int in_attr_num = 0;
  surfAllocAttrs[in_attr_num].type = NVM_SURF_ATTR_WIDTH;
  surfAllocAttrs[in_attr_num++].value = bgr.cols;
  surfAllocAttrs[in_attr_num].type = NVM_SURF_ATTR_HEIGHT;
  surfAllocAttrs[in_attr_num++].value = bgr.rows;
  surfAllocAttrs[in_attr_num].type = NVM_SURF_ATTR_CPU_ACCESS;
  surfAllocAttrs[in_attr_num++].value = NVM_SURF_ATTR_CPU_ACCESS_CACHED;

  NVM_SURF_FMT_DEFINE_ATTR(intput_surface_format_attr);
  NVM_SURF_FMT_SET_ATTR_RGBA(intput_surface_format_attr, RGBA, UINT, 8, PL);
  NvMediaSurfaceType input_type =
      NvMediaSurfaceFormatGetType(intput_surface_format_attr, 7);

  // setup buffer
  NvMediaImageSurfaceMap inputSurfaceMap, outputSurfaceMap;
  NvSciBufAttrList unreconciled_attrlist = NULL;
  NvSciBufAttrList reconciled_attrlist = NULL;
  NvSciBufAttrList conflictlist = NULL;
  NvSciBufType bufType = NvSciBufType_Image;
  NvSciBufAttrValAccessPerm access_perm = NvSciBufAccessPerm_ReadWrite;
  bool cpuaccess_flag = true;
  NvSciBufAttrKeyValuePair attr_kvp[] = {
      {NvSciBufGeneralAttrKey_RequiredPerm, &access_perm, sizeof(access_perm)},
      {NvSciBufGeneralAttrKey_Types, &bufType, sizeof(bufType)},
      {NvSciBufGeneralAttrKey_NeedCpuAccess, &cpuaccess_flag,
       sizeof(cpuaccess_flag)}};
  sci_ret = NvSciBufAttrListCreate(bufModule, &unreconciled_attrlist);
  sci_ret = NvSciBufAttrListSetAttrs(unreconciled_attrlist, attr_kvp, 3);

  status = NvMediaImageFillNvSciBufAttrs(nvmdevice, input_type, surfAllocAttrs,
                                         in_attr_num, 0, unreconciled_attrlist);
  sci_ret = NvSciBufAttrListReconcileAndObjAlloc(&unreconciled_attrlist, 1,
                                                 &bufobj_in, &conflictlist);
  status = NvMediaImageCreateFromNvSciBuf(nvmdevice, bufobj_in, &nvmimg_in);

  // copy RGBA data to NvMediaImage using NvSciBuf
  status =
      NvMediaImageLock(nvmimg_in, NVMEDIA_IMAGE_ACCESS_WRITE, &inputSurfaceMap);
  uint8_t* in_ptr;
  sci_ret = NvSciBufObjGetCpuPtr(bufobj_in, (void**)&in_ptr);
  cv::Mat rgb;
  cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGBA);
  memcpy(in_ptr, rgb.ptr(), rgb.cols * rgb.rows * rgb.channels());
  NvMediaImageUnlock(nvmimg_in);

  // Get data from NvMediaImage and confirm whether RGBA data is set to NvMediaImage correctly.
  cv::Mat getrgba(1080, 1920, CV_8UC4);
  uint32_t dstPitches = 1920 * 4;
  status =
      NvMediaImageLock(nvmimg_in, NVMEDIA_IMAGE_ACCESS_READ, &inputSurfaceMap);
  status =
      NvMediaImageGetBits(nvmimg_in, NULL, (void**)getrgba.data, &dstPitches);
  NvMediaImageUnlock(nvmimg_in);
  cv::imwrite("./scibuf_test.png", getrgba);
}
1 Like

If you don’t mind, please create another topic for this issue. Thanks!

@VickNV
I’ve created a new topic to discuss with usage of NvSciBuf.