Why NvDsUserMetaList is NULL

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) AGXXavier
• DeepStream Version 6.3
• JetPack Version (valid for Jetson only) 5.1.2
• TensorRT Version 8.5.2

I am using gst-dsexample plugin.
Inside plugin cpp file, I tried to get NvDsUserMetaList from NvDsFrameMeta to save image as jpg file as follow.

NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;

But usrMetaList is NULL.

The whole code is here

if (dsexample->process_full_frame) {    
    //std::chrono::time_point<std::chrono::system_clock> before = std::chrono::system_clock::now();
    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {
          NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
          NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;
          char fileFrameNameString[FILE_NAME_SIZE];
          const char *osd_string = "OSD";
          FILE *file;
          g_print("here");
          for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
              g_print("here_1");
              obj_meta = (NvDsObjectMeta *) (l_obj->data);
              if (obj_meta->class_id == PGIE_CLASS_ID_NOHELMET) {
                   g_print("here_2");                   
		   int stream_num = 0;
		   while (usrMetaList != NULL) {
		        g_print("here_3");
			NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
			if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
			  snprintf (fileFrameNameString, FILE_NAME_SIZE, "%s_frame_%d.jpg", osd_string, stream_num++);
			  NvDsObjEncOutParams *enc_jpeg_image = (NvDsObjEncOutParams *) usrMetaData->user_meta_data;
			  file = fopen (fileFrameNameString, "wb");
			  fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t),
			      enc_jpeg_image->outLen, file);
			  fclose (file);
			}
			usrMetaList = usrMetaList->next;
		   }
       
              }
          }
    }

As you know, NvDsUserMeta is user-specific metadata.Customized by user.

I think you may refer to the deepstream-image-meta-test sample.

In this example, NVDS_CROP_IMAGE_META is added by nvds_obj_enc_process.

How do you use dsexample? Do you use the preprocess plugin?

At this time, the pipeline did not detect the object.

I am using gst-dsexample plugin. Inside gstdsexample.cpp, there is gst_dsexample_transform_ip function.

static GstFlowReturn
gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
{
      if (dsexample->process_full_frame) {

      }

}

Inside the function, object detection is already done.

Can I do including frame into NvDsUserMeta using nvds_obj_enc_process as follows.

NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
    /* For demonstration purposes, we will encode the first 10 frames. */
    if(frame_count <= 10) {
      NvDsObjEncUsrArgs frameData = { 0 };
      /* Preset */
      frameData.isFrame = 1;
      /* To be set by user */
      frameData.saveImg = save_img;
      frameData.attachUsrMeta = attach_user_meta;
      /* Set if Image scaling Required */
      frameData.scaleImg = FALSE;
      frameData.scaledWidth = 0;
      frameData.scaledHeight = 0;
      /* Quality */
      frameData.quality = 80;
      /* Main Function Call */
      nvds_obj_enc_process (ctx, &frameData, ip_surf, NULL, frame_meta);
    }

Then I can convert to jpg as follow right after including it.

while (usrMetaList != NULL) {
          NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
          if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
            NvDsObjEncOutParams *enc_jpeg_image =
                (NvDsObjEncOutParams *) usrMetaData->user_meta_data;

            snprintf (fileObjNameString, FILE_NAME_SIZE, "%s_%d_%d_%d_%s.jpg",
                osd_string, frame_number, frame_meta->batch_id, num_rects,
                obj_meta->obj_label);
            /* Write to File */
            file = fopen (fileObjNameString, "wb");
            fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t),
                enc_jpeg_image->outLen, file);
            fclose (file);
            usrMetaList = NULL;
          } else {
            usrMetaList = usrMetaList->next;
          }
        }

Is that ok or any better approach?

Can I do direct conversion as

NvDsObjEncOutParams *enc_jpeg_image = frame data;

Do you want to save all detected objects as jpeg files? I think this approach is possible.

You can call nvds_obj_enc_process encoding in dsexample, and call nvds_obj_enc_finish to wait for completion.

Then access NvDsUserMeta in the downstream plugin, and then save the jpeg file

I have issue creating ctx in gstdsexample.cpp's gst_dsexample_transform_ip function.

static GstFlowReturn
gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
{

    NvDsObjEncCtxHandle obj_ctx_handle = nvds_obj_enc_create_context (dsexample->gpu_id);
  if (!obj_ctx_handle) {
    g_print ("Unable to create context\n");
    return -1;
  }
  gpointer ctx = (gpointer) obj_ctx_handle;

}

My whole code is as follows.

static GstFlowReturn
gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
{
  GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  GstMapInfo in_map_info;
  GstFlowReturn flow_ret = GST_FLOW_ERROR;
  gdouble scale_ratio = 1.0;
  NvBufSurface *surface = NULL;
  NvDsBatchMeta *batch_meta = NULL;
  NvDsFrameMeta *frame_meta = NULL;
  NvDsObjectMeta *obj_meta = NULL;
  NvDsMetaList * l_frame = NULL;
  NvDsMetaList *l_obj = NULL;
  guint i = 0;
  static gchar font_name[] = "Serif";
  char fileFrameNameString[FILE_NAME_SIZE];
  const char *osd_string = "restitude";
  dsexample->frame_num++;
  CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id), "Unable to set cuda device");
  
  NvDsObjEncCtxHandle obj_ctx_handle = nvds_obj_enc_create_context (dsexample->gpu_id);
  if (!obj_ctx_handle) {
    g_print ("Unable to create context\n");
    return -1;
  }
  gpointer ctx = (gpointer) obj_ctx_handle;
  
  memset (&in_map_info, 0, sizeof (in_map_info));
  if (!gst_buffer_map (inbuf, &in_map_info, GST_MAP_READ)) {
    g_print ("Error: Failed to map gst buffer\n");
    goto error;
  }

  nvds_set_input_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  surface = (NvBufSurface *) in_map_info.data;
  GST_DEBUG_OBJECT (dsexample, "Processing Frame %" G_GUINT64_FORMAT " Surface %p\n", dsexample->frame_num, surface);

  if (CHECK_NVDS_MEMORY_AND_GPUID (dsexample, surface))
    goto error;

  batch_meta = gst_buffer_get_nvds_batch_meta (inbuf);
  if (batch_meta == nullptr) {
    GST_ELEMENT_ERROR (dsexample, STREAM, FAILED, ("NvDsBatchMeta not found for input buffer."), (NULL));
    return GST_FLOW_ERROR;
  }
  
  if (dsexample->process_full_frame) {    
    //std::chrono::time_point<std::chrono::system_clock> before = std::chrono::system_clock::now();
    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {
          NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
          NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;
          NvDsObjEncUsrArgs frameData = { 0 };
          /* Preset */
          frameData.isFrame = 1;
          /* To be set by user */
          frameData.saveImg = TRUE;
          frameData.attachUsrMeta = TRUE;
          /* Set if Image scaling Required */
          frameData.scaleImg = FALSE;
          frameData.scaledWidth = 0;
          frameData.scaledHeight = 0;
          /* Quality */
          frameData.quality = 80;
          /* Main Function Call */
          nvds_obj_enc_process (ctx, &frameData, surface, NULL, frame_meta);
          nvds_obj_enc_finish (ctx);
          
          char fileFrameNameString[FILE_NAME_SIZE];
          const char *osd_string = "OSD";
          FILE *file;
          g_print("here");
          for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
              g_print("here_1");
              obj_meta = (NvDsObjectMeta *) (l_obj->data);
              if (obj_meta->class_id == PGIE_CLASS_ID_NOHELMET) {
                   g_print("here_2");                   
		   int stream_num = 0;
		   while (usrMetaList != NULL) {
		        g_print("here_3");
			NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
			if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
			  snprintf (fileFrameNameString, FILE_NAME_SIZE, "%s_frame_%d.jpg", osd_string, stream_num++);
			  NvDsObjEncOutParams *enc_jpeg_image = (NvDsObjEncOutParams *) usrMetaData->user_meta_data;
			  file = fopen (fileFrameNameString, "wb");
			  fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t),
			      enc_jpeg_image->outLen, file);
			  fclose (file);
			}
			usrMetaList = usrMetaList->next;
		   }
       
              }
          }
    }
    //std::chrono::time_point<std::chrono::system_clock> after = std::chrono::system_clock::now();
    //auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(after - before).count();
    //g_print("interval %d \n", (int)millis);
    
  }
  
  return GST_FLOW_OK;

error:

  nvds_set_output_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  gst_buffer_unmap (inbuf, &in_map_info);
  return flow_ret;
}

The errors are as follows.

gstdsexample.cpp:807:13: error: invalid conversion from ‘int’ to ‘GstFlowReturn’ [-fpermissive]
  807 |     return -1;
      |             ^
      |             |
      |             int
gstdsexample.cpp:848:33: error: invalid conversion from ‘gpointer’ {aka ‘void*’} to ‘NvDsObjEncCtxHandle’ {aka ‘_NvDsObjEncCtx*’} [-fpermissive]
  848 |           nvds_obj_enc_process (ctx, &frameData, surface, NULL, frame_meta);
      |                                 ^~~
      |                                 |
      |                                 gpointer {aka void*}
In file included from gstdsexample.cpp:30:
../../includes/nvds_obj_encode.h:92:28: note:   initializing argument 1 of ‘bool nvds_obj_enc_process(NvDsObjEncCtxHandle, NvDsObjEncUsrArgs*, NvBufSurface*, NvDsObjectMeta*, NvDsFrameMeta*)’
   92 | bool nvds_obj_enc_process (NvDsObjEncCtxHandle, NvDsObjEncUsrArgs *,
      |                            ^~~~~~~~~~~~~~~~~~~
gstdsexample.cpp:849:32: error: invalid conversion from ‘gpointer’ {aka ‘void*’} to ‘NvDsObjEncCtxHandle’ {aka ‘_NvDsObjEncCtx*’} [-fpermissive]
  849 |           nvds_obj_enc_finish (ctx);
      |                                ^~~
      |                                |
      |                                gpointer {aka void*}
In file included from gstdsexample.cpp:30:
../../includes/nvds_obj_encode.h:96:27: note:   initializing argument 1 of ‘void nvds_obj_enc_finish(NvDsObjEncCtxHandle)’
   96 | void nvds_obj_enc_finish (NvDsObjEncCtxHandle);
      |                           ^~~~~~~~~~~~~~~~~~~
gstdsexample.cpp:886:1: error: jump to label ‘error’
  886 | error:
      | ^~~~~
gstdsexample.cpp:127:10: note:   from here
  127 |     goto error; \
      |          ^~~~~
gstdsexample.cpp:803:3: note: in expansion of macro ‘CHECK_CUDA_STATUS’
  803 |   CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id), "Unable to set cuda device");
      |   ^~~~~~~~~~~~~~~~~
gstdsexample.cpp:809:12: note:   crosses initialization of ‘void* ctx’
  809 |   gpointer ctx = (gpointer) obj_ctx_handle;
      |            ^~~
gstdsexample.cpp:804:23: note:   crosses initialization of ‘_NvDsObjEncCtx* obj_ctx_handle’
  804 |   NvDsObjEncCtxHandle obj_ctx_handle = nvds_obj_enc_create_context (gpu_id);
      |                       ^~~~~~~~~~~~~~
make: *** [Makefile:80: gstdsexample.o] Error 1

out.patch (3.6 KB)

This is a patch for gst-dsexample. Here I saved the full_frame, If you want to save the object, you need to modify it on this basis.

Another tip, if saveImg is set to true, the jpeg file will be saved directly,

and If attachUsrMeta is true, it means that the jpeg buffer will be stored to NvDsUserMeta.

A simple test method
Add the following configuration to source2_1080p_dec_infer-resnet_demux_int8.txt, and then run the following command

[ds-example]
enable=1
gpu-id=0
unique-id=0
full-frame=1

./deepstream-app -c /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/source2_1080p_dec_infer-resnet_demux_int8.txt

Thanks but sorry to raise question again.
How do I use this out.patch?
Where should I put?

diff --git a/sources/gst-plugins/gst-dsexample/gstdsexample.cpp b/sources/gst-plugins/gst-dsexample/gstdsexample.cpp
index 7d19e1f..38a1c36 100644
--- a/sources/gst-plugins/gst-dsexample/gstdsexample.cpp
+++ b/sources/gst-plugins/gst-dsexample/gstdsexample.cpp
@@ -423,6 +423,8 @@ gst_dsexample_start (GstBaseTransform * btrans)
   GST_DEBUG_OBJECT (dsexample, "created CV Mat\n");
 #endif
 
+  dsexample->frame_count = 0;
+  dsexample->obj_ctx_handle = nvds_obj_enc_create_context (dsexample->gpu_id);
   return TRUE;
 error:
   if (dsexample->host_rgb_buf) {
@@ -473,6 +475,8 @@ gst_dsexample_stop (GstBaseTransform * btrans)
 
   GST_DEBUG_OBJECT (dsexample, "ctx lib released \n");
 
+  /* Destroy context for Object Encoding */
+  nvds_obj_enc_destroy_context (dsexample->obj_ctx_handle);
   return TRUE;
 }
 
@@ -732,6 +736,7 @@ gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
       l_frame = l_frame->next)
     {
       frame_meta = (NvDsFrameMeta *) (l_frame->data);
+#if 0
       NvOSD_RectParams rect_params;
 
       /* Scale the entire frame to processing resolution */
@@ -761,8 +766,24 @@ gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
       attach_metadata_full_frame (dsexample, frame_meta, scale_ratio, output, i);
       i++;
       free (output);
+#endif
+    if(dsexample->frame_count <= 10) {
+        NvDsObjEncUsrArgs frameData = { 0 };
+        /* Preset */
+        frameData.isFrame = 1;
+        /* To be set by user */
+        frameData.saveImg = false;
+        frameData.attachUsrMeta = true;
+        /* Set if Image scaling Required */
+        frameData.scaleImg = FALSE;
+        frameData.scaledWidth = 0;
+        frameData.scaledHeight = 0;
+        /* Quality */
+        frameData.quality = 80;
+        /* Main Function Call */
+        nvds_obj_enc_process (dsexample->obj_ctx_handle, &frameData, surface, NULL, frame_meta);
+      }
     }
-
   } else {
     /* Using object crops as input to the algorithm. The objects are detected by
      * the primary detector */
@@ -894,7 +915,8 @@ gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
   flow_ret = GST_FLOW_OK;
 
 error:
-
+  nvds_obj_enc_finish (dsexample->obj_ctx_handle);
+  dsexample->frame_count++;
   nvds_set_output_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
   gst_buffer_unmap (inbuf, &in_map_info);
   return flow_ret;
diff --git a/sources/gst-plugins/gst-dsexample/gstdsexample.h b/sources/gst-plugins/gst-dsexample/gstdsexample.h
index c91a539..8f11069 100644
--- a/sources/gst-plugins/gst-dsexample/gstdsexample.h
+++ b/sources/gst-plugins/gst-dsexample/gstdsexample.h
@@ -45,6 +45,8 @@
 #include "gstnvdsmeta.h"
 #include "dsexample_lib/dsexample_lib.h"
 
+#include "nvds_obj_encode.h"
+
 /* Package and library details required for plugin_init */
 #define PACKAGE "dsexample"
 #define VERSION "1.0"
@@ -117,6 +119,9 @@ struct _GstDsExample
 
   // Boolean indicating if to blur the detected objects
   gboolean blur_objects;
+
+  NvDsObjEncCtxHandle obj_ctx_handle;
+  guint frame_count;
 };

Just merge the above modifications into gst-dsexample

I have never patched before. How to merge to gst-dsexample? Thanks a lot.

If you don’t know how to use patch, just copy these changes to gst-dsexample, they are the same

No I just save the whole image only.

I did merged and I can run program.
My code is as follows.

if (dsexample->process_full_frame) {    
    //std::chrono::time_point<std::chrono::system_clock> before = std::chrono::system_clock::now();
    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {
          NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
          NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;
          NvDsObjEncUsrArgs frameData = { 0 };
          /* Preset */
          frameData.isFrame = 1;
          /* To be set by user */
          frameData.saveImg = true;
          frameData.attachUsrMeta = true;
          /* Set if Image scaling Required */
          frameData.scaleImg = FALSE;
          frameData.scaledWidth = 0;
          frameData.scaledHeight = 0;
          /* Quality */
          frameData.quality = 80;
          /* Main Function Call */
          nvds_obj_enc_process (dsexample->obj_ctx_handle, &frameData, surface, NULL, frame_meta);
          
          char fileFrameNameString[FILE_NAME_SIZE];
          const char *osd_string = "OSD";
          FILE *file;
          g_print("here");
          for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
              g_print("here_1");
              obj_meta = (NvDsObjectMeta *) (l_obj->data);
              if (obj_meta->class_id == PGIE_CLASS_ID_NOHELMET) {
                   g_print("here_2");                   
		   int stream_num = 0;
		   while (usrMetaList != NULL) {
		        g_print("here_3");
			NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
			if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
			  snprintf (fileFrameNameString, FILE_NAME_SIZE, "%s_frame_%d.jpg", osd_string, stream_num++);
			  NvDsObjEncOutParams *enc_jpeg_image = (NvDsObjEncOutParams *) usrMetaData->user_meta_data;
			  file = fopen (fileFrameNameString, "wb");
			  fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t), enc_jpeg_image->outLen, file);
			  fclose (file);
			}
			usrMetaList = usrMetaList->next;
		   }
       
              }
          }
    }
    //std::chrono::time_point<std::chrono::system_clock> after = std::chrono::system_clock::now();
    //auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(after - before).count();
    //g_print("interval %d \n", (int)millis);
    
  }

First usermeta is attached using nvds_obj_enc_process.
Then while (usrMetaList != NULL) is not NULL, then print jpg.

But usrMetaList is still NULL.
I am using this config file because I also need to display.
rectitude_config_main.txt (6.6 KB)

error:
  nvds_obj_enc_finish (dsexample->obj_ctx_handle);
  dsexample->frame_count++;
  if (dsexample->process_full_frame) {
    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
      l_frame = l_frame->next) {
      g_print("here \n");
      NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
      NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;
      char fileFrameNameString[128] = { 0 };
      const char *osd_string = "OSD";
      while (usrMetaList != NULL) {
        NvDsUserMeta *usrMetaData = (NvDsUserMeta *) usrMetaList->data;
        if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
          snprintf (fileFrameNameString, 128, "%s_frame_%d.jpg", osd_string, dsexample->frame_count);
          NvDsObjEncOutParams *enc_jpeg_image = (NvDsObjEncOutParams *) usrMetaData->user_meta_data;
          FILE *file = fopen (fileFrameNameString, "wb");
          fwrite (enc_jpeg_image->outBuffer, sizeof (uint8_t), enc_jpeg_image->outLen, file);
          fclose (file);
        }
        usrMetaList = usrMetaList->next;
      }
    }
  }
  nvds_set_output_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  gst_buffer_unmap (inbuf, &in_map_info);
  return flow_ret;
}

nvds_obj_enc_process is an asynchronously executed function. You must call nvds_obj_enc_finish and wait for completion.

And then NvDsFrameMeta will be modified.

Now I can save.
I just need the following code to save jpg.

if (dsexample->process_full_frame) {    
    //std::chrono::time_point<std::chrono::system_clock> before = std::chrono::system_clock::now();
    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {
          NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);          
          
          
          for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) {
              obj_meta = (NvDsObjectMeta *) (l_obj->data);
              if (obj_meta->class_id == PGIE_CLASS_ID_NOHELMET) {
                   NvDsObjEncUsrArgs frameData = { 0 };
		   /* Preset */
		   frameData.isFrame = 1;
		   /* To be set by user */
		   frameData.saveImg = true;
		   frameData.attachUsrMeta = false;
		   /* Set if Image scaling Required */
		   frameData.scaleImg = FALSE;
		   frameData.scaledWidth = 0;
		   frameData.scaledHeight = 0;
		   /* Quality */
		   frameData.quality = 80;
	           nvds_obj_enc_process (dsexample->obj_ctx_handle, &frameData, surface, NULL, frame_meta);
              }
          }
          //dsexample->frame_count++;
    }

But wonder how file names are created. Even dsexample->frame_count is disabled, file names are still with increasing numbers. Where file names are created?

As I said above, if you set saveImg to true, nvds_obj_enc_process will automatically name and save the jpeg file

1 Like

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.