Unable to Propagate SGIE Tensor Outputs Across Demux into OSD

Accelerated Computing Intelligent Video Analytics DeepStream SDK
,
1

• Hardware Platform (Jetson)
• DeepStream Version 7.1
• JetPack Version (6.2)
• Issue Type(Question)

Hi,

Im am trying to create a Deepstream App based on the multi in multi out sample app that has a PGIE that detects people, and then it does gaze estimation, age detection and so on with multiple SGIE. So far, I have been able to implement one SGIE, which manages to detect the gazes and print them on screen, it was possible with your help at [question link], but now I am running into a problem printing the gazes on screen.

My past implementation managed to print the gazes, but this was with a probe in the SGIE that printed the gazes there, which was not optimal as it deleted the gazes quickly and only printed one each frame (I want an approach such as the gazes are calculated each 3-4 frames and the prints are maintained for some frames until new gazes are detected, else delete the past gazes on screen), and the SGIE did not run on all faces at once. I think that to print them all at once I had to place the probe at the OSD component, which would give me all the data at once, but I am struggling on passing the metadata from the tensor outputs to the OSD component, as it seems that it gets deleted at the demux, so I managed to pass it as a .miscObject metadata component, which works, but it is unable to have the complete tensor output, rounding the value. I was trying to replicate the approach from the custom binding test sample app, but I reach the same problem where I don’t find data on frame_user_meta_list.

Basically, I wanted help on

  1. Find a way to propagate the metadata from my SGIEs to the OSD
  2. If my implementation/architecture of the position of probes is correct

The overall structure of the app is

[ uridecodebin ]
↓
[ nvstreammux ]
↓
[queue1]
↓
[pgie] (PeopleNet, unique-id=1)
↓
[nvtracker]
↓
[preprocess] (nvdspreprocess)
↓
[sgie] (ResNet34 gaze, unique-id=2)
↓ ← PAD PROBE @ sgie src pad: sgie_src_pad_buffer_probe
[nvstreamdemux]
↓
[ queue_i → nvvidconv_i → nvdsosd_i (OSD) → sink_i ]
↓ ← PAD PROBE @ nvdsosd sink pad: osd_buffer_probe

Here is my SGIE probe

def sgie_src_pad_buffer_probe(pad, info, u_data):
    gst_buffer = info.get_buffer()
    if not gst_buffer:
        return Gst.PadProbeReturn.OK


    batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))

    pyds.nvds_acquire_meta_lock(batch_meta)

    l_frame    = batch_meta.frame_meta_list

    while l_frame:
        frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)

        # User meta 
        user_meta = pyds.nvds_acquire_user_meta_from_pool(batch_meta)

        # 2) Walk all object metas
        l_obj = frame_meta.obj_meta_list
        while l_obj:
            obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)

            if obj_meta.class_id == PGIE_CLASS_ID_FACE:

                l_user = obj_meta.obj_user_meta_list
                while l_user:

                    um = pyds.NvDsUserMeta.cast(l_user.data)
                    if um.base_meta.meta_type == pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META:  # raw SGIE output

                        infer_meta = pyds.NvDsInferTensorMeta.cast(um.user_meta_data)

                        # 4) Softmax → compute pitch & yaw
                        pitch, yaw = get_gaze_angle_from_tensor(infer_meta, _BINWIDTH, _ANGLE_OFFSET)

                        user_meta = pyds.nvds_acquire_user_meta_from_pool(batch_meta)
                        # Here I am not sure if It is correct to do this, or if I have to create a custom meta type
                        payload = json.dumps({"pitch": pitch, "yaw": yaw})

                        data = pyds.alloc_custom_struct(user_meta)

                        data.message = payload 

                        user_meta.user_meta_data = data
                        user_meta.base_meta.meta_type = pyds.NvDsMetaType.NVDS_USER_META

                        # attach to the object, not the frame
                        pyds.nvds_add_user_meta_to_obj(obj_meta, user_meta)

                        # In order to get the data I need to scale it, but it results in lower precision, which I want to avoid
                        obj_meta.misc_obj_info[0] = int(pitch   * SCALE)
                        obj_meta.misc_obj_info[1] = int(yaw   * SCALE)

                        print(f"[SGIE] found obj {obj_meta.object_id} has pitch: {pitch:.2f} yaw: {yaw:.2f} during frame {frame_meta.frame_num}")

                    try:
                        l_user = l_user.next
                    except StopIteration:
                        break

            l_obj = l_obj.next
        l_frame = l_frame.next
    pyds.nvds_release_meta_lock(batch_meta)
    return Gst.PadProbeReturn.OK

From the OSD probe, I removed some things, but the core is

def osd_buffer_probe(pad, info, user_data):

    gst_buffer = info.get_buffer()
    if not gst_buffer:
        return Gst.PadProbeReturn.OK

    batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
    l_frame = batch_meta.frame_meta_list
    while l_frame is not None:
        try:
            frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data) #Objeto con metadatos de frame
            current_frame = frame_meta.frame_num #frame actual
            
            try:
                l_user = frame_meta.frame_user_meta_list
                print(l_user)
                
            except Exception as e:
                        print('l_user cast failed')
                        print(e)    

            l_obj = frame_meta.obj_meta_list


            while l_obj is not None:
                try:
                    obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
                    
                    if obj_meta.class_id == PGIE_CLASS_ID_FACE:
                        # **READ** them back
                        raw_pitch = obj_meta.misc_obj_info[0]/ SCALE
                        raw_yaw   = obj_meta.misc_obj_info[1]/ SCALE

                        print(f"[OSD] found obj {obj_meta.object_id} has pitch: {raw_pitch} yaw: {raw_yaw:.2f} during frame {current_frame}")
                
                    try:
                        l_obj = l_obj.next
                    except StopIteration:
                        break
                except StopIteration:
                        break

            try:
                l_frame = l_frame.next
            except StopIteration:
                break
        except StopIteration:
                break
    return Gst.PadProbeReturn.OK

This results in outputs like (note that the objects that are noted here are all faces)

[SGIE] found obj 2 has pitch: -0.48 yaw: 0.05 during frame 228
[SGIE] found obj 4 has pitch: -0.53 yaw: 0.07 during frame 228
None
[OSD] found obj 2 has pitch: -0.482 yaw: 0.04 during frame 225
[OSD] found obj 1 has pitch: 0.0 yaw: 0.00 during frame 225
[OSD] found obj 4 has pitch: -0.519 yaw: 0.07 during frame 225
[SGIE] found obj 21 has pitch: -0.53 yaw: 0.07 during frame 229
None
[SGIE] found obj 1 has pitch: -0.50 yaw: 0.07 during frame 229
[OSD] found obj 1 has pitch: -0.495 yaw: 0.07 during frame 226
[OSD] found obj 2 has pitch: 0.0 yaw: 0.00 during frame 226
[OSD] found obj 4 has pitch: 0.0 yaw: 0.00 during frame 226
None
[OSD] found obj 1 has pitch: 0.0 yaw: 0.00 during frame 227
[OSD] found obj 2 has pitch: 0.0 yaw: 0.00 during frame 227
[OSD] found obj 4 has pitch: 0.0 yaw: 0.00 during frame 227
[SGIE] found obj 2 has pitch: -0.48 yaw: 0.02 during frame 231
[SGIE] found obj 4 has pitch: -0.53 yaw: 0.07 during frame 231
None
[OSD] found obj 1 has pitch: 0.0 yaw: 0.00 during frame 228
[OSD] found obj 2 has pitch: -0.484 yaw: 0.05 during frame 228
[OSD] found obj 4 has pitch: -0.525 yaw: 0.07 during frame 228

The SGIE is frames in advance, and it is not calculating all the faces at the same time, which could be due to my SGIE settings or due to timing, but eventually I am only able to print it thanks to the mis_obj_info, as frame_user_meta_list is None

batch-size=1
secondary-reinfer-interval=2
classifier-async-mode=0
...

I am running this through the deepstream:7.1-triton-multiarch docker container and ran two sh files that are required for python dev.

3 
streamux --> pgie --> sgie --> nvstreamdemux --> nvvideoconvert --> nvdsosd --> sink
                            ^                                    ^
                            |                                    |
      parse tensor and add user meta in sgie src pad probe 
                                                                 |
                                                                 |
                                       get output tensor and user meta in osd sink pad probe

I added probe functions to sgie and nvdsosd respectively, and everything works fine.

The following is the key code, please refer to solve your own problems

sgie src pad probe

if user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META:
  infer_meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
  print(f"[DBG] num_output_layers={infer_meta.num_output_layers}")
  for layer_idx in range(infer_meta.num_output_layers):
	  layer = pyds.get_nvds_LayerInfo(infer_meta, layer_idx)
	  array = layer_tensor_to_ndarray(layer)
	  print(f"[DBG] {layer.layerName} layer_info.buffer address = {type(array)}")
  ##################################################
  user_meta = pyds.nvds_acquire_user_meta_from_pool(batch_meta)
  # make a fake payload
  payload = json.dumps({"pitch": 1.234, "yaw": 1.234})
  data = pyds.alloc_custom_struct(user_meta)
  data.message = payload
  data.structId = frame_number
  data.sampleInt = frame_number + 1
  user_meta.user_meta_data = data
  user_meta.base_meta.meta_type = pyds.NvDsMetaType.NVDS_USER_META
  pyds.nvds_add_user_meta_to_obj(obj_meta, user_meta)
  ##################################################
  break

osd sink pad probe

if user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDS_USER_META:
        customdata = pyds.CustomDataStruct.cast(user_meta.user_meta_data)
        print(f"customdata message {pyds.get_string(customdata.message)} structId {customdata.structId} sampleInt {customdata.sampleInt}")
elif user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META:
        infer_meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
        print(f"[osd] osd num_output_layers={infer_meta.num_output_layers}")
        for layer_idx in range(infer_meta.num_output_layers):
                 layer = pyds.get_nvds_LayerInfo(infer_meta, layer_idx)
                 array = layer_tensor_to_ndarray(layer)
                 print(f"[osd] {layer.layerName} layer_info.buffer address = {type(array)}")
4

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.