Getting Metadata keypoints from FPENet on python deepstream

Accelerated Computing Intelligent Video Analytics DeepStream SDK
1

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) Nvidia RTX4080
• DeepStream Version 7.0
• TensorRT Version TensorRT v8601
• NVIDIA GPU Driver Version (valid for GPU only) Driver Version: 535.183.01

I try to get the landmark of an image in deepstream with a sgie, and I get only a number not an array. Here is my configuration file and python probe:

[property]
gpu-id=0
model-engine-file=landmarks/faciallandmark.etlt_b32_gpu0_int8.engine
tlt-model-key=nvidia_tlt
tlt-encoded-model=landmarks/model.etlt
int8-calib-file=landmarks/int8_calibration.txt
#dynamic batch size
batch-size=32

0=FP32, 1=INT8, 2=FP16 mode

network-mode=1
num-detected-classes=1
output-blob-names=softargmax;softargmax:1;conv_keypoints_m80
#0=Detection 1=Classifier 2=Segmentation 100=other
network-type=100

Enable tensor metadata output

output-tensor-meta=1
#1-Primary 2-Secondary
process-mode=2
gie-unique-id=2
operate-on-gie-id=1
operate-on-class-ids=2
net-scale-factor=1.0
offsets=0.0
input-object-min-width=5
input-object-min-height=5
#0=RGB 1=BGR 2=GRAY
model-color-format=2

[class-attrs-all]
threshold=0.0

def sgie_pad_buffer_probe(pad, info, u_data):
gst_buffer = info.get_buffer()
if not gst_buffer:
print("Unable to get GstBuffer ")
return
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))

if not batch_meta:
    return Gst.PadProbeReturn.OK

l_frame = batch_meta.frame_meta_list

while l_frame is not None:
    frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
    l_obj = frame_meta.obj_meta_list

    while l_obj is not None:
        obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)

        if not obj_meta:
            l_obj = l_obj.next
            continue

        l_user = obj_meta.obj_user_meta_list

        while l_user is not None:
            user_meta = pyds.NvDsUserMeta.cast(l_user.data)

            if user_meta.base_meta.meta_type != pyds.NVDSINFER_TENSOR_OUTPUT_META:
                l_user = l_user.next
                continue

            meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
            heatmap_data = None
            confidence = None

            # softargmax
            # softargmax:1
            # conv_keypoints_m80


            for i in range(meta.num_output_layers):
                layer_info = pyds.get_nvds_LayerInfo(meta, i)
                #layer_info.buffer = meta.out_buf_ptrs_host
                print(type(layer_info.layerName), layer_info.layerName, pyds.get_ptr(layer_info.buffer))
                if layer_info.layerName == "softargmax":
                    heatmap_data = pyds.get_ptr(layer_info.buffer)
                    print(heatmap_data, 'heatmap_data')
                elif layer_info.layerName == "softargmax:1":
                    confidence = pyds.get_ptr(layer_info.buffer)
                    print(confidence, 'confidence')

            if heatmap_data is not None:
                heatmap_array = np.ctypeslib.as_array(heatmap_data, shape=(meta.num_output_layers,))
                print(heatmap_array,'heatmap_array')
                # marks = [{"x": mark[0], "y": mark[1]} for mark in heatmap_array]

                # if not nvds_add_facemark_meta(batch_meta, obj_meta, marks, confidence):
                #     print("Failed to add facemark metadata")

            l_user = l_user.next

        l_obj = l_obj.next

    l_frame = l_frame.next

return Gst.PadProbeReturn.OK

Constants for facial landmark indices

LEFT_EYE_MARKS_START_INDEX = 42
LEFT_EYE_MARKS_END_INDEX = 47
RIGHT_EYE_MARKS_START_INDEX = 36
RIGHT_EYE_MARKS_END_INDEX = 41

Define metadata type

NVDS_USER_RIVA_META_FACEMARK = 0x12345678

3

Have you tested this native sample? It should run correctly.

It seems that you did not enter the shape parameter correctly. Refer to the code below and modify it according to the model output

    import ctypes
    if box_layer.dataType == pyds.NvDsInferDataType.FLOAT:
        # print(f"int_addr {type(pyds.get_ptr(box_layer.buffer))}") <class 'int'>
        addr = pyds.get_ptr(box_layer.buffer)
        # print(f"addr {type(addr)}") <class 'ctypes.c_int'>
        data_ptr = ctypes.cast(addr, ctypes.POINTER(ctypes.c_float))
        num_dims = box_layer.inferDims.numDims
        shape = []
        for i in range(num_dims):
            shape.append(box_layer.inferDims.d[i])
        # print(f"{shape}")
        box_array = np.ctypeslib.as_array(data_ptr, shape=shape)
        # print(f"box_array {type(box_array)}")
        boxes = np.frombuffer(box_array, dtype=np.float32)
        for i in range(num_detection):
            top = clip(boxes[i * 4], 0.0, 1.0)
            left = clip(boxes[i * 4 + 1], 0.0, 1.0)
            height = clip(boxes[i * 4 + 2], 0.0, 1.0) - top
            width = clip(boxes[i * 4 + 3], 0.0, 1.0) - left
            # if box_size_param.is_percentage_sufficiant(height, width):
            print(f"len {len(boxes)} boxes {i} == left {left} top {top} width {width} height {height}")