Regarding dbouts output FPENet algorithm

• Hardware Platform (Jetson / GPU) -Jetson AGX
• DeepStream Version - 6.4
• JetPack Version (valid for Jetson only) - 6.0+b106
• TensorRT Version - 8.6.2
• NVIDIA GPU Driver Version (valid for GPU only) - 12.2
• Issue Type( questions, new requirements, bugs) - we are using FPENet algorithm for facial landmark extraction and we used primary model for face detection as facenet and seconday model used for FPENet and we can able to extract 80 landmark point and not sure about that particular detected face landmark so need to give solution for this

def tiler_sink_pad_buffer_probe(pad, info, u_data):
global attendace_data, prediction, frame_no, obj_id_confidences, previous_obj_id, obj_id, count

gst_buffer = info.get_buffer()
if not gst_buffer:
    print("Unable to get GstBuffer")
    return Gst.PadProbeReturn.OK

# Retrieve batch metadata from the buffer
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list

while l_frame is not None:
    try:
        frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
    except StopIteration:
        break

    obj_source_id = frame_meta.source_id
    l_obj = frame_meta.obj_meta_list
    frame_no += 1

    # Get frame surface and convert to numpy array
    n_frame = pyds.get_nvds_buf_surface(hash(gst_buffer), frame_meta.batch_id)
    frame = np.array(n_frame, copy=True, order='C')
    bgr_frame = cv2.cvtColor(frame, cv2.COLOR_RGBA2BGR)

    while l_obj is not None:
        try:
            obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
        except StopIteration:
            break

        # Process objects from the primary model
        if obj_meta.unique_component_id == 1:
            obj_id, confidence = obj_meta.object_id, obj_meta.confidence

            x, y = obj_meta.rect_params.left, obj_meta.rect_params.top
            bbox_width, bbox_height = obj_meta.rect_params.width, obj_meta.rect_params.height

            x1, y1 = max(int(x - padding), 0), max(int(y - padding), 0)
            x2, y2 = int(x + bbox_width + padding), int(y + bbox_height + padding)

            if obj_id not in obj_id_confidences:
                obj_id_confidences[obj_id] = {'confidences': [confidence], 'exec': False, 'uuid': uuid4().hex}
            else:
                obj_id_confidences[obj_id]["confidences"].append(confidence)

            current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

            if confidence >= 0.4 and not obj_id_confidences[obj_id]['exec']:
                current_confidence = obj_id_confidences[obj_id]['confidences'][-1]
                img_file_name = f"{obj_id_confidences[obj_id]['uuid']}_{round(confidence, 2)}_{current_time}.jpg"

                # Crop and convert image for saving
                cropped_image = bgr_frame[y1:y2, x1:x2]
                rgb_image = cv2.cvtColor(cropped_image, cv2.COLOR_BGR2RGB)

                # Access the secondary model output (landmarks)
                l_user_meta = obj_meta.obj_user_meta_list
                while l_user_meta is not None:
                    try:
                        user_meta = pyds.NvDsUserMeta.cast(l_user_meta.data)
                    except StopIteration:
                        break
                    if user_meta and user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META:
                        try:
                            tensor_meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
                        except StopIteration:
                            break

                        # Extract landmarks from tensor output
                        for i in range(tensor_meta.num_output_layers):
                            layer = pyds.get_nvds_LayerInfo(tensor_meta, i)
                            if layer is not None and layer.buffer:
                                layer_name = layer.layerName
                                if layer_name == "conv_keypoints_m80":
                                    output = []
                                    # Assuming buffer contains float32 data
                                    for j in range(80):  # Extract 80 keypoints
                                        x = pyds.get_detections(layer.buffer, j * 2)   # X-coordinate
                                        y = pyds.get_detections(layer.buffer, j * 2 + 1)  # Y-coordinate
                                        if x is not None and y is not None:
                                            output.append((x, y))  # Store as tuple of (x, y) coordinates

                                    print(f"Extracted {len(output)} landmarks: {output}")
                                    
                                    # Draw circles on landmarks
                                    for (x, y) in output:
                                        x = int(x)
                                        y = int(y)
                                        cv2.circle(rgb_image, (x, y), 5, (0, 255, 0), -1)  # Draw a green circle

                                    # Save the image with landmarks
                                    img_file_path = f"landmark_img/{count}_{img_file_name}"
                                    cv2.imwrite(img_file_path, rgb_image)
                                    count += 1
                                    print(f"Saved image with landmarks to {img_file_path}")
                    try:
                        l_user_meta = l_user_meta.next
                    except StopIteration:
                        break

        try:
            l_obj = l_obj.next
        except StopIteration:
            break

    try:
        l_frame = l_frame.next
    except StopIteration:
        break

return Gst.PadProbeReturn.OK

give right way to implement triton server on jetson orin with cuda
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)

Thank you for sharing the code. Have you met any issue? Do you have any question for us?

we can able to get output of the model but not sure about exact output of layer which it gives 80 landmark
1.my first question is that how to give input detected face image to second model as input for getting landmark
2.second one is that i would like to know about output layer of the fpenet
like below mentioned

  • N X 2 keypoint locations
  • N X 1 keypoint confidence

N is the number of keypoints. It can have a value of 68, 80, or 104.

0 INPUT kFLOAT input_face_images 1x80x80
1 OUTPUT kFLOAT conv_keypoints_m80 80x80x80
2 OUTPUT kFLOAT softargmax 80x2
3 OUTPUT kFLOAT softargmax:1 80

can you tell us about this output layer and which one should we use?

Extracted 80 landmarks: [(-312.65728759765625, -134.0671844482422), (-403.81781005859375, -188.78839111328125), (-406.3051452636719, -318.9999694824219), (-386.28216552734375, -267.7610168457031), (-383.7948303222656, -302.3348693847656), (-457.4197082519531, -261.2939758300781), (-465.75225830078125, -364.51806640625), (-497.4656982421875, -256.4436950683594), (-454.8080139160156, -379.9394836425781), (-527.5623779296875, -283.4311828613281), (-484.03411865234375, -413.2696533203125), (-557.0371704101562, -326.33758544921875), (-484.2828674316406, -388.6451416015625), (-580.293701171875, -315.3933410644531), (-469.6076354980469, -360.538330078125), (-540.8695678710938, -301.83740234375), (-462.1456298828125, -352.45452880859375), (-542.7350463867188, -265.1493225097656), (-396.2314758300781, -221.24801635742188), (-387.0283508300781, -284.55047607421875), (-354.94183349609375, -237.16690063476562), (-442.6201171875, -218.76068115234375), (-385.03851318359375, -289.525146484375), (-323.72589111328125, -258.0604553222656), (-374.3429870605469, -233.4359130859375), (-294.8728942871094, -249.7279052734375), (-416.6275634765625, -219.755615234375), (-308.05572509765625, -261.5426940917969), (-413.5184020996094, -219.13377380371094), (-372.3531188964844, -264.65185546875), (-275.720458984375, -240.89788818359375), (-323.8502502441406, -290.0226135253906), (-371.1094665527344, -316.263916015625), (-370.1145324707031, -339.8935241699219), (-412.1503601074219, -314.0252990722656), (-375.0892028808594, -306.5633239746094), (-385.4115905761719, -322.6065979003906), (-350.091552734375, -264.2787780761719), (-418.7417907714844, -281.6900634765625), (-354.5687561035156, -313.1547546386719), (-385.03851318359375, -45.6427116394043), (-547.2122192382812, -235.42576599121094), (-573.7022705078125, -92.6531982421875), (-562.8823852539062, -263.15948486328125), (-533.6563110351562, -81.33585357666016), (-698.6904296875, -253.2101593017578), (-611.8827514648438, -123.99351501464844), (-727.4190673828125, -379.5663757324219), (-583.5272216796875, -146.25509643554688), (-792.4627075195312, -302.7079772949219), (-684.0151977539062, -168.0192108154297), (-778.5336303710938, -432.9195556640625), (-741.5968627929688, -139.7880401611328), (-805.5211791992188, -427.4474182128906), (-789.1047973632812, -149.86172485351562), (-722.320068359375, -466.2497253417969), (-927.027099609375, -149.7373504638672), (-794.576904296875, -336.5356140136719), (-840.5924682617188, -148.74241638183594), (-691.1041259765625, -384.5410461425781), (-773.93212890625, -162.04962158203125), (-648.0733642578125, -421.8509521484375), (-795.19873046875, -184.06246948242188), (-622.080810546875, -353.07635498046875), (-559.8975830078125, -131.82859802246094), (-639.8651733398438, -316.0151672363281), (-568.3544921875, -164.16384887695312), (-598.9486083984375, -317.507568359375), (-547.5853271484375, -123.12295532226562), (-507.7880859375, -301.7130432128906), (-590.8648071289062, -238.16183471679688), (-511.14599609375, -294.99725341796875), (-553.8036499023438, -105.96039581298828), (-471.59747314453125, -191.40008544921875), (-434.1632080078125, -243.6339569091797), (-464.0111389160156, -193.51431274414062), (-476.3233947753906, -98.6227798461914), (-462.0212707519531, -165.03440856933594), (-529.4278564453125, -176.60047912597656), (-448.7140808105469, -152.72215270996094)]

Please refer to the deepstream-faciallandmark-app and deepstream-emotion-app samples.

The deepstream-faciallandmark-app sample is for the FPENet only. The deepstream-emotion-app sample use the output faciallandmark coordinates as the input of the EmotionNet model’s inputs for inferencing.

ok thank you

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.