Xavier NX add new object in dsexample

[yangyi]

Hello everyone and professors:

I meet a problem about embedding Deesptream into Xavier NX. I have created a topic in deepstream board, but I haven’t received the response. So I come here, and hope someone help me.

In deepstream, I add a new object in gstdsexample. The code as below:
NvDsObjectMeta *my_obj_meta = nvds_acquire_obj_meta_from_pool (batch_meta);
my_obj_meta->unique_component_id = 1;
my_obj_meta->confidence = 1;
my_obj_meta->class_id = 5;
my_obj_meta->object_id = 100;
NvOSD_RectParams & set_rect_params = my_obj_meta->rect_params;
set_rect_params.left = area_left;
set_rect_params.top = area_top;
set_rect_params.width = area_width;
set_rect_params.height = area_height;
set_rect_params.has_bg_color = 0;
set_rect_params.border_width = 3;
my_obj_meta->detector_bbox_info.org_bbox_coords.left = set_rect_params.left;
my_obj_meta->detector_bbox_info.org_bbox_coords.top = set_rect_params.top;
my_obj_meta->detector_bbox_info.org_bbox_coords.width = set_rect_params.width;
my_obj_meta->detector_bbox_info.org_bbox_coords.height = set_rect_params.height;
nvds_add_obj_meta_to_frame (frame_meta, my_obj_meta, NULL);

and I use deepstream-app.c->process_meta
static void
process_meta (AppCtx * appCtx, NvDsBatchMeta * batch_meta)
{

if (appCtx->pipeline.common_elements.secondary_gie_bin.bin)
{
    GstPad *my_pad = gst_element_get_static_pad (appCtx->pipeline.common_elements.secondary_gie_bin.bin, "src");
    if (!my_pad)
        NVGSTDS_ERR_MSG_V ("Unable to get SGIE src pad\n");
    else
    {
        gst_pad_add_probe (my_pad, GST_PAD_PROBE_TYPE_BUFFER,
        sgie_pad_buffer_probe, NULL, NULL);
        gst_object_unref (my_pad);
    }
}

}

static GstPadProbeReturn
sgie_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data)
{
NvDsBatchMeta batch_meta = gst_buffer_get_nvds_batch_meta (GST_BUFFER (info->data));
/ Iterate each frame metadata in batch */
for(NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next)
{
printf(“========================frame list========================\n”);
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) l_frame->data;
for(NvDsMetaList * l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next)
{
printf(“---------------------list obj--------------------------\n”);
NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) l_obj->data;
printf(“l_obj id = %d\n”, obj_meta->class_id);

  //---- show the classifer result label-------//
  for(NvDsMetaList *l = obj_meta->classifier_meta_list; l != NULL; l = l->next)
  {
     printf("-------classifer_list-----\n");
     NvDsClassifierMeta * classifierMeta = (NvDsClassifierMeta *) (l->data);
     for (NvDsMetaList * n = classifierMeta->label_info_list; n != NULL; n = n->next)
     {
         NvDsLabelInfo *labelInfo = (NvDsLabelInfo*) (n->data);
         printf("label: %s \n ", &labelInfo->result_label[0]);
     }
  }
  //------------ show the raw data  --------//
  /* Iterate user metadata in object to search SGIE's tensor data */
  for (NvDsMetaList * l_user = obj_meta->obj_user_meta_list; l_user != NULL;  l_user = l_user->next)
  {
    printf("*****user_meta_list*****\n");
    NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data;
    if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META)
      continue;
  }
}

}
return GST_PAD_PROBE_OK;
}

The up code works fine in GPU. however, in Jetson, It likes the object can not be recognized by SGIE. My configure as below:
[application]
enable-perf-measurement=1
perf-measurement-interval-sec=5

[ds-example]
enable=1
processing-width=1024
processing-height=768
full-frame=0
unique-id=12
gpu-id=0
blur-objects=0
nvbuf-memory-type=3

[tiled-display]
enable=1
rows=1
columns=1
width=1920
height=1080
gpu-id=0
nvbuf-memory-type=0

[source0]
enable=1
type=3
uri=file:/home/d219/yangyi_Data/HuaFeiSavePicture/coal_less.mp4
num-sources=1
gpu-id=0
cudadec-memtype=0

[source1]
enable=0
type=4
uri=rtsp://admin:root1234@192.168.1.61
num-sources=1
gpu-id=0
cudadec-memtype=0

[sink0]
enable=1
type=4
codec=1
enc-type=0
sync=0
#iframeinterval=10
bitrate=400000
profile=0
rtsp-port=8559
udp-port=5410

[sink1]
enable=1
#Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming
type=3
#1=mp4 2=mkv
container=1
#1=h264 2=h265
codec=1
#encoder type 0=Hardware 1=Software
enc-type=0
sync=0
#iframeinterval=10
bitrate=2000000
#H264 Profile - 0=Baseline 2=Main 4=High
#H265 Profile - 0=Main 1=Main10
profile=0
output-file=out.mp4
source-id=0

[osd]
enable=1
gpu-id=0
border-width=1
text-size=15
text-color=1;1;1;1;
text-bg-color=0.3;0.3;0.3;1
font=Serif
show-clock=0
clock-x-offset=800
clock-y-offset=820
clock-text-size=12
clock-color=1;0;0;0
nvbuf-memory-type=0

[streammux]
gpu-id=0
live-source=0
buffer-pool-size=4
batch-size=4
batched-push-timeout=40000
width=1920
height=1080
enable-padding=0
nvbuf-memory-type=0

[primary-gie]
enable=1
gpu-id=0
gie-unique-id=1
nvbuf-memory-type=0
config-file=config_infer_primary_yoloV5.txt

[secondary-gie0]
enable=1
batch-size=1
gpu-id=0
gie-unique-id=2
operate-on-gie-id=1
operate-on-class-ids=5
config-file=./config_SGIE.txt

[tracker]
enable=1
tracker-width=640
tracker-height=384
ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
ll-config-file=config_tracker_NvDCF_perf.yml
gpu-id=0
enable-batch-process=1
enable-past-frame=1
display-tracking-id=1

[tests]
file-loop=0

PGIE config:
[property]
gpu-id=0
net-scale-factor=0.0039215697906911373
model-color-format=0
custom-network-config=yolov5_belt_best.cfg
model-file=yolov5_belt_best.wts
model-engine-file=model_b1_gpu0_fp32.engine
labelfile-path=labels.txt
batch-size=1
network-mode=0
num-detected-classes=1
interval=0
gie-unique-id=1
process-mode=1
network-type=0
cluster-mode=4
maintain-aspect-ratio=1
parse-bbox-func-name=NvDsInferParseYolo
custom-lib-path=nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so
engine-create-func-name=NvDsInferYoloCudaEngineGet

[class-attrs-all]
pre-cluster-threshold=0

SGIE config:
[property]
gpu-id=0
net-scale-factor=0.0039215697906911373
#offsets=97.6980; 99.9760; 88.8259

labelfile-path=./SGIElabels.txt
onnx-file=./ResNet18_without_softmax_1000_200.onnx
model-engine-file=./ResNet18_without_softmax_1000_200.onnx_b1_gpu0_fp16.engine
batch-size=1
infer-dims = 3;200;1000
uff-input-order = 0

force-implicit-batch-dim=0

network-mode=2 ## 0=FP32, 1=INT8, 2=FP16 mode
is-classifier=1

input-object-min-width = 50
input-object-min-height = 50
process-mode=2
#network-type: 100 output the tensor; 1:classifier
network-type=1
num-detected-classes=3
secondary-reinfer-interval=0
model-color-format=1
gie-unique-id=3
operate-on-gie-id=1
operate-on-class-ids=5
output-blob-names=predictions/Softmax
classifier-async-mode=0
classifier-threshold=0
output-tensor-meta=1
classifier-type=carcolor
#scaling-filter=0
#scaling-compute-hw=0

[class-attrs-all]
pre-cluster-threshold=0.25

As you can see, the object id =5000, is the added object. However, It can not be find by SGIE.

This problem makes me in big trouble server days. I desire someone could help me. Thank you very much.

[yuweiw]

Why do you set the nvbuf-memory-type=3 in your gstexample?
I try to build your environment on my device.So I need to know more details like:
How do you run the app, deepstream-app -c configfile?
Where did you add the sgie_pad_buffer_probe function?
Could you just attach your code file, video stream, and all the config files instead of just the code segment?

[yangyi]

Thank you very much! just now I find my problem.
(1) The nvbuf-memory-type=0.
(2) I add the new object in dsexample. But my pipeline is default. I checke it as : PGIE->Tracker->SGIE->Dsexample. So I modify the pipeline as
PGIE->dsexample->Tracker->SGIE. Hence, under this pipeline I get my answer.

(3) I add the "sgie_pad_buffer_probe " in the deepstream-app.c, the point to load this function is in process-meta function.

Thank you very very very much.