Deepstream 7 on Jetson, PGIE + 3SGIE, save vehicle information

Accelerated Computing Intelligent Video Analytics DeepStream SDK
, , ,
1

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson Orin NX16)
• DeepStream Version 7
• JetPack Version 6 36.3

I’m working on ainvr app, i’ve deployed deepstream 7 with yolov8s as PGIE and lpd/LPRnet/VehicleMakeNet as SGIEs, my question is, how can i save the information of the license plate and vehicle make in one vehicle objects, instead of seperate ones:
image
The above screenshot shows information for the same vehicle, but they’re saved in seperate objects.
Here’s deepstream-test5 app code, i’ve changed the parts where the data is being saved:


static void
generate_event_msg_meta(AppCtx *appCtx, gpointer data, gint class_id, gboolean useTs,
                        GstClockTime ts, gchar *src_uri, gint stream_id, guint sensor_id,
                        NvDsObjectMeta *obj_params, float scaleW, float scaleH,
                        NvDsFrameMeta *frame_meta)
{
    NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *)data;
    GstClockTime ts_generated = 0;

    meta->objType = NVDS_OBJECT_TYPE_UNKNOWN; /**< object unknown */
    /* The sensor_id is parsed from the source group name which has the format
     * [source<sensor-id>]. */
    meta->sensorId = sensor_id;
    meta->placeId = sensor_id;
    meta->moduleId = sensor_id;
    meta->frameId = frame_meta->frame_num;
    meta->ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1);
    meta->objectId = (gchar *)g_malloc0(MAX_LABEL_SIZE);
    strncpy(meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE);

    /** INFO: This API is called once for every 30 frames (now) */
    if ((useTs && src_uri) || appCtx->config.source_attr_all_config.type == NV_DS_SOURCE_IPC)
    {
        ts_generated =
            generate_ts_rfc3339_from_ts(meta->ts, MAX_TIME_STAMP_LEN, ts, src_uri,
                                        stream_id);
    }
    else
    {
        generate_ts_rfc3339(meta->ts, MAX_TIME_STAMP_LEN);
    }

    meta->bbox.left = obj_params->rect_params.left * scaleW;
    meta->bbox.top = obj_params->rect_params.top * scaleH;
    meta->bbox.width = obj_params->rect_params.width * scaleW;
    meta->bbox.height = obj_params->rect_params.height * scaleH;

    /** tracking ID */
    meta->trackingId = obj_params->object_id;

    /** sensor ID when streams are added using nvmultiurisrcbin REST API */
    NvDsSensorInfo *sensorInfo = get_sensor_info(appCtx, stream_id);
    if (sensorInfo)
    {
        /** this stream was added using REST API; we have Sensor Info! */
        LOGD("this stream [%d:%s] was added using REST API; we have Sensor Info\n",
             sensorInfo->source_id, sensorInfo->sensor_id);
        meta->sensorStr = g_strdup(sensorInfo->sensor_id);
    }

    (void)ts_generated;

    if (model_used == APP_CONFIG_ANALYTICS_YOLOV8S_PGIE_3SGIE)
    {

        if ((class_id == LPD_CLASS_ID && (strcmp(obj_params->obj_label, "") == 0 || obj_params->obj_label == NULL)) ||
            class_id == 2 || class_id == 3 || class_id == 5 || class_id == 7)

        {
            meta->type = NVDS_EVENT_MOVING;
            meta->objType = NVDS_OBJECT_TYPE_VEHICLE;
            meta->objClassId = RESNET10_PGIE_3SGIE_TYPE_COLOR_MAKECLASS_ID_CAR;

            NvDsVehicleObject *obj =
                (NvDsVehicleObject *)g_malloc0(sizeof(NvDsVehicleObject));
            schema_fill_sample_sgie_vehicle_metadata(obj_params, obj);

            meta->extMsg = obj;
            meta->extMsgSize = sizeof(NvDsVehicleObject);
        }

    }
}


static void
schema_fill_sample_sgie_vehicle_metadata(NvDsObjectMeta *obj_params,
                                         NvDsVehicleObject *obj)
{
    if (!obj_params || !obj)
    {
        return;
    }

    /** The JSON obj->classification, say type, color, or make
     * according to the schema shall have null (unknown)
     * classifications (if the corresponding sgie failed to provide a label)
     */
    obj->type = NULL;
    obj->make = NULL;
    obj->model = NULL;
    obj->color = NULL;
    obj->license = NULL;
    obj->region = NULL;

    GList *l;
    for (l = obj_params->classifier_meta_list; l != NULL; l = l->next)
    {
        NvDsClassifierMeta *classifierMeta = (NvDsClassifierMeta *)(l->data);
        g_print("Component id (SGIE) %d\n", classifierMeta->unique_component_id);

        switch (classifierMeta->unique_component_id)
        {
        case SECONDARY_GIE_VEHICLE_TYPE_UNIQUE_ID:
            obj->type = get_first_result_label(classifierMeta);
            break;
        case SECONDARY_GIE_VEHICLE_LICENSE_PLATE_UNIQUE_ID:
            obj->license = get_first_result_label(classifierMeta);
            break;
        case SECONDARY_GIE_VEHICLE_MAKE_UNIQUE_ID:
            obj->make = get_first_result_label(classifierMeta);
            break;
        case SECONDARY_GIE_VEHICLE_COLOR_UNIQUE_ID:
            obj->color = get_first_result_label(classifierMeta);
            break;
        default:
            break;
        }
    }
}

Here’s my config:

[primary-gie]
enable=1
gpu-id=0
gie-unique-id=1
nvbuf-memory-type=4
config-file=config_infer_primary_yoloV8_nx16.txt
model-engine-file=/yolov8s/model_b4_gpu0_int8.engine
batch-size=4
bbox-border-color0=1;0;0;1
bbox-border-color1=0;1;1;1
bbox-border-color2=0;0;1;1
bbox-border-color3=0;1;0;1
interval=0

[tracker]
enable=1
tracker-width=960
tracker-height=544
ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
ll-config-file=config_tracker_NvDCF_PNv2.6_Interval_1_PVA.yml;config_tracker_NvDCF_PNv2.6_Interval_1_PVA.yml
sub-batches=2:2
gpu-id=0
display-tracking-id=1

[secondary-gie0]
enable=1
gpu-id=0
batch-size=4
gie-unique-id=4
operate-on-gie-id=1
config-file=lpd_yolov4-tiny_us.txt

[secondary-gie1]
enable=1
gpu-id=0
batch-size=4
gie-unique-id=5
operate-on-gie-id=4
operate-on-class-ids=0;
config-file=lpr_config_sgie_us.txt

[secondary-gie2]
enable=1
gpu-id=0
batch-size=4
gie-unique-id=6
operate-on-gie-id=1
config-file=config_infer_secondary_vehicleMake.txt

This code saves the data, but i need it to be in the same objects, is there a way to do that?

I’ve also tried to change the configuration of LPD model to operate on VehicleMake model, so i would have yolov8s (PGIE) → VehicleMakeNet (SGIE0) → LPDnet (SGIE1) → LPRnet (SGIE2) , but i’m not having any output from LPDnet and LPRnet, here are my configs for the sgies:

LPDnet config

[property]
gpu-id=0
net-scale-factor=0.0039215697906911373
#offsets=103.939;116.779;123.68
model-color-format=0
labelfile-path=/yolov8s/yolov4/usa_lpd_label.txt
model-engine-file=/yolov8s/model/LPDNet_usa_pruned_tao5.onnx_b40_gpu0_int8.engine
onnx-file=/yolov8s/model/LPDNet_usa_pruned_tao5.onnx
int8-calib-file=/yolov8s/model/usa_cal_8.5.3.bin
tlt-model-key=nvidia_tlt
infer-dims=3;480;640
uff-input-dims=3;480;640;0
uff-input-order=0
uff-input-blob-name=input_1
batch-size=16
## 0=FP32, 1=INT8, 2=FP16 mode
#process-mode=1
network-mode=1
num-detected-classes=1
interval=0
gie-unique-id=1
network-type=0
operate-on-gie-id=1
#operate-on-class-ids=0
cluster-mode=3
output-blob-names=output_cov/Sigmoid;output_bbox/BiasAdd
input-object-min-height=30
input-object-min-width=40


[class-attrs-all]
pre-cluster-threshold=0.01
roi-top-offset=0
roi-bottom-offset=0
detected-min-w=0
detected-min-h=0
detected-max-w=0
detected-max-h=0

VehicleMakeNet config

[property]
gpu-id=0
net-scale-factor=1
tlt-model-key=tlt_encode
tlt-encoded-model=/opt/nvidia/deepstream/deepstream-7.0/samples/models/Secondary_VehicleMake/resnet18_vehiclemakenet.etlt
model-engine-file=/yolov8s/Secondary_VehicleMake/resnet18_vehiclemakenet.etlt_b40_gpu0_int8.engine
labelfile-path=/yolov8s/Secondary_VehicleMake/labels.txt
int8-calib-file=/opt/nvidia/deepstream/deepstream-7.0/samples/models/Secondary_VehicleMake/cal_trt.bin
force-implicit-batch-dim=1
batch-size=4
# 0=FP32 and 1=INT8 mode
network-mode=1
network-type=1
#input-object-min-width=64
#input-object-min-height=64
model-color-format=0
gpu-id=0
gie-unique-id=2
operate-on-gie-id=1
#operate-on-class-ids=0
#is-classifier=1
uff-input-blob-name=input_1
output-blob-names=predictions/Softmax
classifier-async-mode=1
classifier-threshold=0.5
process-mode=2
#scaling-filter=0
#scaling-compute-hw=0
infer-dims=3;224;224

So is it possible to make this pipeline work?

3

Where did you put the “generate_event_msg_meta()” in your pipeline?

For the SGIE detector, you can get the “parent” in NvDsObjectMeta before nvvideoconvert and nvmultistreamtiler. Then you can get which PGIE object is related the the SGIE object.

4

For the generate_event_msg_meta, i put it in the bbox_generated_probe_after_analytics function, i left it as it was by default in deepstream-test5 app

static void
bbox_generated_probe_after_analytics(AppCtx *appCtx, GstBuffer *buf,
                                     NvDsBatchMeta *batch_meta, guint index)
{
    NvDsObjectMeta *obj_meta = NULL;
    GstClockTime buffer_pts = 0;
    guint32 stream_id = 0;

    for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL;
         l_frame = l_frame->next)
    {
        NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data;
        stream_id = frame_meta->source_id;
        GstClockTime buf_ntp_time = 0;
        if (playback_utc == FALSE)
        {
            StreamSourceInfo *src_stream = &testAppCtx->streams[stream_id];
            buf_ntp_time = frame_meta->ntp_timestamp;

            if (buf_ntp_time < src_stream->last_ntp_time)
            {
                GST_WARNING("Source %d: NTP timestamps are backward in time."
                            " Current: %lu previous: %lu \n",
                            stream_id, buf_ntp_time, src_stream->last_ntp_time);
            }
            src_stream->last_ntp_time = buf_ntp_time;
        }

        GList *l;
        for (l = frame_meta->obj_meta_list; l != NULL; l = l->next)
        {

            obj_meta = (NvDsObjectMeta *)(l->data);

            {
                float scaleW = 0;
                float scaleH = 0;

                buffer_pts = frame_meta->buf_pts;
                if (!appCtx->config.streammux_config.pipeline_width || !appCtx->config.streammux_config.pipeline_height)
                {
                    g_print("invalid pipeline params\n");
                    return;
                }
                LOGD("stream %d==%d [%d X %d]\n", frame_meta->source_id,
                     frame_meta->pad_index, frame_meta->source_frame_width,
                     frame_meta->source_frame_height);
                scaleW =
                    (float)frame_meta->source_frame_width /
                    appCtx->config.streammux_config.pipeline_width;
                scaleH =
                    (float)frame_meta->source_frame_height /
                    appCtx->config.streammux_config.pipeline_height;

                if (playback_utc == FALSE)
                {
                    buffer_pts = buf_ntp_time;
                }
                NvDsEventMsgMeta *msg_meta =
                    (NvDsEventMsgMeta *)g_malloc0(sizeof(NvDsEventMsgMeta));
                generate_event_msg_meta(appCtx, msg_meta, obj_meta->class_id, TRUE,

                                        buffer_pts,
                                        appCtx->config.multi_source_config[stream_id].uri, stream_id,
                                        appCtx->config.multi_source_config[stream_id].camera_id,
                                        obj_meta, scaleW, scaleH, frame_meta);
                testAppCtx->streams[stream_id].meta_number++;
                NvDsUserMeta *user_event_meta =
                    nvds_acquire_user_meta_from_pool(batch_meta);
                if (user_event_meta)
                {
                    user_event_meta->user_meta_data = (void *)msg_meta;
                    user_event_meta->base_meta.batch_meta = batch_meta;
                    user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META;
                    user_event_meta->base_meta.copy_func =
                        (NvDsMetaCopyFunc)meta_copy_func;
                    user_event_meta->base_meta.release_func =
                        (NvDsMetaReleaseFunc)meta_free_func;
                    nvds_add_user_meta_to_frame(frame_meta, user_event_meta);
                }
                else
                {
                    g_print("Error in attaching event meta to buffer\n");
                }
            }
        }
        testAppCtx->streams[stream_id].frameCount++;
    }
}

For the Lpd SGIE, it’s not taking it into consideration at all, i can only see the result from VehicleTypeNet, nothing from LPD (that’s when i put it after VehicleTypeNet model)

5

Do you mean you can’t get the LPD object meta in bbox_generated_probe_after_analytics?

Have you checked the all the " unique_component_id" of the object meta you got in bbox_generated_probe_after_analytics?

6

yes, i’m printing the component id each time, it neither prints the lpd id nor the lpr id, but when i added vehicle types, and vehicle make it did print them without problems, also i tried those two and made them operate on pgie resuts (operate_on_gie_id = 1) like i did with the LPD model, so now i have:
YoloV8s (PGIE) → LPD (SGIE 0) → LPR (SGIE1)
YoloV8s (PGIE) → VehicleMake (SGIE2)
YoloV8s (PGIE) → VehicleType (SGIE1)
i can get the results from VehicleMake and vehicleTypes in one Vehicle object (as i want it to be) , but the result from LPD comes alone, not even in a VehicleObject, as if it’s a result from PGIE model, although i’m testing if the class_id from PGIE is (car, truck, bike…) to save the result in vehicle object, as shown in the screenshot below:

image
image1102×501 28.9 KB

7

LPD is a detector, it output lpd objects. You can get its related vehicle object by the method I mentioned in Deepstream 7 on Jetson, PGIE + 3SGIE, save vehicle information - #3 by Fiona.Chen

I don’t know what your code is doing. But all the objects have their own classes. If you want to save the class id of the PGIE objects, you can use the “” to filter out the PGIE objects.

E.G.
As this deepstream-test5-app configuration, the PGIE unique_component_id is set as 1, the LPD SGIE unique_component_id is set as 4, the other SGIEs are all classifiers which do not output any objects, then we will get object meta with unique_component_id 1 and unique_component_id 4.

test_config.txt (4.7 KB)

In the bbox_generated_probe_after_analytics() function in /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5/deepstream_test5_app_main.c, we can add the following code to get the LPD output object meta only, by these LPD object meta, we can also get the related vehicle object meta, then we get the whol information for the LPD objects.

......
GList *l;
    for (l = frame_meta->obj_meta_list; l != NULL; l = l->next) {
      /* Now using above information we need to form a text that should
       * be displayed on top of the bounding box, so lets form it here. */

      obj_meta = (NvDsObjectMeta *) (l->data);
+      if(obj_meta->unique_component_id == 4) {
+        //it is LPD object
+        if(obj_meta->parent) {
+          //The related Vehicle object meta
+          NvDsObjectMeta *parent_obj_meta = (NvDsObjectMeta *)obj_meta->parent;
+          g_print("parent obj unique id %d\n", parent_obj_meta->unique_component_id);
+        }
+    }

     {
        /**
         * Enable only if this callback is after tiler
         * NOTE: Scaling back code-commented
         * now that bbox_generated_probe_after_analytics() is post analytics
         * (say pgie, tracker or sgie)
         * and before tiler, no plugin shall scale metadata and will be
         * corresponding to the nvstreammux resolution
         */
        float scaleW = 0;
......
8

My problem is, i’m saving a vehicle object only when PGIE detects the classes (2, 3, 5, 7), when those classes are detected, schema_fill_sample_sgie_vehicle_metadata function is called to save the type, make, license, color of the vehicle, this does work for vehicle typenet, and vehicle makenet models, the function is executed and in redis i get an object as follows:
|id|coord x1|coord x2|coord y1|coord y2|Vehicle|||type|make|
So i can see that it’s accessing the two sgie models, but here i also have LPD model, it’s not accessed at all when the classes (2, 3, 5, 7) from PGIE are detected, i’m printing the component ids of the SGIEs, 6 and 7 are printed (for VehicleType and VehicleMake) but 4 and 5 aren’t printed.
I tried something else as i noticed the LPD object is saved in redis, but not as Vehicle object. I tried to add another condition to save as vehicle as shown in the code portion below:

if(class_id == LPD_CLASS_ID && (strcmp(obj_params->obj_label, "") == 0 || obj_params->obj_label == NULL)) ||
 (class_id == 2 || class_id == 3 || class_id == 5 || class_id == 7)

This is to check if it detected the LPD class id (0) and that the label is empty (i’m not showing the label LPD) so it enters the condition and executes schema_fill_sample_sgie_vehicle_metadata function, here, the LPR SGIE component id is printed (5) and an object is saved as follows:
|id|coord x1|coord x2|coord y1|coord y2|Vehicle||licenseplate|||
So in total i’d have for the same vehicle, two objects saved, one containing the license plate, and one containing the type and the make. however, i’d like to have only one object.

image

Maybe there’s something i’m doing wrong in my LPD config file?

9

No. Please use LPD object as the start of the filterring.

When you get the lpd object, you will also get its parent vehicle object, you can check the vehicle object class id, if the class id is what you want, you can use this lpd object+vehicle object to generate the message you want to send. It is not a good idea to start with the vehicle objects. All the objects meta are available, you just need to design a reasonable way to get the information you want.

10 
......
GList *l;
    for (l = frame_meta->obj_meta_list; l != NULL; l = l->next) {
      /* Now using above information we need to form a text that should
       * be displayed on top of the bounding box, so lets form it here. */

      obj_meta = (NvDsObjectMeta *) (l->data);
+      if(obj_meta->unique_component_id == 4) {
+        //it is LPD object
+        if(obj_meta->parent) {
+          //The related Vehicle object meta
+          NvDsObjectMeta *parent_obj_meta = (NvDsObjectMeta *)obj_meta->parent;
+          g_print("parent obj unique id %d\n", parent_obj_meta->unique_component_id);
+          if (parent_obj_meta->class_id == 2 || parent_obj_meta->class_id == 3 || parent_obj_meta->class_id == 5 || parent_obj_meta->class_id == 7) {
+             //generate the cloud message here
+               ......
+           }
+        }
+    }

     {
        /**
         * Enable only if this callback is after tiler
         * NOTE: Scaling back code-commented
         * now that bbox_generated_probe_after_analytics() is post analytics
         * (say pgie, tracker or sgie)
         * and before tiler, no plugin shall scale metadata and will be
         * corresponding to the nvstreammux resolution
         */
        float scaleW = 0;
......
11

This solution allows me to save the lpd result in a Vehicle object, but this is not the mail issue, the issue is, that the results from VehicleType and VehicleMake come together (as they have the same parent gie with id 1) but the lpd one even though it has the same parent as the other two (PGIE with is 1) it comes seperately, and here when i check the component ids i only get two ids (1 for the PGIE and 4 for LPD), and when i print the parent id for LPD it gives 1:

GList *l;
    for (l = frame_meta->obj_meta_list; l != NULL; l = l->next) {
      /* Now using above information we need to form a text that should
       * be displayed on top of the bounding box, so lets form it here. */

        obj_meta = (NvDsObjectMeta *) (l->data);
        g_print("obj unique id %d\n", obj_meta->unique_component_id);
       //here it prints 1 and then 4 (No 6 , 7)
+      if(obj_meta->unique_component_id == 4) {
+        //it is LPD object
+        if(obj_meta->parent) {
+          //The related Vehicle object meta
+          NvDsObjectMeta *parent_obj_meta = (NvDsObjectMeta *)obj_meta->parent;
+          g_print("parent obj unique id %d\n", parent_obj_meta->unique_component_id);
+          if (parent_obj_meta->class_id == 2 || parent_obj_meta->class_id == 3 || parent_obj_meta->class_id == 5 || parent_obj_meta->class_id == 7) {
+             //generate the cloud message here
+               ......
+           }
+        }
+    }

     {
        /**
         * Enable only if this callback is after tiler
         * NOTE: Scaling back code-commented
         * now that bbox_generated_probe_after_analytics() is post analytics
         * (say pgie, tracker or sgie)
         * and before tiler, no plugin shall scale metadata and will be
         * corresponding to the nvstreammux resolution
         */
        float scaleW = 0;
......

so is it considering two pipelines or something? it’s very confusing

12

There are only two detectors in your app, so only two “unique_component_id” will appear in the detected object meta. LPD is a SGIE but not PGIE, so LPD 's object meta contains the “unique_component_id 4”, while the parent is the Vehicle detector’s “unique_component_id 1”. It is correct.

13

ok, i understand now, thanks for the clarification