Cropping a stream and uploading RTP stream

[pdboef]

Hi,

We have the following use case: we use DeepStream to detect certain objects from a live source.
Then, when the object is detected, we would like to store a crop of the region around the object for further processing outside of DeepStream.
We have used the sample redaction_with_deepstream_app as a starting point, modifying the callback of the OSD plugin to implement cropping for the relevant objects.
Simultaneously, we want the ability to view the detections using an RTP stream. However, we face problems using the following pipeline:

v4l2src ! nvv4l2decoder ! nvvideoconvert ! ‘video/x-raw(NVMM), format=NV12’ ! nvstreammux ! queue ! nvinfer ! nvmultistream ! nvvideoconvert ! osd ! queue ! nvvideoconvert ! nvv4l2h264enc ! rtph264pay ! udpsink

One or two of the following errors occur, but only if an object is actually detected (thereby triggering the cropping operation):

• Segmentation fault (core dumped)
• Cuda failure: status=101
  nvbufsurface: Error(-1) in releasing cuda memory
  Segmentation fault (core dumped)
• free invalid pointer

To crop the image, we create a new NvBufSurface object and then crop to it from the GstBuffer using NvBufSurfTransform.
We have a few questions:
1. Are we using the correct plugins, in the correct order, to set up an RTP stream like this? Or will the RTP stream be “corrupted” because it includes custom message data (the crop).
2. Are we doing the cropping in the right way, especially with regards to allocating and freeing memory for the surfaces?

The code for cropping the image is as follows:

NvBufSurface *crop_image(GstBuffer *buffer, NvOSD_RectParams *crop_rect_params) {
    NvBufSurface *src_surface = NULL;
    NvBufSurface *dst_surface = NULL;
    NvBufSurfTransformRect src_rect;
    NvBufSurfTransformRect dst_rect;
    GstMapInfo in_map_info;
    NvBufSurfaceCreateParams create_params;
    NvBufSurfTransformParams transform_params;
    memset(&in_map_info, 0, sizeof(in_map_info));
    NvBufSurfTransform_Error transform_error;

    if (!gst_buffer_map(buffer, &in_map_info, (GstMapFlags) (GST_MAP_READ | GST_MAP_WRITE))) {
        g_printerr("Failed to map gst buffer.\n");
        return NULL;
    }

    src_surface = (NvBufSurface *) in_map_info.data;

    //TODO set to write class id
    if (1) {
        if (settings) {
            int new_left = GST_ROUND_UP_2(crop_rect_params->left -settings->crop_margin);
            if (new_left < 0) {
                new_left = 0;
            }
            crop_rect_params->left = new_left;

            int new_top = GST_ROUND_UP_2(crop_rect_params->top -settings->crop_margin);
            if (new_top < 0) {
                new_top = 0;
            }
            crop_rect_params->top = new_top;

            int new_width = GST_ROUND_DOWN_2(crop_rect_params->width + 2 * settings->crop_margin);
            if (new_left + new_width > 1280) {
                new_width = 1280 - new_left;
            }
            crop_rect_params->width = new_width;

            int new_height = GST_ROUND_DOWN_2(crop_rect_params->height + 2 * settings->crop_margin);
            if (new_left + new_width > 1280) {
                new_height = 1280 - new_top;
            }
            crop_rect_params->height = new_height;
        } else {
            crop_rect_params->left = GST_ROUND_UP_2(crop_rect_params->left);
            crop_rect_params->top = GST_ROUND_UP_2(crop_rect_params->top);
            crop_rect_params->width = GST_ROUND_DOWN_2(crop_rect_params->width);
            crop_rect_params->height = GST_ROUND_DOWN_2(crop_rect_params->height);
        }

        src_rect.left = crop_rect_params->left;
        src_rect.top = crop_rect_params->top;
        src_rect.width = crop_rect_params->width;
        src_rect.height = crop_rect_params->height;

        dst_rect = {0, 0, (guint) src_rect.width, (guint) src_rect.height};

        create_params.gpuId = 0;
        create_params.width = src_rect.width;
        create_params.height = src_rect.height;
        create_params.size = 0;
        create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
        create_params.layout = NVBUF_LAYOUT_PITCH;
        create_params.memType = NVBUF_MEM_CUDA_PINNED;

        NvBufSurfaceCreate(&dst_surface, 1, &create_params);

        dst_rect.left = 0;
        dst_rect.top = 0;
        dst_rect.width = src_rect.width;
        dst_rect.height = src_rect.height;

        // Set the transform parameters
        transform_params.src_rect = &src_rect;
        transform_params.dst_rect = &dst_rect;
        transform_params.transform_flag = NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_CROP_DST;
        transform_params.transform_filter = NvBufSurfTransformInter_Default;

        transform_error = NvBufSurfTransform(src_surface, dst_surface, &transform_params);
        //TODO: NvBufSurfTransformError_Succes
        if (transform_error != 0) {
            g_printerr("NvBufSurfaceTransform failed with error %d while converting buffer.\n", transform_error);
        }
    }
    gst_buffer_unmap(buffer, &in_map_info);
    return dst_surface;
}

We also use a copy and free func:

static gpointer meta_copy_func (gpointer data, gpointer user_data)
{
    NvDsUserMeta *user_meta = (NvDsUserMeta *) data;
    NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data;
    NvDsEventMsgMeta *dstMeta = NULL;

    dstMeta = (NvDsEventMsgMeta *) g_memdup (srcMeta, sizeof(CustomObject));

    if (srcMeta->ts)
        dstMeta->ts = g_strdup (srcMeta->ts);

    if(srcMeta->objClassId)
        dstMeta->objClassId = srcMeta->objClassId;

    if (srcMeta->extMsgSize > 0) {
        // Allocate extra space in the message for custom message data
        if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) {
            CustomObject *srcObj = (CustomObject *) srcMeta->extMsg;
            CustomObject *obj = (CustomObject *) g_malloc0 (sizeof (CustomObject));
            if (srcObj->frame)
                obj->frame = srcObj->frame;
            if (srcObj->vehicleObject) {
                NvDsVehicleObject * newVehicleObj = (NvDsVehicleObject *) g_malloc0 (sizeof (NvDsVehicleObject));
                obj->vehicleObject = newVehicleObj;
                if (srcObj->vehicleObject->license)
                    obj->vehicleObject->license = g_strdup(srcObj->vehicleObject->license);
            }
            dstMeta->extMsg = obj;
            dstMeta->extMsgSize = sizeof (NvDsVehicleObject);
        }
    }
    if (user_meta->user_meta_data)
        g_free(user_meta->user_meta_data);
    return dstMeta;
}

static void meta_free_func (gpointer data, gpointer user_data)
{
    NvDsUserMeta *user_meta = (NvDsUserMeta *) data;
    NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data;

    if (srcMeta->ts)
        g_free (srcMeta->ts);

    if(srcMeta->objClassId)
        g_free (srcMeta->objectId);

    // Free custom message data
    if (srcMeta->extMsgSize > 0) {
        if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) {
            CustomObject *obj = (CustomObject *) srcMeta->extMsg;
            if (obj->frame)
                NvBufSurfaceDestroy(obj->frame);
            if (obj->vehicleObject) {
                NvDsVehicleObject * vehicleObject = (NvDsVehicleObject *) obj->vehicleObject;
                if (vehicleObject->license)
                    g_free (vehicleObject->license);
                g_free(obj->vehicleObject);
            }
        }
        g_free (srcMeta->extMsg);
        srcMeta->extMsgSize = 0;
    }
    if (user_meta->user_meta_data)
        g_free (user_meta->user_meta_data);
    user_meta->user_meta_data = NULL;
}

[DaneLLL]

Hi,
We would suggest you use dsexample plugin. The code is in

deepstream_sdk_v4.0.2_jetson\sources\gst-plugins\gst-dsexample

Please check README to compile/install the plugin.

There are samples of customization, please refer to 2, 5 in FAQ

[pdboef]

Thanks for your suggestion. We are trying this out now and will post the solution here if we find it.

[pdboef]

We have managed to crop the images and get the RTP stream to work.
For this, instead of using the dsexample plugin, we modified the nvmsgconv.cpp source file to add the cropping to custom metadata directly. It can retrieve the frame buffer because we store a reference to it in an earlier OSD plugin. Using a preprocessor flag, the code can run on both Tegra and non-Tegra hardware. I have included the relevant part of our modifications to nvmsgconv.cpp below.

/*
 * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
 *
 * NVIDIA Corporation and its licensors retain all intellectual property
 * and proprietary rights in and to this software, related documentation
 * and any modifications thereto.  Any use, reproduction, disclosure or
 * distribution of this software and related documentation without an express
 * license agreement from NVIDIA Corporation is strictly prohibited.
 *
 */

#include "nvmsgconv.h"
#include <json-glib/json-glib.h>
#include <uuid.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <cstring>
#include <vector>
#include <unordered_map>
#include "customobject.h"
#include <cuda_runtime_api.h>
#include <cuda.h>

using namespace std;

unsigned int *array_pointer = NULL;

int crop_to_frame_index(int x, int y, int frame_width, int crop_left, int crop_top) {
    return (crop_top + y) * frame_width + crop_left + x;
}

/**
 * Generates the message object from the current context.
 * @param ctx is a pointer to an NvDsMsg2pCtx object which holds the current context
 * @param meta is a pointer to an NvDsEventMsgMeta which holds the meta data such as the object type of the detection
 * @return a pointer to a JsonObject which holds the message
 */
static JsonObject *
generate_object_object(NvDsMsg2pCtx *ctx, NvDsEventMsgMeta *meta) {
    JsonObject *objectObj;
    JsonObject *jobject;
    guint i;
    gchar tracking_id[64];

    // object object
    objectObj = json_object_new();
    if (snprintf(tracking_id, sizeof(tracking_id), "%d", meta->trackingId) >= (int) sizeof(tracking_id))
        g_warning("Not enough space to copy trackingId");
    json_object_set_string_member(objectObj, "id", tracking_id);

    switch (meta->objType) {
        case NVDS_OBJECT_TYPE_VEHICLE:
            // vehicle sub object
            jobject = json_object_new();

            if (meta->extMsgSize) {
                CustomObject *customObject = (CustomObject *) meta->extMsg;
                NvDsVehicleObject *dsObj = (NvDsVehicleObject *) customObject->vehicleObject;
                if (dsObj) {
                    json_object_set_string_member(jobject, "license", dsObj->license);
                    json_object_set_double_member(jobject, "confidence", meta->confidence);
                    if (customObject->buffer) {
                        GstMapInfo in_map_info;
                        NvBufSurface *frame = NULL;

                        memset (&in_map_info, 0, sizeof (in_map_info));
                        if (!gst_buffer_map (customObject->buffer, &in_map_info, GST_MAP_READ)) {
                            g_print ("Error: Failed to map gst buffer\n");
                            break;
                        }

                        frame = (NvBufSurface *) in_map_info.data;

#ifdef PLATFORM_TEGRA
                        NvBufSurfaceMap(frame, 0, 0, NVBUF_MAP_READ);
                        NvBufSurfaceSyncForCpu(frame, 0, 0);
                        array_pointer = (unsigned int *) frame->surfaceList[0].mappedAddr.addr[0];
#else
                        if(array_pointer == NULL) {
                            g_print("Error: failed to malloc array_pointer \n");
                        }
                        cudaMemcpy(
                                (char *) array_pointer,
                                (char *) frame->surfaceList[0].dataPtr,
                                frame->surfaceList[0].dataSize,
                                cudaMemcpyDeviceToHost
                        );
#endif
                        int left = meta->bbox.left;
                        int top = meta->bbox.top;
                        int width = meta->bbox.width;
                        int height = meta->bbox.height;
                        int pitch_width = frame->surfaceList[0].pitch / 4;
                        int length = width * height;
                        JsonArray *array = json_array_sized_new(length);
                        for (int j = 0; j < height; j++) {
                            for (int i = 0; i < width; i++) {
                                int index_in_surface = crop_to_frame_index(i, j, pitch_width, left, top);
                                json_array_add_int_element(array, array_pointer[index_in_surface]);
                            }
                        }
                        json_object_set_array_member(jobject, "frame", array);
                        json_object_set_int_member(jobject, "width", width);
                        json_object_set_int_member(jobject, "height", height);
#ifdef PLATFORM_TEGRA
                        NvBufSurfaceUnMap(frame, 0, 0);
#endif
                        gst_buffer_unmap(customObject->buffer, &in_map_info);
                    }
                }
            } else {
                // No vehicle object in meta data. Attach empty vehicle sub object.
                json_object_set_string_member(jobject, "license", "no vehicle detected");
                json_object_set_double_member(jobject, "confidence", 1.0);
            }
            json_object_set_object_member(objectObj, "vehicle", jobject);
            break;
        case NVDS_OBJECT_TYPE_UNKNOWN:
            if (!meta->objectId) {
                break;
            }
            /** No information to add; object type unknown within NvDsEventMsgMeta */
            jobject = json_object_new();
            json_object_set_object_member(objectObj, meta->objectId, jobject);
            break;
        default:
            cout << "Object type not implemented" << endl;
    }

    // bbox sub object
    jobject = json_object_new();
    json_object_set_int_member(jobject, "topleftx", meta->bbox.left);
    json_object_set_int_member(jobject, "toplefty", meta->bbox.top);
    json_object_set_int_member(jobject, "bottomrightx", meta->bbox.left + meta->bbox.width);
    json_object_set_int_member(jobject, "bottomrighty", meta->bbox.top + meta->bbox.height);
    json_object_set_object_member(objectObj, "bbox", jobject);

    // signature sub array
    if (meta->objSignature.size) {
        JsonArray *jArray = json_array_sized_new(meta->objSignature.size);

        for (i = 0; i < meta->objSignature.size; i++) {
            json_array_add_double_element(jArray, meta->objSignature.signature[i]);
        }
        json_object_set_array_member(objectObj, "signature", jArray);
    }

    return objectObj;
}