Hi guys!
I am running a custom Darknet model, and there seems to be a bug in the NvDCF tracker causing it to randomly fail after a few frames. I have changed two primary files:
- Nvdsanalytics-test file: to use my model file, and dump the output to a file using
filesink - Custom Yolo bbox parser script: basically my model uses 4 masks instead of 3 so I had to add minor changes.
deepstream_nvdsanalytics_test.cpp file:
#include <gst/gst.h>
#include <glib.h>
#include <stdio.h>
#include <math.h>
#include <string.h>
#include <sys/time.h>
#include <iostream>
#include <fstream>
#include <vector>
#include <unordered_map>
#include "gstnvdsmeta.h"
#include "nvds_analytics_meta.h"
#ifndef PLATFORM_TEGRA
#include "gst-nvmessage.h"
#endif
#define MAX_DISPLAY_LEN 64
#define PGIE_CLASS_ID_PERSON 0
/* The muxer output resolution must be set if the input streams will be of
* different resolution. The muxer will scale all the input frames to this
* resolution. */
#define MUXER_OUTPUT_WIDTH 1920
#define MUXER_OUTPUT_HEIGHT 1080
/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set
* based on the fastest source's framerate. */
#define MUXER_BATCH_TIMEOUT_USEC 40000
#define TILED_OUTPUT_WIDTH 1920
#define TILED_OUTPUT_HEIGHT 1080
/* NVIDIA Decoder source pad memory feature. This feature signifies that source
* pads having this capability will push GstBuffers containing cuda buffers. */
#define GST_CAPS_FEATURES_NVMM "memory:NVMM"
#define OUTPUT_TEXT_FILE "output_file.txt"
#define MYMODEL_NET_CONFIG "model.txt"
gchar pgie_classes_str[1][32] = { "Person" };
/* nvdsanalytics_src_pad_buffer_probe will extract metadata received on tiler sink pad
* and extract nvanalytics metadata etc. */
static GstPadProbeReturn
nvdsanalytics_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info,
gpointer u_data)
{
std::fstream output_file;
output_file.open(OUTPUT_TEXT_FILE, std::fstream::out | std::fstream::app);
GstBuffer *buf = (GstBuffer *) info->data;
guint num_rects = 0;
NvDsObjectMeta *obj_meta = NULL;
guint person_count = 0;
NvDsMetaList * l_frame = NULL;
NvDsMetaList * l_obj = NULL;
guint lc_count = 0;
guint roi_count = 0;
bool overcrowding = false;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data);
for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
l_obj = l_obj->next) {
obj_meta = (NvDsObjectMeta *) (l_obj->data);
if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
person_count++;
num_rects++;
}
// Access attached user meta for each object
for (NvDsMetaList *l_user_meta = obj_meta->obj_user_meta_list; l_user_meta != NULL;
l_user_meta = l_user_meta->next) {
NvDsUserMeta *user_meta = (NvDsUserMeta *) (l_user_meta->data);
if(user_meta->base_meta.meta_type == NVDS_USER_OBJ_META_NVDSANALYTICS)
{
NvDsAnalyticsObjInfo * user_meta_data = (NvDsAnalyticsObjInfo *)user_meta->user_meta_data;
if (user_meta_data->dirStatus.length()){
g_print ("object %lu moving in %s\n", obj_meta->object_id, user_meta_data->dirStatus.c_str());
}
}
}
}
roi_count = 0;
lc_count = 0;
overcrowding = false;
/* Iterate user metadata in frames to search analytics metadata */
for (NvDsMetaList * l_user = frame_meta->frame_user_meta_list;
l_user != NULL; l_user = l_user->next) {
NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data;
if (user_meta->base_meta.meta_type != NVDS_USER_FRAME_META_NVDSANALYTICS)
continue;
/* convert to metadata */
NvDsAnalyticsFrameMeta *meta =
(NvDsAnalyticsFrameMeta *) user_meta->user_meta_data;
/* Get the labels from nvdsanalytics config file */
roi_count = meta->objInROIcnt["RF"];
lc_count = meta->objLCCumCnt["Exit"];
overcrowding = meta->ocStatus["OC"];
}
g_print ("Frame Number = %d of Stream = %d, Number of objects = %d "
"Person Count = %d Objs in ROI = %d LC count = %d Overcrowding = %d\n",
frame_meta->frame_num, frame_meta->pad_index,
num_rects, person_count, roi_count, lc_count,overcrowding);
}
return GST_PAD_PROBE_OK;
}
static gboolean
bus_call (GstBus * bus, GstMessage * msg, gpointer data)
{
GMainLoop *loop = (GMainLoop *) data;
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_EOS:
g_print ("End of stream\n");
g_main_loop_quit (loop);
break;
case GST_MESSAGE_WARNING:
{
gchar *debug;
GError *error;
gst_message_parse_warning (msg, &error, &debug);
g_printerr ("WARNING from element %s: %s\n",
GST_OBJECT_NAME (msg->src), error->message);
g_free (debug);
g_printerr ("Warning: %s\n", error->message);
g_error_free (error);
break;
}
case GST_MESSAGE_ERROR:
{
gchar *debug;
GError *error;
gst_message_parse_error (msg, &error, &debug);
g_printerr ("ERROR from element %s: %s\n",
GST_OBJECT_NAME (msg->src), error->message);
if (debug)
g_printerr ("Error details: %s\n", debug);
g_free (debug);
g_error_free (error);
g_main_loop_quit (loop);
break;
}
#ifndef PLATFORM_TEGRA
case GST_MESSAGE_ELEMENT:
{
if (gst_nvmessage_is_stream_eos (msg)) {
guint stream_id;
if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) {
g_print ("Got EOS from stream %d\n", stream_id);
}
}
break;
}
#endif
default:
break;
}
return TRUE;
}
static void
cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data)
{
g_print ("In cb_newpad\n");
GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad);
const GstStructure *str = gst_caps_get_structure (caps, 0);
const gchar *name = gst_structure_get_name (str);
GstElement *source_bin = (GstElement *) data;
GstCapsFeatures *features = gst_caps_get_features (caps, 0);
/* Need to check if the pad created by the decodebin is for video and not
* audio. */
if (!strncmp (name, "video", 5)) {
/* Link the decodebin pad only if decodebin has picked nvidia
* decoder plugin nvdec_*. We do this by checking if the pad caps contain
* NVMM memory features. */
if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) {
/* Get the source bin ghost pad */
GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src");
if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad),
decoder_src_pad)) {
g_printerr ("Failed to link decoder src pad to source bin ghost pad\n");
}
gst_object_unref (bin_ghost_pad);
} else {
g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n");
}
}
}
static void
decodebin_child_added (GstChildProxy * child_proxy, GObject * object,
gchar * name, gpointer user_data)
{
g_print ("Decodebin child added: %s\n", name);
if (g_strrstr (name, "decodebin") == name) {
g_signal_connect (G_OBJECT (object), "child-added",
G_CALLBACK (decodebin_child_added), user_data);
}
}
static GstElement *
create_source_bin (guint index, gchar * uri)
{
GstElement *bin = NULL, *uri_decode_bin = NULL;
gchar bin_name[16] = { };
g_snprintf (bin_name, 15, "source-bin-%02d", index);
/* Create a source GstBin to abstract this bin's content from the rest of the
* pipeline */
bin = gst_bin_new (bin_name);
/* Source element for reading from the uri.
* We will use decodebin and let it figure out the container format of the
* stream and the codec and plug the appropriate demux and decode plugins. */
uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin");
if (!bin || !uri_decode_bin) {
g_printerr ("One element in source bin could not be created.\n");
return NULL;
}
/* We set the input uri to the source element */
g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL);
/* Connect to the "pad-added" signal of the decodebin which generates a
* callback once a new pad for raw data has beed created by the decodebin */
g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added",
G_CALLBACK (cb_newpad), bin);
g_signal_connect (G_OBJECT (uri_decode_bin), "child-added",
G_CALLBACK (decodebin_child_added), bin);
gst_bin_add (GST_BIN (bin), uri_decode_bin);
/* We need to create a ghost pad for the source bin which will act as a proxy
* for the video decoder src pad. The ghost pad will not have a target right
* now. Once the decode bin creates the video decoder and generates the
* cb_newpad callback, we will set the ghost pad target to the video decoder
* src pad. */
if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src",
GST_PAD_SRC))) {
g_printerr ("Failed to add ghost pad in source bin\n");
return NULL;
}
return bin;
}
int
main (int argc, char *argv[])
{
GMainLoop *loop = NULL;
GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL,
*nvtracker = NULL, *nvdsanalytics = NULL, *nvvidconv2 = NULL, *encoder = NULL, *mux = NULL,
*nvvidconv = NULL, *nvosd = NULL, *tiler = NULL;
#ifdef PLATFORM_TEGRA
GstElement *transform = NULL;
#endif
GstBus *bus = NULL;
guint bus_watch_id;
GstPad *nvdsanalytics_src_pad = NULL;
guint i, num_sources;
guint tiler_rows, tiler_columns;
guint pgie_batch_size = 16;
/* Check input arguments */
if (argc < 2) {
g_printerr ("Usage: %s <uri1> [uri2] ... [uriN] \n", argv[0]);
return -1;
}
num_sources = argc - 1;
/* Standard GStreamer initialization */
gst_init (&argc, &argv);
loop = g_main_loop_new (NULL, FALSE);
/* Create gstreamer elements */
/* Create Pipeline element that will form a connection of other elements */
pipeline = gst_pipeline_new ("nvdsanalytics-test-pipeline");
/* Create nvstreammux instance to form batches from one or more sources. */
streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");
if (!pipeline || !streammux) {
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
gst_bin_add (GST_BIN (pipeline), streammux);
for (i = 0; i < num_sources; i++) {
GstPad *sinkpad, *srcpad;
gchar pad_name[16] = { };
GstElement *source_bin = create_source_bin (i, argv[i + 1]);
if (!source_bin) {
g_printerr ("Failed to create source bin. Exiting.\n");
return -1;
}
gst_bin_add (GST_BIN (pipeline), source_bin);
g_snprintf (pad_name, 15, "sink_%u", i);
sinkpad = gst_element_get_request_pad (streammux, pad_name);
if (!sinkpad) {
g_printerr ("Streammux request sink pad failed. Exiting.\n");
return -1;
}
srcpad = gst_element_get_static_pad (source_bin, "src");
if (!srcpad) {
g_printerr ("Failed to get src pad of source bin. Exiting.\n");
return -1;
}
if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) {
g_printerr ("Failed to link source bin to stream muxer. Exiting.\n");
return -1;
}
gst_object_unref (srcpad);
gst_object_unref (sinkpad);
}
/* Use nvinfer to infer on batched frame. */
pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine");
/* Use nvtracker to track detections on batched frame. */
nvtracker = gst_element_factory_make ("nvtracker", "nvtracker");
/* Use nvdsanalytics to perform analytics on object */
nvdsanalytics = gst_element_factory_make ("nvdsanalytics", "nvdsanalytics");
/* Use nvtiler to composite the batched frames into a 2D tiled array based
* on the source of the frames. */
tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler");
/* Use convertor to convert from NV12 to RGBA as required by nvosd */
nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
/* Create OSD to draw on the converted RGBA buffer */
nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay");
nvvidconv2 = gst_element_factory_make ("nvvideoconvert", "nvvideo-transform");
encoder = gst_element_factory_make ("x264enc", "h264-encoder");
mux = gst_element_factory_make ("qtmux", "mux");
/* Finally render the osd output */
#ifdef PLATFORM_TEGRA
transform = gst_element_factory_make ("nvegltransform", "nvegl-transform");
#endif
// sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
sink = gst_element_factory_make ("filesink", "nvvideo-store");
g_object_set (G_OBJECT (sink), "location", "output.mp4", NULL);
if (!pgie || !nvtracker || !nvdsanalytics || !tiler || !nvvidconv ||
!nvosd || !nvvidconv2 || !encoder || !mux || !sink) {
g_printerr ("One element could not be created. Exiting.\n");
return -1;
}
#ifdef PLATFORM_TEGRA
if(!transform) {
g_printerr ("One tegra element could not be created. Exiting.\n");
return -1;
}
#endif
g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height",
MUXER_OUTPUT_HEIGHT, "batch-size", num_sources,
"batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);
/* Configure the nvinfer element using the nvinfer config file. */
g_object_set (G_OBJECT (pgie),
"config-file-path", MYMODEL_NET_CONFIG , NULL);
/* Configure the nvtracker element for using the particular tracker algorithm. */
g_object_set (G_OBJECT (nvtracker),
"ll-lib-file", "/opt/nvidia/deepstream/deepstream-5.0/lib/libnvds_nvdcf.so",
"ll-config-file", "tracker.yml", "tracker-width", 640, "tracker-height", 480,
NULL);
/* Using KLT tracker works fine! Uncomment to test
g_object_set(G_OBJECT(nvtracker),
"ll-lib-file", "/opt/nvidia/deepstream/deepstream-5.0/lib/libnvds_mot_klt.so",
NULL);
*/
/* Configure the nvdsanalytics element for using the particular analytics config file*/
g_object_set (G_OBJECT (nvdsanalytics),
"config-file", "config_nvdsanalytics.txt",
NULL);
/* Override the batch-size set in the config file with the number of sources. */
g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL);
if (pgie_batch_size != num_sources) {
g_printerr
("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n",
pgie_batch_size, num_sources);
g_object_set (G_OBJECT (pgie), "batch-size", num_sources, NULL);
}
tiler_rows = (guint) sqrt (num_sources);
tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows);
/* we set the tiler properties here */
g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_columns,
"width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL);
/* we add a message handler */
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
gst_object_unref (bus);
/* Set up the pipeline */
/* we add all elements into the pipeline */
#ifdef PLATFORM_TEGRA
gst_bin_add_many (GST_BIN (pipeline), pgie, nvtracker, nvdsanalytics , tiler,
nvvidconv, nvosd, transform, sink,
NULL);
/* we link the elements together
* nvstreammux -> nvinfer -> nvtracker -> nvdsanalytics -> nvtiler ->
* nvvideoconvert -> nvosd -> transform -> sink
*/
if (!gst_element_link_many (streammux, pgie, nvtracker, nvdsanalytics, tiler,
nvvidconv, nvosd, transform, sink, NULL)) {
g_printerr ("Elements could not be linked. Exiting.\n");
return -1;
}
#else
gst_bin_add_many (GST_BIN (pipeline), pgie, nvtracker, nvdsanalytics, tiler,
nvvidconv, nvosd, nvvidconv2, encoder, mux, sink, NULL);
/* we link the elements together
* nvstreammux -> nvinfer -> nvtracker -> nvdsanalytics -> nvtiler ->
* nvvideoconvert -> nvosd -> sink
*/
if (!gst_element_link_many (streammux, pgie, nvtracker, nvdsanalytics,
tiler, nvvidconv, nvosd, nvvidconv2, encoder, mux, sink, NULL)) {
g_printerr ("Elements could not be linked. Exiting.\n");
return -1;
}
#endif
/* Lets add probe to get informed of the meta data generated, we add probe to
* the sink pad of the nvdsanalytics element, since by that time, the buffer
* would have had got all the metadata.
*/
nvdsanalytics_src_pad = gst_element_get_static_pad (nvdsanalytics, "src");
if (!nvdsanalytics_src_pad)
g_print ("Unable to get src pad\n");
else
gst_pad_add_probe (nvdsanalytics_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
nvdsanalytics_src_pad_buffer_probe, NULL, NULL);
/* Set the pipeline to "playing" state */
g_print ("Now playing:");
for (i = 0; i < num_sources; i++) {
g_print (" %s,", argv[i + 1]);
}
g_print ("\n");
gst_element_set_state (pipeline, GST_STATE_PLAYING);
/* Wait till pipeline encounters an error or EOS */
g_print ("Running...\n");
g_main_loop_run (loop);
/* Out of the main loop, clean up nicely */
g_print ("Returned, stopping playback\n");
gst_element_set_state (pipeline, GST_STATE_NULL);
g_print ("Deleting pipeline\n");
gst_object_unref (GST_OBJECT (pipeline));
g_source_remove (bus_watch_id);
g_main_loop_unref (loop);
return 0;
}
Output-
Now playing: file:///opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps/analytics_app/sample_1080p_h264.mp4,
gstnvtracker: Loading low-level lib at /opt/nvidia/deepstream/deepstream-5.0/lib/libnvds_nvdcf.so
gstnvtracker: Optional NvMOT_RemoveStreams not implemented
gstnvtracker: Batch processing is ON
[NvDCF] Initialized
Deserialize yoloLayerV3 plugin: yolo_168
Deserialize yoloLayerV3 plugin: yolo_176
WARNING: ../nvdsinfer/nvdsinfer_func_utils.cpp:34 [TRT]: Current optimization profile is: 0. Please ensure there are no enqueued operations pending in this context prior to switching profiles
0:00:02.684912813 4301 0x563f0b5c5200 INFO nvinfer gstnvinfer.cpp:602:gst_nvinfer_logger:<primary-nvinference-engine> NvDsInferContext[UID 1]: Info from NvDsInferContextImpl::deserializeEngineAndBackend() <nvdsinfer_context_impl.cpp:1577> [UID = 1]: deserialized trt engine from :/opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps/deepstream-nvdsanalytics-sagar/model_b1_gpu0_fp16.engine
INFO: ../nvdsinfer/nvdsinfer_model_builder.cpp:685 [Implicit Engine Info]: layers num: 3
0 INPUT kFLOAT data 3x608x608
1 OUTPUT kFLOAT yolo_168 24x38x38
2 OUTPUT kFLOAT yolo_176 24x76x76
0:00:02.685023281 4301 0x563f0b5c5200 INFO nvinfer gstnvinfer.cpp:602:gst_nvinfer_logger:<primary-nvinference-engine> NvDsInferContext[UID 1]: Info from NvDsInferContextImpl::generateBackendContext() <nvdsinfer_context_impl.cpp:1681> [UID = 1]: Use deserialized engine model: /opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps/deepstream-nvdsanalytics-sagar/model_b1_gpu0_fp16.engine
0:00:02.687534605 4301 0x563f0b5c5200 INFO nvinfer gstnvinfer_impl.cpp:311:notifyLoadModelStatus:<primary-nvinference-engine> [UID 1]: Load new model:model.txt sucessfully
Decodebin child added: source
Decodebin child added: decodebin0
Running...
Decodebin child added: qtdemux0
Decodebin child added: multiqueue0
Decodebin child added: h264parse0
Decodebin child added: capsfilter0
Decodebin child added: aacparse0
Decodebin child added: avdec_aac0
Decodebin child added: nvv4l2decoder0
In cb_newpad
In cb_newpad
Frame Number = 0 of Stream = 0, Number of objects = 0 Person Count = 0 Objs in ROI = 0 LC count = 0 Overcrowding = 0
Frame Number = 1 of Stream = 0, Number of objects = 0 Person Count = 0 Objs in ROI = 0 LC count = 0 Overcrowding = 0
Frame Number = 2 of Stream = 0, Number of objects = 0 Person Count = 0 Objs in ROI = 0 LC count = 0 Overcrowding = 0
Frame Number = 3 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 4 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 5 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 6 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 7 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 8 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 9 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
ERROR: nvdsinfer_context_impl.cpp:1448 Failed to synchronize on cuda copy-coplete-event, cuda err_no:700, err_str:cudaErrorIllegalAddress
0:00:04.449649163 4301 0x563f0b5c1e80 WARN nvinfer gstnvinfer.cpp:1983:gst_nvinfer_output_loop:<primary-nvinference-engine> error: Failed to dequeue output from inferencing. NvDsInferContext error: NVDSINFER_CUDA_ERROR
[ERROR] 2020-05-15 17:09:19 CUDA error 700 (cudaErrorIllegalAddress): an illegal memory access was encountered
0:00:04.449766604 4301 0x563f0b5c1e80 WARN nvinfer gstnvinfer.cpp:599:gst_nvinfer_logger:<primary-nvinference-engine> NvDsInferContext[UID 1]: Warning from NvDsInferContextImpl::releaseBatchOutput() <nvdsinfer_context_impl.cpp:1482> [UID = 1]: Tried to release an outputBatchID which is already with the context
~~ CLOG[src/modules/cuDCF/featureExtractor.cu, extractFeatures_VPI() @line 644]: VPI_STATUS_INTERNAL
ERROR from element primary-nvinference-engine: Failed to dequeue output from inferencing. NvDsInferContext error: NVDSINFER_CUDA_ERROR
Error details: gstnvinfer.cpp(1983): gst_nvinfer_output_loop (): /GstPipeline:nvdsanalytics-test-pipeline/GstNvInfer:primary-nvinference-engine
Cuda failure: status=700 in CreateTextureObj at line 2496
nvbufsurftransform.cpp(2369) : getLastCudaError() CUDA error : Recevied NvBufSurfTransformError_Execution_Error : (400) invalid resource handle.
Returned, stopping playback
!![Exception] [FAILED] vpiDeviceSync() Failed!
Cuda failure: status=46
nvbufsurface: Error(-1) in releasing cuda memory
[ERROR] 2020-05-15 17:09:19 CUDA error 46 (cudaErrorDevicesUnavailable): all CUDA-capable devices are busy or unavailable
[ERROR] 2020-05-15 17:09:19 Error destroying cuda device: VPI_STATUS_INTERNAL
(../inc/nvcudautils) Error ResourceError: e: all CUDA-capable devices are busy or unavailable (cudaErrorDevicesUnavailable) (propagating from /home/rlima/src/vpi/ext/nvcudautils/src/AllocMem.cpp, function freeMem(), line 283)
(../inc/nvcudautils) Error ResourceError: (propagating from /home/rlima/src/vpi/ext/nvcudautils/inc/nvcudautils/detail/../AllocMem.h, function operator()(), line 45)
(../inc/nvcudautils) Error ResourceError: e: all CUDA-capable devices are busy or unavailable (cudaErrorDevicesUnavailable) (propagating from /home/rlima/src/vpi/ext/nvcudautils/src/AllocMem.cpp, function freeMem(), line 283)
(../inc/nvcudautils) Error ResourceError: (propagating from /home/rlima/src/vpi/ext/nvcudautils/inc/nvcudautils/detail/../AllocMem.h, function operator()(), line 60)
(../inc/nvcudautils) Error ResourceError: e: all CUDA-capable devices are busy or unavailable (cudaErrorDevicesUnavailable) (propagating from /home/rlima/src/vpi/ext/nvcudautils/src/AllocMem.cpp, function freeMem(), line 283)
(../inc/nvcudautils) Error ResourceError: (propagating from /home/rlima/src/vpi/ext/nvcudautils/inc/nvcudautils/detail/../AllocMem.h, function operator()(), line 60)
Segmentation fault (core dumped)
Now, if I run the exact same file once again:
Now playing: file:///opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps/analytics_app/sample_1080p_h264.mp4,
gstnvtracker: Loading low-level lib at /opt/nvidia/deepstream/deepstream-5.0/lib/libnvds_nvdcf.so
gstnvtracker: Optional NvMOT_RemoveStreams not implemented
gstnvtracker: Batch processing is ON
[NvDCF] Initialized
Deserialize yoloLayerV3 plugin: yolo_168
Deserialize yoloLayerV3 plugin: yolo_176
WARNING: ../nvdsinfer/nvdsinfer_func_utils.cpp:34 [TRT]: Current optimization profile is: 0. Please ensure there are no enqueued operations pending in this context prior to switching profiles
0:00:02.779754233 13602 0x555791fa1200 INFO nvinfer gstnvinfer.cpp:602:gst_nvinfer_logger:<primary-nvinference-engine> NvDsInferContext[UID 1]: Info from NvDsInferContextImpl::deserializeEngineAndBackend() <nvdsinfer_context_impl.cpp:1577> [UID = 1]: deserialized trt engine from :/opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps/deepstream-nvdsanalytics-sagar/model_b1_gpu0_fp16.engine
INFO: ../nvdsinfer/nvdsinfer_model_builder.cpp:685 [Implicit Engine Info]: layers num: 3
0 INPUT kFLOAT data 3x608x608
1 OUTPUT kFLOAT yolo_168 24x38x38
2 OUTPUT kFLOAT yolo_176 24x76x76
0:00:02.779830049 13602 0x555791fa1200 INFO nvinfer gstnvinfer.cpp:602:gst_nvinfer_logger:<primary-nvinference-engine> NvDsInferContext[UID 1]: Info from NvDsInferContextImpl::generateBackendContext() <nvdsinfer_context_impl.cpp:1681> [UID = 1]: Use deserialized engine model: /opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps/deepstream-nvdsanalytics-sagar/model_b1_gpu0_fp16.engine
0:00:02.782137633 13602 0x555791fa1200 INFO nvinfer gstnvinfer_impl.cpp:311:notifyLoadModelStatus:<primary-nvinference-engine> [UID 1]: Load new model:model.txt sucessfully
Decodebin child added: source
Decodebin child added: decodebin0
Running...
Decodebin child added: qtdemux0
Decodebin child added: multiqueue0
Decodebin child added: h264parse0
Decodebin child added: capsfilter0
Decodebin child added: aacparse0
Decodebin child added: avdec_aac0
Decodebin child added: nvv4l2decoder0
In cb_newpad
In cb_newpad
Frame Number = 0 of Stream = 0, Number of objects = 0 Person Count = 0 Objs in ROI = 0 LC count = 0 Overcrowding = 0
Frame Number = 1 of Stream = 0, Number of objects = 0 Person Count = 0 Objs in ROI = 0 LC count = 0 Overcrowding = 0
Frame Number = 2 of Stream = 0, Number of objects = 0 Person Count = 0 Objs in ROI = 0 LC count = 0 Overcrowding = 0
Frame Number = 3 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 4 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 5 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 6 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 7 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 8 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 9 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 10 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 11 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 12 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 13 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
Frame Number = 14 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
Frame Number = 15 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
Frame Number = 16 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
Frame Number = 17 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
Frame Number = 18 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 19 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 20 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 21 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 22 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 23 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 24 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 25 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 26 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 27 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 28 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 29 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 30 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 31 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 32 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 33 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 34 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 35 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 36 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 37 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 38 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 39 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 40 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 41 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 42 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 43 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 44 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 45 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 46 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 47 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 48 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 49 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 50 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 51 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 52 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 53 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 54 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 55 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 56 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 57 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 58 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 59 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 60 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 61 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 62 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 63 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 64 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 65 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 66 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in North
object 2617188315236925440 moving in North
Frame Number = 67 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 68 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 69 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 0 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 70 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 71 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 72 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 73 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 74 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 75 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 76 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 77 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 78 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 79 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 80 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 81 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 82 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 83 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 84 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 85 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 86 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 87 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 88 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 89 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 90 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 91 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 92 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 93 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 94 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 95 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 96 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 97 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 98 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 99 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 100 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in North
Frame Number = 101 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 102 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 103 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 104 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 105 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 106 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925441 moving in South
object 2617188315236925440 moving in North
Frame Number = 107 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 108 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 109 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 110 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 111 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
Frame Number = 112 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
Frame Number = 113 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
Frame Number = 114 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
Frame Number = 115 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
Frame Number = 116 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
object 2617188315236925441 moving in South
Frame Number = 117 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
object 2617188315236925441 moving in South
Frame Number = 118 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
object 2617188315236925441 moving in South
Frame Number = 119 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
object 2617188315236925441 moving in South
Frame Number = 120 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in South
object 2617188315236925441 moving in South
Frame Number = 121 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 122 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
object 2617188315236925441 moving in South
Frame Number = 123 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 124 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 4 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 125 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 126 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 127 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 128 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 129 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 130 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 131 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 132 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 133 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 134 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 135 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 136 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 137 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 138 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 139 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 140 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 141 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 142 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 143 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 144 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 145 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 146 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 147 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 148 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 149 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 150 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 151 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 152 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
object 2617188315236925440 moving in North
Frame Number = 153 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 154 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 155 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 156 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 157 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 158 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 159 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 160 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 161 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 162 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 3 LC count = 1 Overcrowding = 0
Frame Number = 163 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 164 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 165 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 166 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 167 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 168 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 169 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 170 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 171 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 172 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 173 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 174 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 175 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 176 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 177 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 178 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 179 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 180 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 181 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 182 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 183 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 184 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 185 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 186 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 187 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 188 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 0 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 189 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 0 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 190 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 0 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 191 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 0 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 192 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 0 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 193 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 0 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 194 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 0 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 195 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 196 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 197 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 198 of Stream = 0, Number of objects = 5 Person Count = 5 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 199 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 200 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 201 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 202 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 203 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 204 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 205 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 206 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 207 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 208 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 209 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 210 of Stream = 0, Number of objects = 4 Person Count = 4 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 211 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 212 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 213 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 214 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 215 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 216 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 217 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 218 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 219 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 220 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 221 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 222 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 223 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 224 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 225 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 226 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 227 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 228 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 229 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 230 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 231 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 232 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 233 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 234 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 235 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 236 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 237 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 238 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 239 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 240 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 241 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 242 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 243 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 244 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 245 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 246 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 247 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 248 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 249 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 250 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 251 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 252 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 253 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 254 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 255 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 256 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 257 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 258 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 259 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 260 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 261 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 262 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 263 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 264 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 265 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 266 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 267 of Stream = 0, Number of objects = 1 Person Count = 1 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 268 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 269 of Stream = 0, Number of objects = 1 Person Count = 1 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 270 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 271 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 272 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 273 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 274 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 275 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 276 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 277 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 278 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 279 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 280 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 281 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 282 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 283 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 284 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 285 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 286 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 287 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 288 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 289 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 290 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 291 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 292 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 293 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 294 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 295 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 296 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 297 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 298 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 299 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 300 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 301 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 302 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 303 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 304 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 305 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 306 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 307 of Stream = 0, Number of objects = 1 Person Count = 1 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 308 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 309 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 310 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 311 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 312 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 313 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 314 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 315 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 316 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 317 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 318 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 319 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 320 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in North
Frame Number = 321 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 322 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 323 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
object 2617188315236925447 moving in South
Frame Number = 324 of Stream = 0, Number of objects = 3 Person Count = 3 Objs in ROI = 2 LC count = 1 Overcrowding = 0
Frame Number = 325 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 326 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 327 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 328 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 329 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 330 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 331 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 332 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 333 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 334 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 335 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 336 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 337 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 338 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 339 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 340 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 341 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 342 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 343 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 344 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 345 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 346 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Frame Number = 347 of Stream = 0, Number of objects = 2 Person Count = 2 Objs in ROI = 1 LC count = 1 Overcrowding = 0
Cuda failure: status=700 in CreateTextureObj at line 2496
nvbufsurftransform.cpp(2369) : getLastCudaError() CUDA error : Recevied NvBufSurfTransformError_Execution_Error : (700) an illegal memory access was encountered.
[ERROR] 2020-05-15 17:14:57 CUDA error 700 (cudaErrorIllegalAddress): an illegal memory access was encountered
~~ CLOG[src/modules/cuDCF/featureExtractor.cu, extractFeatures_VPI() @line 644]: VPI_STATUS_INTERNAL
!![Exception] [FAILED] vpiDeviceSync() Failed!
[ERROR] 2020-05-15 17:14:57 CUDA error 46 (cudaErrorDevicesUnavailable): all CUDA-capable devices are busy or unavailable
[ERROR] 2020-05-15 17:14:57 Error destroying cuda device: VPI_STATUS_INTERNAL
(../inc/nvcudautils) Error ResourceError: e: all CUDA-capable devices are busy or unavailable (cudaErrorDevicesUnavailable) (propagating from /home/rlima/src/vpi/ext/nvcudautils/src/AllocMem.cpp, function freeMem(), line 283)
(../inc/nvcudautils) Error ResourceError: (propagating from /home/rlima/src/vpi/ext/nvcudautils/inc/nvcudautils/detail/../AllocMem.h, function operator()(), line 45)
(../inc/nvcudautils) Error ResourceError: e: all CUDA-capable devices are busy or unavailable (cudaErrorDevicesUnavailable) (propagating from /home/rlima/src/vpi/ext/nvcudautils/src/AllocMem.cpp, function freeMem(), line 283)
(../inc/nvcudautils) Error ResourceError: (propagating from /home/rlima/src/vpi/ext/nvcudautils/inc/nvcudautils/detail/../AllocMem.h, function operator()(), line 60)
(../inc/nvcudautils) Error ResourceError: e: all CUDA-capable devices are busy or unavailable (cudaErrorDevicesUnavailable) (propagating from /home/rlima/src/vpi/ext/nvcudautils/src/AllocMem.cpp, function freeMem(), line 283)
(../inc/nvcudautils) Error ResourceError: (propagating from /home/rlima/src/vpi/ext/nvcudautils/inc/nvcudautils/detail/../AllocMem.h, function operator()(), line 60)
Cuda failure: status=46 in CreateTextureObj at line 2513
Segmentation fault (core dumped)
This time it ran longer. A couple of things to highlight:
- The object IDs are weird, like
2617188315236925447 - Just before it stops working, the object IDs are not detected and then it fails with an unrelated error.
- Most importantly, if I replace NvDCF with KLT tracker, everything works fine
- If I disable tracking and analytics, and simply run my model for object detection, everything runs fine. This means the problem is not my custom model.
- I tried it on a RTX 2070 machine, and the problem still persists, it still randomly fails but with lesser probability. Sometimes it works fine.
Would like to have your advice. Thanks in advance.
• Hardware Platform: T4 and RTX 2070
• DeepStream Version: 5.0
• TensorRT Version: 7.0
• NVIDIA GPU Driver Version (valid for GPU only): 440.33.01