Please provide complete information as applicable to your setup.
• Hardware Platform (Jetson / GPU)
Jetson
• DeepStream Version
6.1 and 6.2
• JetPack Version (valid for Jetson only)
5.0.2 and 5.1
• NVIDIA GPU Driver Version (valid for GPU only)
• Issue Type( questions, new requirements, bugs)
bugs
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
Hi, I have a sample app that uses multifilesrc, jpegparse, and nvv4l2decoder to process a folder of images. This version of the app works on DeepStream 5.0 and 6.0, but it shows the following errors on DeepStream 6.1 and 6.2:
[NvTiler::CheckVICMaxScaleFactorCompatibility] ERROR: Scaling requested from [4096 X 2160] to [458315037 X -180471528] exceeds max scaling factor (16.000000) supported by VIC
[NvTiler::CheckVICMaxScaleFactorCompatibility] Please use a larger tiled window resolution or reduce number of streams
0:00:03.799784567 10583 0xaaab0213d300 WARN nvinfer gstnvinfer.cpp:2369:gst_nvinfer_output_loop:<primary-nvinference-engine> error: Internal data stream error.
0:00:03.799807928 10583 0xaaab0213d300 WARN nvinfer gstnvinfer.cpp:2369:gst_nvinfer_output_loop:<primary-nvinference-engine> error: streaming stopped, reason error (-5)
nvstreammux: Successfully handled EOS for source_id=65535
the code:
/*
* Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
// #include <gst/gst.h>
// #include <glib.h>
#ifndef PLATFORM_TEGRA
#include "gst-nvmessage.h"
#endif
#include "gstnvdsmeta.h"
#include <array>
#include <boost/property_tree/ini_parser.hpp>
#include <boost/property_tree/ptree.hpp>
#include <cstdlib>
#include <dirent.h>
#include <glib.h>
#include <glob.h>
#include <gst/gst.h>
#include <iostream>
#include <iterator>
#include <list>
#include <math.h>
#include <memory>
#include <stdio.h>
#include <string.h>
#include <string>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <vector>
#include <thread>
#include <boost/algorithm/string.hpp>
// #include <memory>
std::string input_path;
#ifndef PLATFORM_TEGRA
#include "gst-nvmessage.h"
#endif
#define MAX_DISPLAY_LEN 64
double intersection_threshold;
int big_counter = 0;
/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set
* based on the fastest source's framerate. */
#define MUXER_BATCH_TIMEOUT_USEC 40000
// #define MEMORY_FEATURES "memory:NVMM"
#include <cuda_runtime_api.h>
#include "nvbufsurface.h"
/* NVIDIA Decoder source pad memory feature. This feature signifies that source
* pads having this capability will push GstBuffers containing cuda buffers. */
#define GST_CAPS_FEATURES_NVMM "memory:NVMM"
#define CONFIG_FILE_VW "config/config_vw_detection.txt"
// static struct timeval start_time = { };
// static guint probe_counter = 0;
/* tiler_sink_pad_buffer_probe will extract metadata received on OSD sink pad
* and update params for drawing rectangle, object information etc. */
using namespace std;
/*static clock_t start_time = NULL;
static guint probe_counter = 0;
list<int> full_list;
int count_array = 0;
static int count = 0;
int a[20];
*/
gint frame_number = 0;
int counter = 0;
// #TODO remove hardcoded lines
static GstElement *create_source_bin(guint index, gchar *uri)
{
GstElement *bin = NULL /*, *uri_decode_bin = NULL*/;
gchar bin_name[16] = {};
gboolean multi_file_src = FALSE;
g_snprintf(bin_name, 15, "source-bin-%02d", index);
/* Create a source GstBin to abstract this bin's content from the rest of the
* pipeline */
bin = gst_bin_new(bin_name);
GstElement *source, *jpegparser, *decoder;
if (strstr(uri, "%"))
{
source = gst_element_factory_make("multifilesrc", "source");
multi_file_src = TRUE;
}
else
{
source = gst_element_factory_make("filesrc", "source");
}
jpegparser = gst_element_factory_make("jpegparse", "jpeg-parser");
decoder = gst_element_factory_make("nvv4l2decoder", "nvv4l2-decoder");
if (!source || !jpegparser || !decoder)
{
return NULL;
}
if (strstr(uri, "%"))
{
/* Set properties of the multifilesrc element */
GstCaps *caps =
gst_caps_new_simple("image/jpeg", "format", G_TYPE_STRING, "RGBA",
"framerate", GST_TYPE_FRACTION, 35, 1, NULL);
// GstCapsFeatures *feature = gst_caps_features_new (GST_CAPS_FEATURES_NVMM,
// NULL); gst_caps_set_features (caps, 0, feature);
g_object_set(G_OBJECT(source), "location", uri, "caps", caps, NULL);
}
else
{
g_object_set(G_OBJECT(source), "location", uri, NULL);
}
const char *dot = strrchr(uri, '.');
if ((!strcmp(dot + 1, "mjpeg")) || (!strcmp(dot + 1, "mjpg")) ||
(multi_file_src == TRUE))
{
#ifdef PLATFORM_TEGRA
g_object_set(G_OBJECT(decoder), "mjpeg", 1, NULL);
#endif
}
gst_bin_add_many(GST_BIN(bin), source, jpegparser, decoder, NULL);
gst_element_link_many(source, jpegparser, decoder, NULL);
/* We need to create a ghost pad for the source bin which will act as a proxy
* for the video decoder src pad. The ghost pad will not have a target right
* now. Once the decode bin creates the video decoder and generates the
* cb_newpad callback, we will set the ghost pad target to the video decoder
* src pad. */
if (!gst_element_add_pad(bin,
gst_ghost_pad_new_no_target("src", GST_PAD_SRC)))
{
return NULL;
}
GstPad *srcpad = gst_element_get_static_pad(decoder, "src");
if (!srcpad)
{
return NULL;
}
GstPad *bin_ghost_pad = gst_element_get_static_pad(bin, "src");
if (!gst_ghost_pad_set_target(GST_GHOST_PAD(bin_ghost_pad), srcpad))
{
return NULL;
}
return bin;
}
bool Is_path_exist(const std::string &s)
{
struct stat buffer;
return (stat(s.c_str(), &buffer) == 0);
}
std::string get_image_name(std::string image_name)
{
std::string cmd = "find " + input_path + " -samefile " +
input_path + "/tmp/" + image_name;
std::array<char, 128> buffer;
std::string result;
std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"),
pclose);
if (!pipe)
{
throw std::runtime_error("popen() failed!");
}
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
{
std::string buffer_data = buffer.data();
if (buffer_data.find(image_name) == std::string::npos)
result += buffer.data();
}
return result;
}
std::string exec(const char *cmd)
{
std::array<char, 128> buffer;
std::string result;
std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd, "r"), pclose);
if (!pipe)
{
throw std::runtime_error("popen() failed!");
}
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr)
{
result += buffer.data();
}
return result;
}
static GstPadProbeReturn tiler_src_pad_buffer_probe(GstPad *pad,
GstPadProbeInfo *info,
gpointer u_data)
{
NvDsFrameMeta *frame_meta = NULL;
GstBuffer *buf = (GstBuffer *)info->data;
guint num_rects = 0;
NvDsObjectMeta *obj_meta = NULL;
NvDsMetaList *l_frame = NULL;
NvDsMetaList *l_obj = NULL;
NvDsClassifierMetaList *l_classifier = NULL;
NvDsClassifierMeta *class_meta = NULL;
NvDsLabelInfoList *l_label = NULL;
NvDsLabelInfo *label_info = NULL;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
std::string image_name = "";
std::cout<<"do nothing "<<std::endl;
return GST_PAD_PROBE_OK;
}
static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data)
{
GMainLoop *loop = (GMainLoop *)data;
switch (GST_MESSAGE_TYPE(msg))
{
case GST_MESSAGE_EOS:
// loggger.log_info("bus_call", class_name, 22012);
g_main_loop_quit(loop);
break;
case GST_MESSAGE_WARNING:
{
gchar *debug;
GError *error;
gst_message_parse_warning(msg, &error, &debug);
g_free(debug);
// loggger.log_warning(GST_OBJECT_NAME(msg->src),
// class_name, 22507, error->message);
g_error_free(error);
break;
}
case GST_MESSAGE_INFO:
{
gchar *debug;
GError *error;
gst_message_parse_info(msg, &error, &debug);
g_free(debug);
g_error_free(error);
break;
}
case GST_MESSAGE_ERROR:
{
gchar *debug;
GError *error;
gst_message_parse_error(msg, &error, &debug);
// loggger.log_error(GST_OBJECT_NAME(msg->src),
// class_name, 22610, error->message);
if (debug)
g_free(debug);
g_error_free(error);
g_main_loop_quit(loop);
break;
}
#ifndef PLATFORM_TEGRA
case GST_MESSAGE_ELEMENT:
{
if (gst_nvmessage_is_stream_eos(msg))
{
guint stream_id;
if (gst_nvmessage_parse_stream_eos(msg, &stream_id))
{
g_print("Got EOS from stream %d\n", stream_id);
}
}
break;
}
#endif
default:
break;
}
return TRUE;
}
int main(int argc, char *argv[])
{
GMainLoop *loop = NULL;
GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL,
*vw_detection = NULL, *nvvidconv = NULL, *nvosd = NULL,
*tiler = NULL;
#ifdef PLATFORM_TEGRA
GstElement *transform = NULL;
#endif
GstBus *bus = NULL;
guint bus_watch_id;
GstPad *tiler_src_pad = NULL;
guint i, num_sources;
guint tiler_rows, tiler_columns;
guint pgie_batch_size;
// commint
// loggger.init_logging();
int current_device = -1;
cudaGetDevice(¤t_device);
struct cudaDeviceProp prop;
cudaGetDeviceProperties(&prop, current_device);
input_path = "/opt/nvidia/deepstream/deepstream-6.2/app/test/input/";
std::string app_path = "/opt/nvidia/deepstream/deepstream-6.2/app/test";
/* Standard GStreamer initialization */
gst_init(&argc, &argv);
loop = g_main_loop_new(NULL, FALSE);
/* Create gstreamer elements */
/* Create Pipeline element that will form a connection of other elements */
pipeline = gst_pipeline_new("pipeline");
/* Create nvstreammux instance to form batches from one or more sources. */
streammux = gst_element_factory_make("nvstreammux", "stream-muxer");
if (!pipeline || !streammux)
{
return -1;
}
if (!streammux)
{
return -1;
}
gst_bin_add(GST_BIN(pipeline), streammux);
/* Read Settings file */
char tmp[256];
getcwd(tmp, 256);
// Input Preprocessing
gchar *image_folder = new char[input_path.length() + 1];
strcpy(image_folder, input_path.c_str());
chdir(image_folder);
char s[100];
printf("%s\n", getcwd(s, 100));
exec("mkdir tmp");
exec("curIdx=0; for curFile in *.jpg; do outFile=`printf "
"\"./tmp/img_%04d.jpg\" \"$curIdx\"`; echo \'File: \'$curFile \'->\' "
"$outFile; ln $curFile $outFile; curIdx=`expr $curIdx + 1 `; done");
chdir(app_path.c_str());
input_path = input_path + "tmp/";
/*Create full input Path */
std::string inputFormat_ = "img_%04d.jpg";
/*Create full input Path */
std::string full_path_tmp = input_path + inputFormat_;
gchar *image_file = new char[full_path_tmp.length() + 1];
strcpy(image_file, full_path_tmp.c_str());
GstPad *sinkpad, *srcpad;
gchar pad_name[16] = {};
GstElement *source_bin = create_source_bin(0, image_file);
// log number of images of the input path
if (!source_bin)
{
return -1;
}
gst_bin_add(GST_BIN(pipeline), source_bin);
g_snprintf(pad_name, 15, "sink_%u", i);
sinkpad = gst_element_get_request_pad(streammux, pad_name);
if (!sinkpad)
{
return -1;
}
srcpad = gst_element_get_static_pad(source_bin, "src");
if (!srcpad)
{
return -1;
}
if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK)
{
return -1;
}
gst_object_unref(srcpad);
gst_object_unref(sinkpad);
num_sources = 1;
/* Use nvinfer to infer on batched frame. */
vw_detection =
gst_element_factory_make("nvinfer", "primary-nvinference-engine");
/* Use nvtiler to composite the batched frames into a 2D tiled array based
* on the source of the frames. */
tiler = gst_element_factory_make("nvmultistreamtiler", "nvtiler");
/* Use convertor to convert from NV12 to RGBA as required by nvosd */
nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter");
/* Create OSD to draw on the converted RGBA buffer */
nvosd = gst_element_factory_make("nvdsosd", "nv-onscreendisplay");
/* Finally render the osd output */
if(prop.integrated) {
sink= gst_element_factory_make ("nv3dsink", "nvvideo-renderer");
} else {
sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
}
g_object_set (G_OBJECT (sink), "sync", 0, NULL);
if (!vw_detection)
{
return -1;
}
if (!sink)
{
return -1;
}
if (!tiler)
{
return -1;
}
if (!nvvidconv)
{
return -1;
}
g_object_set(G_OBJECT(streammux), "width", 4096, "height",
2160, "batch-size", num_sources,
"batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);
/* Configure the nvinfer element using the nvinfer config file. */
g_object_set(G_OBJECT(vw_detection), "config-file-path", CONFIG_FILE_VW,
NULL);
/* Override the batch-size set in the config file with the number of sources.
*/
g_object_get(G_OBJECT(vw_detection), "batch-size", &pgie_batch_size, NULL);
if (pgie_batch_size != num_sources)
{
g_object_set(G_OBJECT(vw_detection), "batch-size", num_sources, NULL);
}
tiler_rows = (guint)sqrt(num_sources);
tiler_columns = (guint)ceil(1.0 * num_sources / tiler_rows);
/* we set the tiler properties here */
g_object_set(G_OBJECT(tiler), "rows", tiler_rows, "columns", tiler_columns,
"width", 4096, "height", 2160,
NULL);
/* we add a message handler */
bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
bus_watch_id = gst_bus_add_watch(bus, bus_call, loop);
gst_object_unref(bus);
gst_bin_add_many(GST_BIN(pipeline), nvvidconv ,vw_detection, tiler,
nvosd, sink, NULL);
gst_element_link_many(streammux,nvvidconv, vw_detection,
tiler, nvosd, sink, NULL);
GstPad *vw_sink_pad = gst_element_get_static_pad(vw_detection, "src");
gst_pad_add_probe(vw_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
tiler_src_pad_buffer_probe, NULL, NULL);
gst_object_unref(vw_sink_pad);
gst_element_set_state(pipeline, GST_STATE_PLAYING);
/* Wait till pipeline encounters an error or EOS */
g_main_loop_run(loop);
gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(pipeline));
g_source_remove(bus_watch_id);
g_main_loop_unref(loop);
return 0;
}
is multifilesrc not supported on 6.1 and 6.2? because it is only process one image, then showing the error
