Nvivafilter EOS C

Hello,
I am trying to write a GStreamer app in C with two pipelines, each using nvivafilter plugin. There seems to be a problem when sending EOS to one of the pipelines. App crashes with error:

NvEGLImageFromFd: Failed to create EGLImage from dma-buf fd (1047)

What is the proper way to send EOS to one pipeline and have the other continue to play? Advice please.

I have attached a basic example below.

#include <gst/gst.h>
#include <signal.h>

static gboolean recording = FALSE;

static GMainLoop *loop;
static GstElement *pipeline1, *pipeline2;

static gboolean
message_cb (GstBus * bus, GstMessage * message, gpointer user_data)
{
switch (GST_MESSAGE_TYPE (message)) {
case GST_MESSAGE_ERROR:{
GError *err = NULL;
gchar *name, *debug = NULL;

  name = gst_object_get_path_string (message->src);
  gst_message_parse_error (message, &err, &debug);

  g_printerr ("ERROR: from element %s: %s\n", name, err->message);
  if (debug != NULL)
    g_printerr ("Additional debug info:\n%s\n", debug);

  g_error_free (err);
  g_free (debug);
  g_free (name);

  g_main_loop_quit (loop);
  break;
}
case GST_MESSAGE_WARNING:{
            GError *err = NULL;
            gchar *name, *debug = NULL;

            name = gst_object_get_path_string (message->src);
            gst_message_parse_warning (message, &err, &debug);

            g_printerr ("ERROR: from element %s: %s\n", name, err->message);
            if (debug != NULL)
            g_printerr ("Additional debug info:\n%s\n", debug);

            g_error_free (err);
            g_free (debug);
            g_free (name);
            break;
}
case GST_MESSAGE_EOS:{
            g_print ("Got EOS\n");
            g_main_loop_quit (loop);
            gst_element_set_state (pipeline1, GST_STATE_NULL);
            g_main_loop_unref (loop);
            gst_object_unref (pipeline1);
            exit(0);
            break;
    }
default:
            break;

}

return TRUE;
}

void startRecording()
{
GstBus *bus2;
GstMessage *msg2;

g_print(“startRecording\n”);

pipeline2 =
gst_parse_launch
(“videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),width=852,height=480 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true ! videorate ! nvv4l2h265enc ! h265parse ! qtmux name=muxer ! filesink location=/tmp/cuda.mp4 audiotestsrc is-live=true ! queue ! audioconvert ! audioresample ! avenc_aac ! muxer.”, NULL);

gst_element_set_state (pipeline2, GST_STATE_PLAYING);

g_print(“Starting Pipeline2”);
recording = TRUE;

bus2 = gst_element_get_bus (pipeline2);
msg2 =
gst_bus_timed_pop_filtered (bus2, GST_CLOCK_TIME_NONE,
GST_MESSAGE_ERROR | GST_MESSAGE_EOS);
if (GST_MESSAGE_TYPE (msg2) == GST_MESSAGE_ERROR) {
g_error (“An error occured with Pipeline2”);
}

gst_message_unref (msg2);
gst_object_unref (bus2);
gst_element_set_state (pipeline2, GST_STATE_NULL);
gst_object_unref (pipeline2);

}

void stopRecording() {
g_print(“stopRecording\n”);

gst_element_send_event(pipeline2, gst_event_new_eos());

recording = FALSE;
}

int
sigintHandler(int unused) {
g_print(“You ctrl-c-ed!”);
if (recording)
stopRecording();
else
startRecording();
return 0;
}

int
main (int argc, char *argv)
{
GstBus *bus;
GstMessage *msg;

signal(SIGINT, sigintHandler);
/* Initialize Gstreamer */
gst_init (&argc, &argv);

/* Build the pipeline */
pipeline1 =
gst_parse_launch
(“videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),width=852,height=480 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true ! videorate ! nvv4l2h264enc insert-sps-pps=1 idrinterval=4 ! rtspclientsink location=rtsp://localhost:8554/mystream”, NULL);

/* Start playing */

gst_element_set_state (pipeline1, GST_STATE_PLAYING);

startRecording();

g_print(“Starting camera”);

bus = gst_element_get_bus (pipeline1);
msg =
gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,
GST_MESSAGE_ERROR | GST_MESSAGE_EOS);
if (GST_MESSAGE_TYPE (msg) == GST_MESSAGE_ERROR) {
g_error (“An error occured with v4l2”);
}

gst_message_unref (msg);
gst_object_unref (bus);
gst_element_set_state (pipeline1, GST_STATE_NULL);
gst_object_unref (pipeline1);
loop = g_main_loop_new(NULL, FALSE);

bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline1));
gst_bus_add_signal_watch(bus);
g_signal_connect(G_OBJECT(bus), “message”, G_CALLBACK(message_cb), NULL);

return 0;
}
nviva.c (4.1 KB)

Hi,
Please refer to this sample:
GStreamer freeze when using qtmux and NVIDIA-accelerated h264/h265 encoding - #7 by DaneLLL

You can send EoS by calling:

gst_element_send_event (src, gst_event_new_eos ());

Please try this method and see if it works.

Hi DaneLLL,
This does not seem to work.

I still get the error

NvEGLImageFromFd: Failed to create EGLImage from dma-buf fd (1047)

I believe there is something in the nvivafilter that is causing this issue. If only one of the pipelines uses the nvivafilter, recording and stopping works fine. It is when nvivafilter is used twice, sending eos to one pipeline causes the app to crash. Could you please look into it?

#include <gst/gst.h>
#include <signal.h>

static gboolean recording = FALSE;

static GMainLoop *loop;
static GstElement *pipeline1, *pipeline2;

void startRecording()
{
GstBus *bus2;
GstMessage *msg2;

g_print(“startRecording\n”);

pipeline2 =
gst_parse_launch
(“videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),width=852,height=480 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true ! videorate ! nvv4l2h265enc ! h265parse ! qtmux name=muxer ! filesink location=/tmp/cuda.mp4 audiotestsrc is-live=true ! queue ! audioconvert ! audioresample ! avenc_aac ! muxer.”, NULL);

gst_element_set_state (pipeline2, GST_STATE_PLAYING);

g_print(“Starting Pipeline2”);
recording = TRUE;

bus2 = gst_pipeline_get_bus (GST_PIPELINE(pipeline2));
msg2 = gst_bus_poll(bus2, GST_MESSAGE_EOS, GST_CLOCK_TIME_NONE);

gst_message_unref (msg2);
gst_object_unref (bus2);
gst_element_set_state((GstElement*)pipeline2, GST_STATE_NULL);
gst_object_unref (GST_OBJECT(pipeline2));
g_print(“clean”);

}

void stopRecording() {
g_print(“stopRecording\n”);

gst_element_send_event(pipeline2, gst_event_new_eos());

recording = FALSE;
}

int
sigintHandler(int unused) {
g_print(“You ctrl-c-ed!”);
if (recording)
stopRecording();
else
startRecording();
return 0;
}

int
main (int argc, char *argv)
{
GstBus *bus;
GstMessage *msg;

signal(SIGINT, sigintHandler);
/* Initialize Gstreamer */
gst_init (&argc, &argv);

/* Build the pipeline */
pipeline1 =
gst_parse_launch
(“videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),width=852,height=480 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true ! videorate ! nvv4l2h264enc insert-sps-pps=1 idrinterval=4 ! rtspclientsink location=rtsp://localhost:8554/mystream”, NULL);

/* Start playing */

gst_element_set_state (pipeline1, GST_STATE_PLAYING);

startRecording();

g_print(“Starting camera”);

bus = gst_pipeline_get_bus (GST_PIPELINE(pipeline1));
msg =
gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,
GST_MESSAGE_ERROR | GST_MESSAGE_EOS);
if (GST_MESSAGE_TYPE (msg) == GST_MESSAGE_ERROR) {
g_error (“An error occured with v4l2”);
}
msg = gst_bus_poll(bus, GST_MESSAGE_EOS, GST_CLOCK_TIME_NONE);
gst_element_set_state((GstElement*)pipeline1, GST_STATE_NULL);

gst_message_unref (msg);
gst_object_unref (bus);
gst_object_unref (pipeline1);
loop = g_main_loop_new(NULL, FALSE);

return 0;
}

nvivafiltereos.c (2.4 KB)

Hi,
So single pipeline works and if there are two pipelines launched, the issue is observed. IS this correct?

Please also share your release version and make command to build the test sample.

Hi,
Thank you for getting back to me.

To clarify what I’ve learned about using nvivafilter:

1.An app with two pipelines, one with nvivafilter, and one without, will start and stop recording either pipeline OK.

2.An app with two pipelines, each with nvivafilter, crashes if one of them gets EOS.

3.If I launch two nvivafilter pipelines in CLI, each runs independently and can stop and start without any problems.

The use case is to stream a resized v4l2src with a cuda-filter via RTSP for live preview, and record the same v4l2src full size with the same cuda-filter intermittently.

R32 (release), REVISION: 7.2, GCID: 30192233, BOARD: t210ref, EABI: aarch64, DATE: Wed Apr 20 21:34:48 UTC 2022

example compiled with:

gcc nvivafiltereos.c -o nvivafiltereos pkg-config --cflags --libs gstreamer-1.0

Hi,
We have tried the test sample and don’t hit the issue on r32.7.2/Jetson Nano developer kit. Please check if there is any additional step for replicating the issue. Or please list the steps in detail for reference.

Hmm, I’m still getting the error…
I will try a fresh install.

Hi,
Starting with a new sd card, apt update and upgrade,

R32 (release), REVISION: 7.2, GCID: 30192233, BOARD: t210ref, EABI: aarch64, DATE: Wed Apr 20 21:34:48 UTC 2022

I still get the same error:

NvEGLImageFromFd: Failed to create EGLImage from dma-buf fd (1047)

Seems rather an error from your nvivafilter lib than from the code you’ve provided above.

Does your nvivafilter lib expect RGBA or NV12 format ?
You would specify this as output caps of nvivafilter (that prepares output buffer, where your filter can then make changes).

You may attach your nvivafilter for better reproducibility.

Thank you, I have tried adding video/x-raw(memory:NVMM),format=NV12 but still get the same error. I am also using the default libnvsample_cudaprocess.so (the one with the green box).

The NvEGLImageFromFd error only occurs when I send an EOS to one of the playing pipelines. The error is caused by the pipeline that did not receive the EOS. Both pipelines are with green box.

I tried reflashing the Nano with SDKManager and still I get the error.

Maybe you wouldn’t unref pipeline2 when playing starts…
Try this and tell if it gets better:

#include <gst/gst.h>
#include <signal.h>

static gboolean recording = FALSE;

static GMainLoop *loop;
static GstElement *pipeline1, *pipeline2;


void startRecording()
{
	GstBus *bus2;
	GstMessage *msg2;

	g_print("startRecording\n");

	pipeline2 = 
		gst_parse_launch
		("videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),width=852,height=480 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true ! videorate ! nvv4l2h265enc ! h265parse ! qtmux name=muxer ! filesink location=/tmp/cuda.mp4 audiotestsrc is-live=true ! queue ! audioconvert ! audioresample ! avenc_aac ! muxer.", NULL);

	gst_element_set_state (pipeline2, GST_STATE_PLAYING);
	
	g_print("Starting Pipeline2");

	bus2 = gst_pipeline_get_bus (GST_PIPELINE(pipeline2));
	msg2 = gst_bus_poll(bus2, GST_MESSAGE_EOS, GST_CLOCK_TIME_NONE);

	gst_message_unref (msg2);
	gst_object_unref (bus2);
	gst_element_set_state((GstElement*)pipeline2, GST_STATE_NULL);
	//gst_object_unref (GST_OBJECT(pipeline2));
	g_print("clean");
	recording = TRUE;
}

void stopRecording() {
	g_print("stopRecording\n");

	gst_element_send_event(pipeline2, gst_event_new_eos());
	GstBus *bus2 = gst_pipeline_get_bus(GST_PIPELINE(pipeline2));
	GstMessage *msg2 = gst_bus_poll(bus2, GST_MESSAGE_EOS, GST_CLOCK_TIME_NONE);
	gst_message_unref (msg2);
	gst_object_unref (bus2);
	gst_element_set_state((GstElement*)pipeline2, GST_STATE_NULL);
	gst_object_unref (GST_OBJECT(pipeline2));
	g_print("clean");

	recording = FALSE;
}

void
sigintHandler(int unused) {
	g_print("You ctrl-c-ed!");
	if (recording)
		stopRecording();
	else
		startRecording();
	return;
}

int
main (int argc, char *argv[])
{
	GstBus *bus;
	GstMessage *msg;

	signal(SIGINT, sigintHandler);
	/* Initialize Gstreamer */
	gst_init (&argc, &argv);

	/* Build the pipeline */
	pipeline1 = 
		gst_parse_launch
		("videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),width=852,height=480 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true ! videorate ! nvv4l2h264enc insert-sps-pps=1 idrinterval=4 ! rtspclientsink location=rtsp://localhost:8554/mystream", NULL);

	/* Start playing */

	gst_element_set_state (pipeline1, GST_STATE_PLAYING);

	startRecording();
	
	g_print("Starting camera");

	bus = gst_pipeline_get_bus (GST_PIPELINE(pipeline1));
	msg = 
		gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,
				GST_MESSAGE_ERROR | GST_MESSAGE_EOS);
	if (GST_MESSAGE_TYPE (msg) == GST_MESSAGE_ERROR) {
		g_error ("An error occured with v4l2");
	}
	msg = gst_bus_poll(bus, GST_MESSAGE_EOS, GST_CLOCK_TIME_NONE);
	gst_element_set_state((GstElement*)pipeline1, GST_STATE_NULL);

	gst_message_unref (msg);
	gst_object_unref (bus);
	gst_object_unref (pipeline1);
	loop = g_main_loop_new(NULL, FALSE);

	return 0;
}

Hi @Honey_Patouceul @DaneLLL,

Just wanted to report I was able to get this working by using ipcpipeline to split the nvivfilters to two processes

Hi,
If it is good to use two processes, please use the solution. Somehow we set up and test but do not observe the failure. Not sure what is missing in the setup.

It’s not optimal…
Also I notice lots of timing related issues when adding audio ie dropped frames and major stutters. Not sure where to begin debugging that.

I learned something tho. Thank you.

Again I think that the problem may be in your nvivafilter code that may not be reentrant or somewhat not supporting two instances in the same process.

For video/audio synchronization, be aware that video may be late as compared to audio, so it may be useful to use queue or some audio encoding that may supports the delay.

Searching this forum you may find some examples about that.

Have you tried running the example I posted? I’d be curious to know whether it runs on your machine. I have the 2gb model maybe why.

I didn’t write nvivafilter code. I am using the supplied lib.

I also tried an audio source without providing clock, async=false, queue\s) leaky , rearranging the pipeline starting with qtmux ! filesink ! … Maybe I don’t get the right combination but I want qtmux. To clarify, the video also stutters big time. Like showing future frames and jumping time back and forth.

Update:
The stuttering with audio it most likely caused by my nvivafilter custom-lib trying to do too much. Replacing with the default is OK.

@mhd0425,

Sorry I’m unable to check that for some reason (a fuel boiler out of order makes me away from home and Jetsons till end of next week), but I am curious about this case.
I quickly tested your case on AGX Orin and I may have posted a bit early without much testing…

However, I think that basic nvivafilter doesn’t really have a gpu process, so it may just be related to EGL mapping.
Maybe able to test your case in coming days, but you may quicker figure it out.

1 Like

See attached code and nvivafilter:

*Test with a 4k usb camera.

*The filter is for tonemapping slog2 to HLG colorspace and adjusting values.

*Comment out the different pipeline3 to test with and without audio. hlgtonemap.so without audio, the filter can do constant 29.97fps. hlgtonemap.so with audio it is nowhere near 29.97fps. libnvsample_cudaprocess.so with audio is 29.97fps. (yes, it is because my nvivafilter that it stutters.)

*Also requires RidgeRun/gst-interpipe GitHub - RidgeRun/gst-interpipe: GStreamer plug-in for interpipeline communication but could be rewritten to use appsink appsrc.

*For RTSP server, I use aler9/rtsp-simple-server GitHub - aler9/rtsp-simple-server: ready-to-use RTSP / RTMP / LL-HLS server and proxy that allows to read, publish and proxy video and audio streams

*Compile with gcc fixaudio.c -o fixaudio pkg-config --cflags --libs gstreamer-1.0

*RTSP streaming will start immediately when program is run. Ctrl-c will start and stop recording. File is saved to /tmp/cuda.mp4. To exit, you will have to kill both fixaudio processes.

The Orin could probably handle the tonemapping with audio. But I am running on Nano 2GB

fixaudio.c (7.1 KB)
hlgtonemap.so (673.6 KB)

I have been looking at this case, and I think that (at least using R35.1.0 on AGX Orin) there is a problem when stopping nvivafliter plugin, whatever is the customer lib.

I have been narrowing this to these two test cases so that NVIDIA team can easily try and adapt for a future test suite.

1. nvivafilter fails to restart

#include <unistd.h>
#include <gst/gst.h>

int
main (int argc, char *argv[])
{
    /* Initialize GStreamer */
    gst_init (&argc, &argv);

    const gchar * pipeline_desc_0 =
       "videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),format=NV12 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true pre-process=false post-process=false ! video/x-raw(memory:NVMM),format=NV12 ! nvvidconv ! nv3dsink";
       //"videotestsrc is-live=true ! nvvidconv ! nv3dsink";

    g_print("Pipeline0:\n%s\n", pipeline_desc_0);

    GstElement * pipeline0 = gst_parse_launch(pipeline_desc_0, NULL);

    g_print("Start\n");
    gst_element_set_state (pipeline0, GST_STATE_PLAYING);
    sleep(2);

    g_print("Stopping pipeline0\n");
    gst_element_set_state (pipeline0, GST_STATE_NULL);
    sleep(1);

    g_print("Play again. Can you see it ?\n");
    gst_element_set_state (pipeline0, GST_STATE_PLAYING);
    sleep(5);
    
    g_print("Stopping pipeline0\n");
    gst_element_set_state (pipeline0, GST_STATE_NULL);

    return 0;
}


/* Can be built with:  gcc nvivafilter_fails_to_restart.c -o nvivafilter_fails_to_restart $(pkg-config --cflags --libs gstreamer-1.0) */

2. nvivafilter stopping breaks other pipeline

#include <unistd.h>
#include <gst/gst.h>


int
main (int argc, char *argv[])
{
	const gchar * pipeline_desc_0 =
	"videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),format=NV12 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true pre-process=false post-process=false ! video/x-raw(memory:NVMM),format=NV12 ! nvvidconv ! nv3dsink";
	//"videotestsrc is-live=true ! nvvidconv ! nv3dsink";
	//"videotestsrc is-live=true ! nvvidconv ! nvegltransform ! nveglglessink";

	const gchar * pipeline_desc_1 =
	"videotestsrc is-live=true ! nvvidconv ! video/x-raw(memory:NVMM),format=NV12 ! nvivafilter customer-lib-name=libnvsample_cudaprocess.so cuda-process=true pre-process=false post-process=false ! video/x-raw(memory:NVMM),format=NV12 ! nvvidconv ! nv3dsink window-y=360";
	//"videotestsrc is-live=true ! nvvidconv ! nv3dsink window-y=360";

	GstElement * pipeline0, * pipeline1;

	/* Initialize GStreamer */
	gst_init (&argc, &argv);

	g_print("Pipeline0:\n%s\n", pipeline_desc_0);
	g_print("Pipeline1:\n%s\n", pipeline_desc_1);

	pipeline0 = gst_parse_launch(pipeline_desc_0, NULL);
	pipeline1 = gst_parse_launch(pipeline_desc_1, NULL);

	g_print("Start\n");
	gst_element_set_state (pipeline0, GST_STATE_PLAYING);
	gst_element_set_state (pipeline1, GST_STATE_PLAYING);
	sleep(2);

	g_print("Stopping pipeline0. This breaks pipeline1 if pipeline0 had nvivafilter\n");
	gst_element_set_state (pipeline0, GST_STATE_NULL);
	gst_object_unref (pipeline0);
	sleep(1);

	g_print("Play again pipeline0\n");
	pipeline0 = gst_parse_launch(pipeline_desc_0, NULL);
	gst_element_set_state (pipeline0, GST_STATE_PLAYING);
	sleep(5);

	g_print("Stopping pipelines\n");
	gst_element_set_state (pipeline0, GST_STATE_NULL);
	gst_element_set_state (pipeline1, GST_STATE_NULL);

	gst_object_unref (pipeline0);
	gst_object_unref (pipeline1);

	return 0;
}


/* Can be built with:  gcc nvivafilter_stopping_breaks_other_pipeline.c -o nvivafilter_stopping_breaks_other_pipeline $(pkg-config --cflags --libs gstreamer-1.0) */

So even not using EOS for stopping, there are issues when nvivafilter is stopped that would need to be fixed for your case.

NVIDIA experts would better advise for this.

Hi,
Could you try Jetpack 4.6.3? It is latest release for Jetson Nano and please give it a try.