Hello everybody,
I am setting this topic up as I am facing an issue that my poor gstreamer capabilities are not able to deal with. Briefly, I have to make an application that reads an h264 encoded stream from an IP camera via an RTSP stream and allows us to use incoming decoded images with opencv. To do this I am writting a gestreamer application that implements a pipeline with an appsink at the end. My firts test based on examples aims at achieving to put GStreamer and Opencv processing together and it is working fine. It reads not a stream but an internet hosted file sets a position, grabs the image at the current position and stores it as a png file on first hand and process the image with 2 opencv filters (gaussian and canny edge detector) to store edges in a second png file. Well this was just for the principal and to make me more familiar with gstreamer.
The actual issue is that when I want to go to a live stream and not a file I get into troubles and my applications in not able to get a single frame from the stream and I have very few information for debug. This is all the more weird in my gstreamer newby point of view because when I set the pipeline (with xvimagesink instead of appsink) using gst-launch-1.0 I can get the live stream perfectly.
So my first question is does someone have an idea of what I am missing?
here is the gst-launch pipeline to get the RTSP h264 encoded stream (also attached as simple_test.cpp)
gst-launch-1.0 rtspsrc location="rtsp://192.168.1.72:554/stream1" latency=0 ! decodebin ! xvimagesink
Here is the code used for the first test (can be found mostly on the web) but I also made some modifications to the example (Opencv dependent functions have been added):
#include <gst/gst.h>
#include <opencv2/opencv.hpp>
#ifdef HAVE_GTK
#include <gtk/gtk.h>
#include <gdk-pixbuf/gdk-pixbuf.h>
#endif
#include <stdlib.h>
#define CAPS "video/x-raw,format=RGB,width=320,pixel-aspect-ratio=1/1"
int main (int argc, char *argv[])
{
GstElement *pipeline, *sink;
gint width, height;
GstSample *sample;
gchar *descr;
GError *error = NULL;
gint64 duration, position;
GstStateChangeReturn ret;
gboolean res;
GstMapInfo map;
gst_init (&argc, &argv);
if (argc != 2)
{
g_print ("usage: %s <uri>\n Writes snapshot.png in the current directory\n", argv[0]);
exit (-1);
}
/* create a new pipeline */
descr = g_strdup_printf ("uridecodebin uri=%s ! videoconvert ! videoscale ! appsink name=sink caps=\"" CAPS "\"", argv[1]);
pipeline = gst_parse_launch (descr, &error);
if (error != NULL)
{
g_print ("could not construct pipeline: %s\n", error->message);
g_error_free (error);
exit (-2);
}
/* get sink */
sink = gst_bin_get_by_name (GST_BIN (pipeline), "sink");
/* set to PAUSED to make the first frame arrive in the sink */
ret = gst_element_set_state (pipeline, GST_STATE_PAUSED);
switch (ret)
{
case GST_STATE_CHANGE_FAILURE:
g_print ("failed to play the file\n");
exit (-1);
case GST_STATE_CHANGE_NO_PREROLL:
/* for live sources, we need to set the pipeline to PLAYING before we can
* receive a buffer. We don't do that yet */
g_print ("live sources not supported yet\n");
exit (-3);
default:
break;
}
/* This can block for up to 5 seconds. If your machine is really overloaded,
* it might time out before the pipeline prerolled and we generate an error. A
* better way is to run a mainloop and catch errors there. */
ret = gst_element_get_state (pipeline, NULL, NULL, 5 * GST_SECOND);
if (ret == GST_STATE_CHANGE_FAILURE)
{
g_print ("failed to play the file\n");
exit (-4);
}
/* get the duration */
gst_element_query_duration (pipeline, GST_FORMAT_TIME, &duration);
if (duration != -1)
/* we have a duration, seek to 5% */
position = duration * 5 / 100;
else
/* no duration, seek to 1 second, this could EOS */
position = 1 * GST_SECOND;
/* seek to the a position in the file. Most files have a black first frame so
* by seeking to somewhere else we have a bigger chance of getting something
* more interesting. An optimisation would be to detect black images and then
* seek a little more */
gst_element_seek_simple (pipeline, GST_FORMAT_TIME, GST_SEEK_FLAG_KEY_UNIT | GST_SEEK_FLAG_FLUSH, position);
/* get the preroll buffer from appsink, this block untils appsink really
* prerolls */
g_signal_emit_by_name (sink, "pull-preroll", &sample, NULL);
/* if we have a buffer now, convert it to a pixbuf. It's possible that we
* don't have a buffer because we went EOS right away or had an error. */
if (sample)
{
GstBuffer *buffer;
GstCaps *caps;
GstStructure *s;
/* get the snapshot buffer format now. We set the caps on the appsink so
* that it can only be an rgb buffer. The only thing we have not specified
* on the caps is the height, which is dependant on the pixel-aspect-ratio
* of the source material */
caps = gst_sample_get_caps (sample);
if (!caps)
{
g_print ("could not get snapshot format\n");
exit (-5);
}
s = gst_caps_get_structure (caps, 0);
/* we need to get the final caps on the buffer to get the size */
res = gst_structure_get_int (s, "width", &width);
res |= gst_structure_get_int (s, "height", &height);
if (!res)
{
g_print ("could not get snapshot dimension\n");
exit (-6);
}
/* create pixmap from buffer and save, gstreamer video buffers have a stride
* that is rounded up to the nearest multiple of 4 */
buffer = gst_sample_get_buffer (sample);
gst_buffer_map (buffer, &map, GST_MAP_READ);
#ifdef HAVE_GTK
GdkPixbuf *pixbuf = gdk_pixbuf_new_from_data (map.data,
GDK_COLORSPACE_RGB, FALSE, 8, width, height,
GST_ROUND_UP_4 (width * 3), NULL, NULL);
/* save the pixbuf */
gdk_pixbuf_save (pixbuf, "snapshot.png", "png", &error, NULL);
#endif
#ifdef HAVE_OPENCV
cv::Mat image(cv::Size(width, height), CV_8UC3, (char*)map.data, cv::Mat::AUTO_STEP);
cv::Mat edges;
cv::cvtColor(image, edges, CV_RGB2GRAY);
cv::GaussianBlur(edges, edges, cv::Size(7,7), 1.5, 1.5);
cv::Canny(edges, edges, 0, 30, 3);
cv::imwrite("snapshot_edges.png",edges);
#endif
gst_buffer_unmap (buffer, &map);
gst_sample_unref (sample);
}
else
{
g_print ("could not make snapshot\n");
}
/* cleanup and exit */
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
exit (0);
}
To fully compile it one needs to use g++ with opencv, GTK, CUFFT, CUDART and Gstreamer dependencies and declare HAVE_GTK and HAVE_OPENCV. I can provide a CMakeLists.txt file to build it if necessary.
This code works well when used with the following command (obviously simple_test is the program name):
simple_test http://mirrorblender.top-ix.org/peach/bigbuckbunny_movies/big_buck_bunny_480p_h264.mov
The big buck bunny can be replaced by another online video.
My second question is do I did something really bad in this example (except that it is not threaded and not really able to deal with stream (I only get one image and do not implement loop iterations here)?
Finally here is the code that I’d love to have help on (also attached as main.cpp). It should deal with live streams (no preroll is done?) and I don’t know why I never receive a single image (new_sample callback is never run).
#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include <stdlib.h>
#include "opencv2/opencv.hpp"
using namespace cv;
#define CAPS "video/x-raw,height=480,pixel-aspect-ratio=1/1"
// TODO: use synchronized deque
GMainLoop *loop;
std::deque<Mat> frameQueue;
int live_flag = 0;
int quit_flag = 0;
GstFlowReturn new_preroll(GstAppSink *appsink, gpointer data)
{
g_print ("Got preroll!\n");
return GST_FLOW_OK;
}
GstFlowReturn new_sample(GstAppSink *appsink, gpointer data)
{
static int framecount = 0;
framecount++;
static int width=0, height=0 ;
GstSample *sample = gst_app_sink_pull_sample(appsink);
GstCaps *caps = gst_sample_get_caps(sample);
GstBuffer *buffer = gst_sample_get_buffer(sample);
static GstStructure *s;
const GstStructure *info = gst_sample_get_info(sample);
// ---- get width and height
if(framecount==1)
{
if(!caps)
{
g_print("Could not get image info from filter caps");
exit(-11);
}
s = gst_caps_get_structure(caps,0);
gboolean res = gst_structure_get_int(s, "width", &width);
res |= gst_structure_get_int(s, "height", &height);
if(!res)
{
g_print("Could not get image width and height from filter caps");
exit(-12);
}
g_print("Image size: %d\t%d\n",width,height);
}
// ---- Read frame and convert to opencv format ---------------
GstMapInfo map;
gst_buffer_map (buffer, &map, GST_MAP_READ);
// convert gstreamer data to OpenCV Mat, you could actually
// resolve height / width from caps...
Mat frame(Size(width, height), CV_8UC3, (char*)map.data, Mat::AUTO_STEP);
// this lags pretty badly even when grabbing frames from webcam
Mat edges;
cvtColor(frame, edges, CV_RGB2GRAY);
GaussianBlur(edges, edges, Size(7,7), 1.5, 1.5);
Canny(edges, edges, 0, 30, 3);
imshow("stream", edges);
char key = cv::waitKey(10);
if(key!=-1) quit_flag = 1;
gst_buffer_unmap(buffer, &map);
// ------------------------------------------------------------
// print dot every 30 frames
if (framecount%30 == 0) {
g_print (".");
}
// show caps on first frame
if (framecount == 1) {
g_print ("%s\n", gst_caps_to_string(caps));
}
gst_sample_unref (sample);
return GST_FLOW_OK;
}
static gboolean my_bus_callback (GstBus *bus, GstMessage *message, gpointer data)
{
g_print ("Got %s message from %s\n", GST_MESSAGE_TYPE_NAME (message), GST_OBJECT_NAME (message->src));
switch (GST_MESSAGE_TYPE (message))
{
case GST_MESSAGE_ERROR:
{
GError *err;
gchar *debug;
gst_message_parse_error (message, &err, &debug);
g_print ("Error from %s: %s\n", GST_OBJECT_NAME (message->src), err->message);
g_error_free (err);
g_free (debug);
break;
}
case GST_MESSAGE_EOS:
/* end-of-stream */
quit_flag = 1;
break;
case GST_MESSAGE_STATE_CHANGED:
GstState oldstate, newstate;
gst_message_parse_state_changed(message, &oldstate, &newstate, NULL);
g_print ("Element %s changed state from %s to %s.\n",
GST_OBJECT_NAME (message->src),
gst_element_state_get_name (oldstate),
gst_element_state_get_name (newstate));
break;
default:
/* unhandled message */
break;
}
/* we want to be notified again the next time there is a message
* on the bus, so returning TRUE (FALSE means we want to stop watching
* for messages on the bus and our callback should not be called again)
*/
return TRUE;
}
int main (int argc, char *argv[])
{
GError *error = NULL;
GstElement *pipeline, *sink;
GstStateChangeReturn state_ret;
GstSample *sample;
gst_init (&argc, &argv);
gchar *descr = g_strdup(
"rtspsrc location=\"rtsp://192.168.1.72:554/stream1\" latency=0 ! "
"rtph264depay ! "
"decodebin ! "
"appsink name=sink sync=true caps=\"" CAPS "\""
);
pipeline = gst_parse_launch (descr, &error);
if (error != NULL)
{
g_print ("could not construct pipeline: %s\n", error->message);
g_error_free (error);
exit (-1);
}
/* get sink */
sink = gst_bin_get_by_name (GST_BIN (pipeline), "sink");
/*set to pause*/
state_ret = gst_element_set_state(pipeline, GST_STATE_PAUSED);
switch(state_ret)
{
case GST_STATE_CHANGE_FAILURE:
g_print ("failed to play the file\n");
exit (-2);
case GST_STATE_CHANGE_NO_PREROLL:
/* for live sources, we need to set the pipeline to PLAYING before we can
* receive a buffer. */
g_print ("live source detected\n");
live_flag = 1;
break;
default:
break;
}
gst_app_sink_set_emit_signals((GstAppSink*)sink, true);
gst_app_sink_set_drop((GstAppSink*)sink, true);
gst_app_sink_set_max_buffers((GstAppSink*)sink, 1);
GstAppSinkCallbacks callbacks = { NULL, new_preroll, new_sample };
gst_app_sink_set_callbacks (GST_APP_SINK(sink), &callbacks, NULL, NULL);
GstBus *bus;
guint bus_watch_id;
bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
bus_watch_id = gst_bus_add_watch (bus, my_bus_callback, NULL);
gst_object_unref (bus);
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_PLAYING);
namedWindow("stream",1);
loop = g_main_loop_new(NULL,false);
g_main_loop_run(loop);
cv::destroyWindow("stream");
g_print ("Going to end of main!\n");
gst_element_set_state (GST_ELEMENT (pipeline), GST_STATE_NULL);
gst_object_unref (GST_OBJECT (pipeline));
return 0;
}
So the final and most important question is: does someone can explain me what is wrong in this code and induces this behavior?
Thanks you a lot for reading until here! and thank you even more if you have any clue.
Best regards,
Olivier.
simple_test.cpp (4.53 KB)
main.cpp (5.59 KB)