how to use nvidia h264 encoder instead of gstreamer native x264enc

Hello,

I managed to implement a basic pipeline that decodes a h264 stream from RTSP, encode it back again and store it to disk in chunks of a specified size.

I have two questions:

  1. how can I change the encoding element and replace x264enc with nvv4l2h264enc so that the encoding also run in GPU? I tried to simply replace one for the other but I runs into linking issues

  2. is it possible to bypass the decoding/encoding part and directly store on disk the stream as an mp4 containing h264?

I include a simple example of my implementation below.
I use DeepStream 4.0

many thanks in advance,

f

#include <gst/gst.h>
#include <unistd.h>

/* NVIDIA Decoder source pad memory feature. This feature signifies that source
 * pads having this capability will push GstBuffers containing cuda buffers. */
#define GST_CAPS_FEATURES_NVMM "memory:NVMM"

static gboolean
bus_call (GstBus * bus, GstMessage * msg, gpointer data)
{
  GMainLoop *loop = (GMainLoop *) data;
  switch (GST_MESSAGE_TYPE (msg)) {
    case GST_MESSAGE_EOS:
      g_print ("End of stream\n");
      g_main_loop_quit (loop);
      break;
    case GST_MESSAGE_WARNING:
    {
      gchar *debug;
      GError *error;
      gst_message_parse_warning (msg, &error, &debug);
      g_printerr ("WARNING from element %s: %s\n",
          GST_OBJECT_NAME (msg->src), error->message);
      g_free (debug);
      g_printerr ("Warning: %s\n", error->message);
      g_error_free (error);
      break;
    }
    case GST_MESSAGE_ERROR:
    {
      gchar *debug;
      GError *error;
      gst_message_parse_error (msg, &error, &debug);
      g_printerr ("ERROR from element %s: %s\n",
          GST_OBJECT_NAME (msg->src), error->message);
      if (debug)
        g_printerr ("Error details: %s\n", debug);
      g_free (debug);
      g_error_free (error);
      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }
  return TRUE;
}

static void
cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, GstElement * next_element)
{
    g_print ("In cb_newpad\n");
    GstPad *sink_pad = NULL;
    sink_pad = gst_element_get_static_pad (next_element, "sink");
    GstCaps *sink_caps = gst_pad_get_current_caps (sink_pad);
    g_print("***** %d\n", gst_caps_get_size (sink_caps));
    const GstStructure *sink_str = gst_caps_get_structure (sink_caps, 0);
    const gchar *sink_pad_type = gst_structure_get_name (sink_str);

    GstCaps *src_caps = gst_pad_get_current_caps (decoder_src_pad);
    const GstStructure *src_str = gst_caps_get_structure (src_caps, 0);
    const gchar *new_pad_type = gst_structure_get_name (src_str);
    GstCapsFeatures *features = gst_caps_get_features (src_caps, 0);

    /* Need to check if the pad created by the decodebin is for video and not audio. */
    if (g_str_has_prefix (new_pad_type, "video/x-raw")) {
      /* Link the decodebin pad only if decodebin has picked nvidia
       * decoder plugin nvdec_*. We do this by checking if the pad caps contain
       * NVMM memory features. */
        if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) {
            /* Attempt the link */
            GstPadLinkReturn ret = gst_pad_link (decoder_src_pad, sink_pad);
            if (GST_PAD_LINK_FAILED (ret))
                g_print ("Type is '%s' but link failed.\n", new_pad_type);
            else
                g_print ("Link succeeded (source is '%s' and sink is '%s').\n", new_pad_type, sink_pad_type);
        }
    }
    /* Unreference the new pad's caps, if we got them */
    if (src_caps != NULL)
        gst_caps_unref (src_caps);
    if (sink_caps != NULL)
        gst_caps_unref (sink_caps);

    /* Unreference the sink pad */
    gst_object_unref (sink_pad);
}

int main(int argc, char *argv[]) {
    GstElement *pipeline = NULL, *source = NULL, *convert = NULL, *queue = NULL, *encoder = NULL, *muxer = NULL, *sink = NULL;
    GMainLoop *loop = NULL;
    GstBus *bus;
    GstStateChangeReturn ret;

    /* Initialize GStreamer */
    gst_init (&argc, &argv);
    loop = g_main_loop_new (NULL, FALSE);

    /* Create the elements */
    source = gst_element_factory_make ("uridecodebin", "uridecodebin");
    convert = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
    queue = gst_element_factory_make ("queue", NULL);
    encoder = gst_element_factory_make("x264enc", NULL);
/*    encoder = gst_element_factory_make("nvv4l2h264enc", NULL);*/
    sink = gst_element_factory_make("splitmuxsink", NULL);

    /* Create the empty pipeline */
    pipeline = gst_pipeline_new ("test-pipeline");

    /* check whether all elements are created*/
    if (!pipeline || !source || !convert || !sink) {
        g_printerr ("Not all elements could be created.\n");
        return -1;
    }

    /* Build the pipeline without linking source with the rest of the elements */
    gst_bin_add_many (GST_BIN (pipeline), source, convert, queue, encoder, sink, NULL);
    if (!gst_element_link_many (convert, queue, encoder, sink, NULL)) {
        g_printerr ("Elements could not be linked.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Set properties */
    g_object_set (source, "uri", "rtsp://input_uri", NULL);
    g_object_set (sink, "location", "/out_dir/file%02d.mp4", NULL);
    g_object_set (sink, "max-size-bytes", 5 * 1024 * 1024, NULL);
/*    g_object_set (sink, "max-size-time", 10 * 1000 * 1000, NULL);*/

    /* Connect to the pad-added signal */
    g_signal_connect (G_OBJECT (source), "pad-added", G_CALLBACK (cb_newpad), convert);

    /* we add a message handler */
    bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
    guint bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);

// Output a dot file of the pipeline
    // Be sure to $ export GST_DEBUG_DUMP_DOT_DIR=/tmp
    // and run with the --gst-enable-gst-debug command line switch
/*    GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline_graph");*/

    /* Start playing */
    ret = gst_element_set_state (pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) {
        g_printerr ("Unable to set the pipeline to the playing state.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Wait till pipeline encounters an error or EOS */
    g_print ("Running...\n");
    g_main_loop_run (loop);

    /* Free resources */
    gst_object_unref (bus);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    gst_object_unref (pipeline);
    g_source_remove (bus_watch_id);
    g_main_loop_unref (loop);
    return 0;
}

Hi,
You may call gst_element_link_filtered() to configure caps=video/x-raw(memory:NVMM) in both source and sink pads of nvvideoconvert.
Here is a sample of using the function.

Hi,
thanks for your reply;
I tried to implement your suggestion by forcing both the link converter–>queue and queue–>encoder to have required capabilities (video/x-raw(memory:NVMM)). Then, I do a generic link encoder–>sink.
Apparently the last link is the one which raises problems.

In principle, source pad of nvv4l2h264enc has capabilities

SRC template: 'src'
    Availability: Always
    Capabilities:
      video/x-h264
          stream-format: byte-stream
              alignment: au

just like x264enc which linked against the pipeline sink (splitmuxsink) with no problems.

Any clue of what is causing the problem?

Thanks again

I attach the part of code that I modified wrt the code of my previous post

/* Create the elements */
    source = gst_element_factory_make ("uridecodebin", "uridecodebin");
    convert = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
    queue = gst_element_factory_make ("queue", NULL);
/*    encoder = gst_element_factory_make("x264enc", NULL);*/
    encoder = gst_element_factory_make("nvv4l2h264enc", NULL);
    sink = gst_element_factory_make("splitmuxsink", NULL);

    /* Create the empty pipeline */
    pipeline = gst_pipeline_new ("test-pipeline");

    /* check whether all elements are created*/
    if (!pipeline || !source || !convert || !queue || !encoder || !sink) {
        g_printerr ("Not all elements could be created.\n");
        return -1;
    }

    /* set requirements on caps for linking*/
    GstCaps *caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING,"NV12", "width", G_TYPE_INT, 1920, "height", G_TYPE_INT, 1080, "framerate", GST_TYPE_FRACTION, 30, 1, NULL);
    GstCapsFeatures *feature = gst_caps_features_new ("memory:NVMM", NULL);
    gst_caps_set_features (caps, 0, feature);

    /* Build the pipeline without linking source with the rest of the elements */
    gst_bin_add_many (GST_BIN (pipeline), source, convert, queue, encoder, sink, NULL);
    if (!gst_element_link_filtered (convert, queue, caps))
        g_printerr ("converter and queue not linked.\n");
    if (!gst_element_link_filtered (queue, encoder, caps))
        g_printerr ("queue and encoder not linked.\n");
    if (!gst_element_link (encoder, sink))
        g_printerr ("encoder and sink not linked.\n");
    gst_object_unref (pipeline);

Hi,
Please try

... ! nvv4l2h264enc ! h264parse ! splitmuxsink

Hi,
thanks again for replying.

Following your suggestion, I added the parser. Now it looks like I don’t have linking problems, the application seems to execute but it remains in playing state without producing no output

this are the changes I made

/* Create the elements */
    source = gst_element_factory_make ("uridecodebin", "uridecodebin");
    convert = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter1");
    queue = gst_element_factory_make ("queue", NULL);
    encoder = gst_element_factory_make("nvv4l2h264enc", NULL);
    parser = gst_element_factory_make("h264parse", NULL);
    sink = gst_element_factory_make("splitmuxsink", NULL);

    /* Create the empty pipeline */
    pipeline = gst_pipeline_new ("test-pipeline");

    /* check whether all elements are created*/
    if (!pipeline || !source || !convert || !queue || !encoder || !parser || !sink) {
        g_printerr ("Not all elements could be created.\n");
        return -1;
    }

    /* set requirements on caps for linking*/
    GstCaps *caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, "NV12", "width", G_TYPE_INT, 1920, "height", G_TYPE_INT, 1080, "framerate", GST_TYPE_FRACTION, 30, 1, NULL);
    GstCapsFeatures *feature = gst_caps_features_new ("memory:NVMM", NULL);
    gst_caps_set_features (caps, 0, feature);

    /* Build the pipeline without linking source with the rest of the elements */
    gst_bin_add_many (GST_BIN (pipeline), source, convert, queue, encoder, parser, sink, NULL);
    if (!gst_element_link_filtered (convert, queue, caps))
        g_printerr ("converter1 and queue not linked.\n");
    if (!gst_element_link_filtered (queue, encoder, caps))
        g_printerr ("queue and encoder not linked.\n");
    if (!gst_element_link (encoder, parser))
        g_printerr ("encoder and parser not linked.\n");
    if (!gst_element_link (parser, sink))
        g_printerr ("parser and sink not linked.\n");

and these are the warnings I receive, which come from the attempt to dynamic link source and converter. The second time the linking is succesfull.

Running...
In cb_newpad

(example2:20174): GStreamer-CRITICAL **: 15:13:36.598: gst_caps_get_size: assertion 'GST_IS_CAPS (caps)' failed

(example2:20174): GStreamer-CRITICAL **: 15:13:36.598: gst_caps_get_structure: assertion 'GST_IS_CAPS (caps)' failed

(example2:20174): GStreamer-CRITICAL **: 15:13:36.598: gst_structure_get_name: assertion 'structure != NULL' failed
In cb_newpad

(example2:20174): GStreamer-CRITICAL **: 15:13:40.915: gst_caps_get_size: assertion 'GST_IS_CAPS (caps)' failed

(example2:20174): GStreamer-CRITICAL **: 15:13:40.915: gst_caps_get_structure: assertion 'GST_IS_CAPS (caps)' failed

(example2:20174): GStreamer-CRITICAL **: 15:13:40.915: gst_structure_get_name: assertion 'structure != NULL' failed
Link succeeded (source is 'video/x-raw' and sink is '(null)').

Any understanding of what is going on?

Thanks again,

f

Hi,
Please try

... ! nvv4l2h264enc ! h264parse ! matroskamux ! filesink

or

... ! nvv4l2h264enc ! h264parse ! video/x-h264,stream-format=byte-stream ! filesink