File loop failed after use nvstreammux

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) GPU
• DeepStream Version 6.1
• JetPack Version (valid for Jetson only)
• TensorRT Version
• NVIDIA GPU Driver Version (valid for GPU only) 510.47.03
• Issue Type( questions, new requirements, bugs) questions
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)

I’m trying to develop a gstreamer pipeline with deepstream plugin and since the input is mp4 file, so I want enable infinite loop. The pipeline is like following:

gst-launch-1.0 uridecodebin uri="file:///home/user/test.mp4" ! nvstreammux ! tee name=tee0 ! queue ! tee name=t1 ! queue ! nveglglessink tee1. ! queue ! nvvideoconvert ! video/x-raw(memory:NVMM),format=NV12 ! nvv4l2h264enc bitrate=4000000 iframeinterval=30 ! flvmux ! rtmpsink location=rtmp://127.0.0.1:1935/live/test tee0. ! queue ! nvvideoconvert ! video/x-raw(memory:NVMM),format=RGBA ! appsink

I implement the file loop trick from /opt/nvidia/deepstream/deepstream-6.1/sources/apps/apps-common/src/deepstream_source_bin.c, like following:

static gboolean cb_seek_decoded_file(gpointer user_data)
{
    VideoPipeline* vp = static_cast<VideoPipeline*>(user_data);

    LOG_INFO("============================================");
    LOG_INFO("cb_seek_decoded_file called({})", pipeline_id);
    LOG_INFO("============================================");

    gst_element_set_state(vp->m_pipeline, GST_STATE_PAUSED);

    if (!gst_element_seek(vp->m_pipeline, 1.0, GST_FORMAT_TIME,
        (GstSeekFlags)(GST_SEEK_FLAG_KEY_UNIT | GST_SEEK_FLAG_FLUSH),
        GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE)) {
        LOG_WARN("Failed to seed the source file in pipeline");
    }

    gst_element_set_state(vp->m_pipeline, GST_STATE_PLAYING);

    return false;
}

static GstPadProbeReturn cb_reset_stream_probe(
    GstPad* pad,
    GstPadProbeInfo* info,
    gpointer user_data)
{
    VideoPipeline* vp = static_cast<VideoPipeline*>(user_data);
    GstEvent* event = GST_EVENT(info->data);

    if (info->type & GST_PAD_PROBE_TYPE_BUFFER) {
        GST_BUFFER_PTS(GST_BUFFER(info->data)) += vp->m_prev_accumulated_base;
    }

    if (info->type & GST_PAD_PROBE_TYPE_EVENT_BOTH) {
        if (GST_EVENT_TYPE(event) == GST_EVENT_EOS) {
            g_timeout_add(1, cb_seek_decoded_file, vp);
        }

        if (GST_EVENT_TYPE(event) == GST_EVENT_SEGMENT) {
            GstSegment *segment;
            gst_event_parse_segment(event, (const GstSegment **) &segment);
            segment->base = vp->m_accumulated_base;
            vp->m_prev_accumulated_base = vp->m_accumulated_base;
            vp->m_accumulated_base += segment->stop;
        }
        
        switch(GST_EVENT_TYPE(event)) {
            case GST_EVENT_EOS:
            case GST_EVENT_QOS:
            case GST_EVENT_SEGMENT:
            case GST_EVENT_FLUSH_START:
            case GST_EVENT_FLUSH_STOP:
                return GST_PAD_PROBE_DROP;
            default:
                break;
        }
    }

    return GST_PAD_PROBE_OK;
}

static void cb_decodebin_child_added(GstChildProxy* child_proxy, GObject* object,
    gchar* name, gpointer user_data)
{
    VideoPipeline* vp = static_cast<VideoPipeline*>(user_data);

    LOG_INFO("cb_decodebin_child_added called({},'{}' added)", pipeline_id, name);

    if (g_strrstr(name, "nvv4l2decoder") == name) {
        g_object_set(object, "cudadec-memtype", 2, nullptr);

        if (g_strstr_len(vp->m_config.src_uri.c_str(), -1, "file:/") ==
            vp->m_config.src_uri.c_str() && vp->m_config.file_loop) {
            GstPad* gst_pad = gst_element_get_static_pad(GST_ELEMENT(object), "sink");
            vp->m_dec_sink_probe = gst_pad_add_probe(gst_pad, (GstPadProbeType)(
                GST_PAD_PROBE_TYPE_EVENT_BOTH | GST_PAD_PROBE_TYPE_EVENT_FLUSH |
                GST_PAD_PROBE_TYPE_BUFFER), cb_reset_stream_probe, static_cast<void*>(vp), nullptr);
            gst_object_unref(gst_pad);

            vp->m_decoder = GST_ELEMENT(object);
            gst_object_ref(object);
        } else if (g_strstr_len(vp->m_config.src_uri.c_str(), -1, "rtsp:/") ==
            vp->m_config.src_uri.c_str()) {
            vp->m_decoder = GST_ELEMENT(object);
            gst_object_ref(object);
        }
    } else if ((g_strrstr(name, "h264parse") == name) ||
            (g_strrstr(name, "h265parse") == name)) {
        LOG_INFO("set config-interval of {} to {}", name, -1);
        g_object_set(object, "config-interval", -1, nullptr);
    }

done:
    return;
}

But if I use nvstreammux in my pipeline, file loop seek event will fail.
And the problem is I also need to access the GstBuffer via the GstSample output by appsink. But the only way I know is that I need to use nvstreammux to hold a NvBufSurface structure so I can map the data pointer. The callback function of appsink ‘new-sample’ signal is like following:

static GstFlowReturn cb_appsink_new_sample(
    GstElement* appsink,
    gpointer user_data)
{
    // LOG_INFO("cb_appsink_new_sample called");

    VideoPipeline* vp = static_cast<VideoPipeline*>(user_data);
    GstSample* sample = nullptr;
    const GstStructure* info = nullptr;
    GstBuffer* buffer = nullptr;
    GstMapInfo map;
    GstCaps* caps = nullptr;
    int sample_width = 0;
    int sample_height = 0;
    const char* sample_format;
    NvBufSurface* surface;
    NvDsMetaList* l_frame;
    NvDsBatchMeta* batch_meta;

    if (!vp->m_dumped) {
        GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(vp->m_pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "video-pipeline");
        vp->m_dumped = true;
    }

    g_signal_emit_by_name(appsink, "pull-sample", &sample);
    if (!sample) {
        return GST_FLOW_OK;
    }

    // if (vp->m_putFrameFunc) {
    //     vp->m_putFrameFunc(sample, vp->m_putFrameArgs);
    // } else {
    //     gst_sample_unref(sample);
    // }

    buffer = gst_sample_get_buffer(sample);
    if (buffer == nullptr) {
        LOG_ERROR("Can't get buffer from sample.");
        goto err;
    }
    gst_buffer_map(buffer, &map, GST_MAP_READ);

    caps = gst_sample_get_caps(sample);
    if (caps == nullptr) {
        LOG_ERROR("Can't get caps from sample.");
        goto err;
    }

    info = gst_caps_get_structure(caps, 0);
    if (info == nullptr) {
        LOG_ERROR("Can't get info from sample.");
        goto err;
    }
    
    // appsink algorithm productor queue produce
    {
        // init a tmpMat with gst buffer address: deep copy
        batch_meta = gst_buffer_get_nvds_batch_meta(buffer);
        surface = (NvBufSurface *)map.data;

        uint32_t frame_width, frame_height, frame_pitch;
        for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) {

            NvDsFrameMeta *frame_meta = (NvDsFrameMeta*)(l_frame->data);
            frame_width = surface->surfaceList[frame_meta->batch_id].width;
            frame_height = surface->surfaceList[frame_meta->batch_id].height;
            frame_pitch = surface->surfaceList[frame_meta->batch_id].pitch;
            if (NvBufSurfaceMap(surface, 0, 0, NVBUF_MAP_READ_WRITE)) {
                LOG_ERROR("NVMM map failed.");
                goto err;
            }

            cv::Mat tmpMat(frame_height, frame_width, CV_8UC4, (unsigned char*)surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0], frame_pitch);
            tmpMat = tmpMat.clone();
            // cv::imwrite("appsink.jpg", tmpMat);
            vp->m_productQueue->product(std::make_shared<cv::Mat>(tmpMat));
        }
        NvBufSurfaceUnMap(surface, 0, 0);
    }

err:
    if (buffer) {
        gst_buffer_unmap(buffer, &map);
    }
    if (sample) {
        gst_sample_unref(sample);
    }
    return GST_FLOW_OK;
}

So anything wrong with my pipeline or codes? Or whether there is another way to access the GstBuffer data output by appsink?
Thanks.

The logs output when gst_element_seek() called between whether use nvstreammux or not are here:

Pipeline without nvstreammux:

:01:02.751555378 16380 0x55a167818200 INFO              GST_STATES gstelement.c:2675:gst_element_continue_state:<source> completed state change to PAUSED
0:01:02.751564452 16380 0x55a167818200 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<source> notifying about state-changed PLAYING to PAUSED (VOID_PENDING pending)
0:01:02.751575298 16380 0x55a167818200 INFO              GST_STATES gstbin.c:2952:gst_bin_change_state_func:<uridecodebin0> child 'source' changed state to 3(PAUSED) successfully
0:01:02.751584181 16380 0x55a167818200 INFO              GST_STATES gstelement.c:2675:gst_element_continue_state:<uridecodebin0> completed state change to PAUSED
0:01:02.751593038 16380 0x55a167818200 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<uridecodebin0> notifying about state-changed PLAYING to PAUSED (VOID_PENDING pending)
0:01:02.751603522 16380 0x55a167818200 INFO              GST_STATES gstbin.c:2952:gst_bin_change_state_func:<video-pipeline> child 'uridecodebin0' changed state to 3(PAUSED) successfully
0:01:02.751625933 16380 0x55a167818200 INFO               GST_EVENT gstevent.c:1287:gst_event_new_seek: creating seek rate 1.000000, format TIME, flags 5, start_type 1, start 0:00:00.000000000, stop_type 0, stop 99:99:99.999999999
0:01:02.751667556 16380 0x55a167818200 INFO                 qtdemux qtdemux.c:1804:gst_qtdemux_handle_src_event:<qtdemux0> Time taken to parse index 0:00:00.000001512
0:01:02.751728330 16380 0x7efc3c07a060 INFO                    task gsttask.c:312:gst_task_func:<multiqueue0:src_0> Task going to paused
0:01:02.751800956 16380 0x7efc3c07a1e0 INFO                    task gsttask.c:312:gst_task_func:<multiqueue0:src_1> Task going to paused
0:01:02.751806503 16380 0x55a167818200 INFO                 default gstsegment.c:378:gst_segment_do_seek: segment updated: time segment start=0:00:00.000000000, offset=0:00:00.000000000, stop=0:01:02.550000000, rate=1.000000, applied_rate=1.000000, flags=0x01, time=0:00:00.000000000, base=0:00:00.000000000, position 0:00:00.000000000, duration 0:01:02.550000000
0:01:02.751819224 16380 0x55a167a6ede0 INFO              GST_STATES gstbin.c:3421:bin_handle_async_done:<video-pipeline> committing state from PLAYING to PAUSED, old pending PAUSED
0:01:02.751837931 16380 0x55a167818200 INFO               GST_EVENT gstevent.c:557:gst_event_new_flush_stop: creating flush stop 1
0:01:02.751847797 16380 0x55a167a6ede0 INFO              GST_STATES gstbin.c:3444:bin_handle_async_done:<video-pipeline> completed state change, pending VOID
0:01:02.751866561 16380 0x55a167a6ede0 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<video-pipeline> notifying about state-changed PLAYING to PAUSED (VOID_PENDING pending)
0:01:02.751882696 16380 0x7efc3c07a060 INFO                    task gsttask.c:314:gst_task_func:<multiqueue0:src_0> Task resume from paused
0:01:02.751921525 16380 0x55a167a6f300 INFO                    task gsttask.c:314:gst_task_func:<qtdemux0:sink> Task resume from paused
0:01:02.751952254 16380 0x7efc3c07a1e0 INFO                    task gsttask.c:314:gst_task_func:<multiqueue0:src_1> Task resume from paused
0:01:02.751978047 16380 0x55a167a6f300 INFO               GST_EVENT gstevent.c:900:gst_event_new_segment: creating segment event time segment start=0:00:00.066600000, offset=0:00:00.000000000, stop=0:01:02.587600000, rate=1.000000, applied_rate=1.000000, flags=0x01, time=0:00:00.000000000, base=0:00:00.000000000, position 0:00:00.066600000, duration 99:99:99.999999999
0:01:02.752050648 16380 0x55a167818200 INFO               GST_EVENT gstevent.c:1449:gst_event_new_latency: creating latency event 0:00:00.000000000
0:01:02.752083819 16380 0x55a167a6f300 INFO               GST_EVENT gstevent.c:900:gst_event_new_segment: creating segment event time segment start=0:00:00.000000000, offset=0:00:00.000000000, stop=0:01:02.550000000, rate=1.000000, applied_rate=1.000000, flags=0x01, time=0:00:00.000000000, base=0:00:00.000000000, position 0:00:00.000000000, duration 99:99:99.999999999
0:01:02.752088001 16380 0x55a167818200 INFO                     bin gstbin.c:2783:gst_bin_do_latency_func:<video-pipeline> configured latency of 0:00:00.000000000
0:01:02.752111214 16380 0x55a167818200 INFO              GST_STATES gstbin.c:2503:gst_bin_element_set_state:<appsink> current PAUSED pending VOID_PENDING, desired next PLAYING
0:01:02.752122189 16380 0x55a167818200 INFO              GST_STATES gstelement.c:2675:gst_element_continue_state:<appsink> completed state change to PLAYING
0:01:02.752130005 16380 0x55a167818200 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<appsink> notifying about state-changed PAUSED to PLAYING (VOID_PENDING pending)
0:01:02.752140306 16380 0x55a167818200 INFO              GST_STATES gstbin.c:2952:gst_bin_change_state_func:<video-pipeline> child 'appsink' changed state to 4(PLAYING) successfully
0:01:02.752148696 16380 0x55a167818200 INFO              GST_STATES gstbin.c:2503:gst_bin_element_set_state:<nveglglessink0> current PAUSED pending VOID_PENDING, desired next PLAYING

Pipeline with nvstreammux:

0:01:03.494638683 16719 0x55c7a7af6600 INFO              GST_STATES gstelement.c:2675:gst_element_continue_state:<source> completed state change to PAUSED
0:01:03.494645691 16719 0x55c7a7af6600 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<source> notifying about state-changed PLAYING to PAUSED (VOID_PENDING pending)
0:01:03.494652994 16719 0x55c7a7af6600 INFO              GST_STATES gstbin.c:2952:gst_bin_change_state_func:<uridecodebin0> child 'source' changed state to 3(PAUSED) successfully
0:01:03.494658303 16719 0x55c7a7af6600 INFO              GST_STATES gstelement.c:2675:gst_element_continue_state:<uridecodebin0> completed state change to PAUSED
0:01:03.494663595 16719 0x55c7a7af6600 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<uridecodebin0> notifying about state-changed PLAYING to PAUSED (VOID_PENDING pending)
0:01:03.494669640 16719 0x55c7a7af6600 INFO              GST_STATES gstbin.c:2952:gst_bin_change_state_func:<video-pipeline> child 'uridecodebin0' changed state to 3(PAUSED) successfully
0:01:03.494688663 16719 0x55c7a7af6600 INFO               GST_EVENT gstevent.c:1287:gst_event_new_seek: creating seek rate 1.000000, format TIME, flags 5, start_type 1, start 0:00:00.000000000, stop_type 0, stop 99:99:99.999999999
0:01:03.497300253 16719 0x55c7a7d492a0 INFO              GST_STATES gstbin.c:3421:bin_handle_async_done:<video-pipeline> committing state from PLAYING to PAUSED, old pending PLAYING
0:01:03.497330456 16719 0x55c7a7d492a0 INFO              GST_STATES gstbin.c:3452:bin_handle_async_done:<video-pipeline> continue state change, pending PLAYING
0:01:03.497346304 16719 0x55c7a7d492a0 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<video-pipeline> notifying about state-changed PLAYING to PAUSED (PLAYING pending)
0:01:03.497529680 16719 0x7f7b7400a700 INFO              GST_STATES gstbin.c:3248:gst_bin_continue_func:<video-pipeline> continue state change PAUSED to PLAYING, final PLAYING
0:01:03.497676493 16719 0x7f7b7400a700 INFO               GST_EVENT gstevent.c:1449:gst_event_new_latency: creating latency event 0:00:00.000000000
0:01:03.497701447 16719 0x7f7b7400a700 INFO                     bin gstbin.c:2783:gst_bin_do_latency_func:<video-pipeline> configured latency of 0:00:00.000000000
0:01:03.497722424 16719 0x7f7b7400a700 INFO              GST_STATES gstbin.c:2503:gst_bin_element_set_state:<appsink> current PAUSED pending VOID_PENDING, desired next PLAYING
0:01:03.497733465 16719 0x7f7b7400a700 INFO              GST_STATES gstelement.c:2675:gst_element_continue_state:<appsink> completed state change to PLAYING
0:01:03.497740729 16719 0x7f7b7400a700 INFO              GST_STATES gstelement.c:2575:_priv_gst_element_state_changed:<appsink> notifying about state-changed PAUSED to PLAYING (VOID_PENDING pending)
0:01:03.497755657 16719 0x7f7b7400a700 INFO              GST_STATES gstbin.c:2952:gst_bin_change_state_func:<video-pipeline> child 'appsink' changed state to 4(PLAYING) successfully
0:01:03.497763003 16719 0x7f7b7400a700 INFO              GST_STATES gstbin.c:2503:gst_bin_element_set_state:<nveglglessink0> current PAUSED pending VOID_PENDING, desired next PLAYING

Obviously the second log is less than the first one. I have no ideas about why the second one didn’t output gst_segment_do_seek().

You can refer to our demo code to implement file loop: https://github.com/NVIDIA-AI-IOT/deepstream_tao_apps/blob/master/apps/tao_detection/deepstream_det_app.c#L345

Thanks a lot, I imitate this demo and only change the state of uridecodebin rather than the whole pipeline. It worked successfully.

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.