This is a working solution.
The issues I encountered:
- crashes with missing PTS: after adding probes to each pad along the pipeline and checking if the PTS is valid, I find some frames have duplicated PTS from my appsrc.
- GRAY8 to NV12 conversion with nvvidconv: after checking the source code of nvvidconv I find this conversion is not supported.
gchararray FormatLocation(GstElement* splitmux, guint, gpointer) {
gchararray location;
g_object_get(G_OBJECT(splitmux), "location", &location, nullptr);
std::string location_str =
absl::StrCat(location, "_",
std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count(),
".mp4");
return g_strdup(location_str.c_str());
}
class Encoder {
public:
explicit Encoder(const std::string& format, int width, int height,
int framerate_numerator, int framerate_denominator,
const std::string& filesink_location,
const std::string& nvvidconv_output_format) {
pipeline_ = gst_pipeline_new("pipeline");
appsrc_ = gst_element_factory_make("appsrc", "appsrc");
g_object_set(G_OBJECT(appsrc_), "format", GST_FORMAT_TIME, nullptr);
gst_app_src_set_stream_type(GST_APP_SRC(appsrc_),
GST_APP_STREAM_TYPE_STREAM);
gst_app_src_set_caps(
GST_APP_SRC(appsrc_),
gst_caps_new_simple("video/x-raw", "format", G_TYPE_STRING,
format.c_str(), "width", G_TYPE_INT, width,
"height", G_TYPE_INT, height, "framerate",
GST_TYPE_FRACTION, framerate_numerator,
framerate_denominator, nullptr));
// To extract the subtitle:
// ffmpeg -i output00.mp4 -map 0:s:0 subs.srt
// To view PTS:
// ffmpeg -i output00.mp4 -vf "showinfo" -f null -
subtitle_appsrc_ =
gst_element_factory_make("appsrc", "subtitle_appsrc");
g_object_set(G_OBJECT(subtitle_appsrc_), "format", GST_FORMAT_TIME,
nullptr);
gst_app_src_set_stream_type(GST_APP_SRC(subtitle_appsrc_),
GST_APP_STREAM_TYPE_STREAM);
gst_app_src_set_caps(
GST_APP_SRC(subtitle_appsrc_),
gst_caps_new_simple("text/x-raw", "format", G_TYPE_STRING, "utf8",
nullptr));
auto* nvvidconv = gst_element_factory_make("nvvidconv", "nvvidconv");
auto* nvv4l2h265enc =
gst_element_factory_make("nvv4l2h265enc", "nvv4l2h265enc");
g_object_set(G_OBJECT(nvv4l2h265enc), "maxperf-enable", TRUE,
"control-rate", 0, "bitrate", 1'000'000, "peak-bitrate",
8'000'000, nullptr);
auto* h265parse = gst_element_factory_make("h265parse", "h265parse");
auto* mp4mux = gst_element_factory_make("mp4mux", "mp4mux");
g_object_set(G_OBJECT(mp4mux), "fragment-duration", 1000, nullptr);
auto* splitmuxsink =
gst_element_factory_make("splitmuxsink", "splitmuxsink");
g_object_set(G_OBJECT(splitmuxsink), "location",
filesink_location.c_str(), "max-size-bytes", 100'000'000,
"muxer", mp4mux, "max-files", 3, nullptr);
g_signal_connect(splitmuxsink, "format-location",
G_CALLBACK(FormatLocation), nullptr);
gst_bin_add_many(GST_BIN(pipeline_), appsrc_, nvvidconv, nvv4l2h265enc,
h265parse, subtitle_appsrc_, splitmuxsink, nullptr);
CHECK(gst_element_link_filtered(
nvvidconv, nvv4l2h265enc,
gst_caps_from_string(
absl::StrCat("video/x-raw(memory:NVMM),", "format=", "(string)",
nvvidconv_output_format.c_str(), ",width=",
"(int)", width, ",height=", "(int)", height)
.c_str())));
CHECK(gst_element_link(appsrc_, nvvidconv));
CHECK(gst_element_link_pads(subtitle_appsrc_, nullptr, splitmuxsink,
"subtitle_%u"));
CHECK(gst_element_link(nvv4l2h265enc, h265parse));
CHECK(gst_element_link_pads(h265parse, "src", splitmuxsink, "video"));
CHECK(gst_element_set_state(pipeline_, GST_STATE_PLAYING) !=
GST_STATE_CHANGE_FAILURE);
}
void Push(GstBuffer* buffer, GstBuffer* subtitle_buffer = nullptr) {
int64_t pts = GST_BUFFER_PTS(buffer);
if (pts < last_pts_) {
LOG(ERROR) << "Out of order " << pts;
gst_buffer_unref(buffer);
} else if (pts == last_pts_) {
LOG(WARNING) << "Same as the last one " << pts;
gst_buffer_unref(buffer);
} else {
last_pts_ = pts;
gst_app_src_push_buffer(GST_APP_SRC(appsrc_), buffer);
if (subtitle_buffer) {
gst_app_src_push_buffer(GST_APP_SRC(subtitle_appsrc_),
subtitle_buffer);
}
}
}
~Encoder() {
gst_app_src_end_of_stream(GST_APP_SRC(appsrc_));
gst_app_src_end_of_stream(GST_APP_SRC(subtitle_appsrc_));
/* Wait until error or EOS */
auto* bus = gst_element_get_bus(pipeline_);
auto* msg = gst_bus_timed_pop_filtered(
bus, GST_CLOCK_TIME_NONE,
static_cast<GstMessageType>(GST_MESSAGE_ERROR | GST_MESSAGE_EOS));
if (GST_MESSAGE_TYPE(msg) == GST_MESSAGE_ERROR) {
g_error(
"An error occurred! Re-run with the GST_DEBUG=*:WARN "
"environment "
"variable set for more details.");
}
gst_message_unref(msg);
gst_object_unref(bus);
gst_element_set_state(pipeline_, GST_STATE_NULL);
gst_object_unref(pipeline_);
}
private:
GstElement* appsrc_;
GstElement* subtitle_appsrc_;
GstElement* pipeline_;
int64_t last_pts_ = 0;
};