Current Issues:
- In complex scenes, if the quality of the I-frames is ensured, the I-frames become too small while the P-frames become too large. This manifests in the image as a blinking phenomenon where the I-frames appear blurry and the P-frames are clear.
- Under the current low bit rate requirements of 1080p at 3mbps and 1280*1024 at 2mbps, there is excessive detail loss, leading to blurred image details. We aim to maintain clarity while ensuring a stable bit rate.
- The current manual only describes the use and individual effects of each API. There is no information on the combined effects of various APIs and parameters, nor are there any related cases or examples for reference. Could we have more detailed documentation?
Below is the source code
#ifndef ENCODER_PARAM_HPP
#define ENCODER_PARAM_HPP
typedef struct
{
NvVideoEncoder *enc;
uint32_t encoder_pixfmt;
uint32_t raw_pixfmt;
char *in_file_path;
std::ifstream *in_file;
uint32_t width;
uint32_t height;
char *out_file_path;
std::ofstream *out_file;
char *ROI_Param_file_path;
char *Recon_Ref_file_path;
char *RPS_Param_file_path;
char *hints_Param_file_path;
char *GDR_Param_file_path;
char *GDR_out_file_path;
std::ifstream *roi_Param_file;
std::ifstream *recon_Ref_file;
std::ifstream *rps_Param_file;
std::ifstream *hints_Param_file;
std::ifstream *gdr_Param_file;
std::ofstream *gdr_out_file;
uint32_t bitrate;
uint32_t peak_bitrate;
uint32_t profile;
enum v4l2_mpeg_video_bitrate_mode ratecontrol;
uint32_t iframe_interval;
uint32_t idr_interval;
uint32_t level;
uint32_t fps_n;
uint32_t fps_d;
uint32_t gdr_start_frame_number; /* Frame number where GDR has to be started */
uint32_t gdr_num_frames; /* Number of frames where GDR to be applied */
uint32_t gdr_out_frame_number; /* Frames number from where encoded buffers are to be dumped */
enum v4l2_enc_temporal_tradeoff_level_type temporal_tradeoff_level;
enum v4l2_enc_hw_preset_type hw_preset_type;
v4l2_enc_slice_length_type slice_length_type;
uint32_t slice_length;
uint32_t virtual_buffer_size;
uint32_t num_reference_frames;
uint32_t slice_intrarefresh_interval;
uint32_t num_b_frames;
uint32_t nMinQpI; /* Minimum QP value to use for index frames */
uint32_t nMaxQpI; /* Maximum QP value to use for index frames */
uint32_t nMinQpP; /* Minimum QP value to use for P frames */
uint32_t nMaxQpP; /* Maximum QP value to use for P frames */
uint32_t nMinQpB; /* Minimum QP value to use for B frames */
uint32_t nMaxQpB; /* Maximum QP value to use for B frames */
uint32_t sMaxQp; /* Session Maximum QP value */
uint32_t sar_width;
uint32_t sar_height;
uint32_t IinitQP;
uint32_t PinitQP;
uint32_t BinitQP;
uint32_t log2_num_av1rows;
uint32_t log2_num_av1cols;
uint8_t bit_depth;
uint8_t enable_av1ssimrdo;
uint8_t disable_av1cdfupdate;
uint8_t chroma_format_idc;
int output_plane_fd[32];
bool insert_sps_pps_at_idr;
bool enable_slice_level_encode;
bool disable_cabac;
bool insert_vui;
bool enable_extended_colorformat;
bool insert_aud;
bool alliframes;
bool is_semiplanar;
bool enable_initQP;
bool enable_ratecontrol;
bool enable_av1tile;
enum v4l2_memory output_memory_type;
enum v4l2_colorspace cs;
bool report_metadata;
bool input_metadata;
bool copy_timestamp;
uint32_t start_ts;
bool dump_mv;
bool enableGDR;
bool bGapsInFrameNumAllowed;
bool bnoIframe;
uint32_t nH264FrameNumBits;
uint32_t nH265PocLsbBits;
bool externalRCHints;
bool enableROI;
bool b_use_enc_cmd;
bool enableLossless;
bool got_eos;
bool externalRPS;
bool RPS_threeLayerSvc;
RPS_param rps_par;
bool use_gold_crc;
char gold_crc[20];
Crc *pBitStreamCrc;
bool bReconCrc;
uint32_t rl; /* Reconstructed surface Left cordinate */
uint32_t rt; /* Reconstructed surface Top cordinate */
uint32_t rw; /* Reconstructed surface width */
uint32_t rh; /* Reconstructed surface height */
uint64_t timestamp;
uint64_t timestampincr;
bool stats;
std::stringstream *runtime_params_str;
uint32_t next_param_change_frame;
bool got_error;
int stress_test;
uint32_t endofstream_capture;
uint32_t endofstream_output;
uint32_t input_frames_queued_count;
uint32_t startf;
uint32_t endf;
uint32_t num_output_buffers;
int32_t num_frames_to_encode;
uint32_t poc_type;
v4l2_enc_ppe_init_params ppe_init_params; // Configuration params for preprocessing enhancements module
int max_perf;
int blocking_mode; // Set if running in blocking mode
sem_t pollthread_sema; // Polling thread waits on this to be signalled to issue Poll
sem_t encoderthread_sema; // Encoder thread waits on this to be signalled to continue q/dq loop
pthread_t enc_pollthread; // Polling thread, created if running in non-blocking mode.
pthread_t enc_capture_loop; // Encoder capture thread
} context_t;
static context_t ctx;
int InitNvEncoder(const char *name)
{
int ret;
ctx.enc = NvVideoEncoder::createVideoEncoder(name);
// no blocking
// ctx.enc = NvVideoEncoder::createVideoEncoder("enc0", O_NONBLOCK);
if (!ctx.enc)
{
MLOG(info, "can't create enc");
return -1;
}
/* Set encoder capture plane format.
NOTE: It is necessary that Capture Plane format be set before Output Plane
format. It is necessary to set width and height on the capture plane as well */
ret = ctx.enc->setCapturePlaneFormat(ctx.encoder_pixfmt, ctx.width, ctx.height, 2 * 1024 * 1024);
if (ret < 0)
{
MLOG(info, "Could not set capture plane format");
return -1;
}
ctx.raw_pixfmt = V4L2_PIX_FMT_YUV420M;
/* Set encoder output plane format */
ret = ctx.enc->setOutputPlaneFormat(ctx.raw_pixfmt, ctx.width, ctx.height);
if (ret < 0)
{
MLOG(info, "Could not set output plane format");
return -1;
}
ret = ctx.enc->setBitrate(ctx.bitrate);
if (ret < 0)
{
MLOG(info, "Could not set encoder bitrate");
return -1;
}
ret = ctx.enc->setPeakBitrate(ctx.peak_bitrate);
if (ret < 0)
{
MLOG(info, "Could not set peak bitrate");
}
/* Set encoder profile for H264 format */
ret = ctx.enc->setProfile(ctx.profile);
if (ret < 0)
{
MLOG(info, "Could not set encoder profile");
return -1;
}
// /* Set hardware preset value for encoder */
ret = ctx.enc->setHWPresetType(ctx.hw_preset_type);
if (ret < 0)
{
MLOG(info, "setHWPresetType failed");
return -1;
}
ret = ctx.enc->setLevel(ctx.level);
if (ret < 0)
{
MLOG(info, "Could not set encoder level");
return -1;
}
ret = ctx.enc->setRateControlMode(ctx.ratecontrol);
if (ret < 0)
{
MLOG(info, "Could not set encoder rate control mode");
return -1;
}
/* Set IDR frame interval for encoder */
ret = ctx.enc->setIDRInterval(ctx.idr_interval);
if (ret < 0)
{
MLOG(info, "Could not set encoder IDR interval");
return -1;
}
ret = ctx.enc->setVirtualBufferSize(1000000);
if (ret < 0)
{
MLOG(info, "Could not set virtual buffer size");
return -1;
}
/* Set I frame interval for encoder */
ret = ctx.enc->setIFrameInterval(ctx.iframe_interval);
if (ret < 0)
{
MLOG(info, "Could not set encoder I-Frame interval");
return -1;
}
ret = ctx.enc->setInsertVuiEnabled(true);
if (ret < 0)
{
MLOG(info, "Could not setInsertVuiEnabled");
return -1;
}
/* Set framerate for encoder */
ret = ctx.enc->setFrameRate(ctx.fps_n, ctx.fps_d);
if (ret < 0)
{
MLOG(info, "Could not set framerate");
return -1;
}
if ((ctx.nMinQpI != (uint32_t)QP_RETAIN_VAL) ||
(ctx.nMaxQpI != (uint32_t)QP_RETAIN_VAL) ||
(ctx.nMinQpP != (uint32_t)QP_RETAIN_VAL) ||
(ctx.nMaxQpP != (uint32_t)QP_RETAIN_VAL) ||
(ctx.nMinQpB != (uint32_t)QP_RETAIN_VAL) ||
(ctx.nMaxQpB != (uint32_t)QP_RETAIN_VAL))
{
/* Set Min & Max qp range values for I/P/B-frames to be used by encoder */
ret = ctx.enc->setQpRange(ctx.nMinQpI, ctx.nMaxQpI, ctx.nMinQpP,
ctx.nMaxQpP, ctx.nMinQpB, ctx.nMaxQpB);
if (ret < 0)
{
MLOG(info, "Could not set quantization parameters");
return -1;
}
}
/* Enable insert of SPSPPS at IDR frames */
ret = ctx.enc->setInsertSpsPpsAtIdrEnabled(true);
if (ret < 0)
{
MLOG(info, "Could not set insertSPSPPSAtIDR");
return -1;
}
// if (ctx.slice_intrarefresh_interval)
// {
// /* Set slice intra refresh interval value for encoder */
// ret = ctx.enc->setSliceIntrarefresh(ctx.slice_intrarefresh_interval);
// if (ret < 0)
// {
// LOG_INFO("setSliceIntrarefresh error\n");
// return -1;
// }
// }
/* Query, Export and Map the output plane buffers so that we can read
raw data into the buffers */
switch (ctx.output_memory_type)
{
case V4L2_MEMORY_MMAP:
ret = ctx.enc->output_plane.setupPlane(V4L2_MEMORY_MMAP, 10, true, false);
if (ret < 0)
{
MLOG(info, "Could not setup output plane");
return -1;
}
break;
case V4L2_MEMORY_USERPTR:
ret = ctx.enc->output_plane.setupPlane(V4L2_MEMORY_USERPTR, 10, false, true);
if (ret < 0)
{
MLOG(info, "Could not setup output plane");
return -1;
}
break;
case V4L2_MEMORY_DMABUF:
ret = setup_output_dmabuf(&ctx, 10);
if (ret < 0)
{
MLOG(info, "Could not setup output plane");
return -1;
}
break;
default:
MLOG(info, "Not a valid plane");
return -1;
};
/* Query, Export and Map the capture plane buffers so that we can write
encoded bitstream data into the buffers */
ret = ctx.enc->capture_plane.setupPlane(V4L2_MEMORY_MMAP, ctx.num_output_buffers, true, false);
if (ret < 0)
{
MLOG(info, "Could not setup capture plane");
return -1;
}
/* Subscibe for End Of Stream event */
ret = ctx.enc->subscribeEvent(V4L2_EVENT_EOS, 0, 0);
if (ret < 0)
{
MLOG(info, "Could not subscribe EOS event");
return -1;
}
/* set encoder output plane STREAMON */
ret = ctx.enc->output_plane.setStreamStatus(true);
if (ret < 0)
{
MLOG(info, "Error in output plane streamon");
return -1;
}
/* set encoder capture plane STREAMON */
ret = ctx.enc->capture_plane.setStreamStatus(true);
if (ret < 0)
{
MLOG(info, "Error in capture plane streamon");
return -1;
}
/* Set encoder capture plane dq thread callback for blocking io mode */
bool isSuccess = ctx.enc->capture_plane.setDQThreadCallback(encoder_capture_plane_dq_callback);
if (!isSuccess)
{
MLOG(info, "setDQThreadCallback error");
return -1;
}
/* startDQThread starts a thread internally which calls the
encoder_capture_plane_dq_callback whenever a buffer is dequeued
on the plane */
ret = ctx.enc->capture_plane.startDQThread(this);
if (ret < 0)
{
MLOG(info, "startDQThread error");
return -1;
}
/* Enqueue all the empty capture plane buffers. */
for (uint32_t i = 0; i < ctx.enc->capture_plane.getNumBuffers(); i++)
{
struct v4l2_buffer v4l2_buf;
struct v4l2_plane planes[MAX_PLANES];
memset(&v4l2_buf, 0, sizeof(v4l2_buf));
memset(planes, 0, MAX_PLANES * sizeof(struct v4l2_plane));
v4l2_buf.index = i;
v4l2_buf.m.planes = planes;
ret = ctx.enc->capture_plane.qBuffer(v4l2_buf, NULL);
if (ret < 0)
{
MLOG(info, "Error while queueing buffer at capture plane");
abort(&ctx);
return -1;
}
}
return 0;
}
void set_defaults()
{
memset(ctx, 0, sizeof(context_t));
ctx->raw_pixfmt = V4L2_PIX_FMT_YUV420M;
ctx->bitrate = 4 * 1024 * 1024;
ctx->peak_bitrate = 0;
ctx->profile = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE;
ctx->ratecontrol = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR;
ctx->fps_n = 25;
ctx->iframe_interval = ctx->fps_n;
ctx->externalRPS = false;
ctx->enableGDR = false;
ctx->enableROI = false;
ctx->bnoIframe = false;
ctx->bGapsInFrameNumAllowed = false;
ctx->bReconCrc = false;
ctx->enableLossless = false;
ctx->nH264FrameNumBits = 0;
ctx->nH265PocLsbBits = 0;
ctx->idr_interval = ctx->fps_n * 2;
ctx->level = -1;
ctx->fps_d = 1;
ctx->gdr_start_frame_number = 0xffffffff;
ctx->gdr_num_frames = 0xffffffff;
ctx->gdr_out_frame_number = 0xffffffff;
ctx->num_b_frames = (uint32_t)-1;
ctx->nMinQpI = (uint32_t)QP_RETAIN_VAL;
ctx->nMaxQpI = (uint32_t)QP_RETAIN_VAL;
ctx->nMinQpP = (uint32_t)QP_RETAIN_VAL;
ctx->nMaxQpP = (uint32_t)QP_RETAIN_VAL;
ctx->nMinQpB = (uint32_t)QP_RETAIN_VAL;
ctx->nMaxQpB = (uint32_t)QP_RETAIN_VAL;
ctx->use_gold_crc = false;
ctx->pBitStreamCrc = NULL;
ctx->externalRCHints = false;
ctx->input_metadata = false;
ctx->sMaxQp = 51;
ctx->stats = false;
ctx->stress_test = 1;
ctx->output_memory_type = V4L2_MEMORY_DMABUF;
ctx->cs = V4L2_COLORSPACE_SMPTE170M;
ctx->copy_timestamp = false;
ctx->sar_width = 0;
ctx->sar_height = 0;
ctx->start_ts = 0;
ctx->max_perf = 0;
ctx->blocking_mode = 1;
ctx->startf = 0;
ctx->endf = 0;
ctx->num_output_buffers = 6;
ctx->num_frames_to_encode = -1;
ctx->poc_type = 0;
ctx->chroma_format_idc = -1;
ctx->bit_depth = 8;
ctx->is_semiplanar = false;
ctx->enable_initQP = false;
ctx->IinitQP = 0;
ctx->PinitQP = 0;
ctx->BinitQP = 0;
ctx->enable_ratecontrol = true;
ctx->enable_av1tile = false;
ctx->log2_num_av1rows = 0;
ctx->log2_num_av1cols = 0;
ctx->enable_av1ssimrdo = (uint8_t)-1;
ctx->disable_av1cdfupdate = (uint8_t)-1;
ctx->ppe_init_params.enable_ppe = false;
ctx->ppe_init_params.wait_time_ms = -1;
ctx->ppe_init_params.feature_flags = V4L2_PPE_FEATURE_NONE;
ctx->ppe_init_params.enable_profiler = 0;
// ctx->ppe_init_params.width = 0;
// ctx->ppe_init_params.height = 0;
// ctx->ppe_init_params.taq_vic_downsampling = 0;
ctx->ppe_init_params.taq_max_qp_delta = 5;
}
int setParam()
{
set_defaults();
ctx.nMinQpI = 5;
ctx.nMaxQpI = 20;
ctx.nMinQpP = 20;
ctx.nMaxQpP = 45;
ctx.width = 1280;
ctx.height = 1024;
ctx.fps_n = 25;
ctx.iframe_interval = 25;
ctx.idr_interval = 25;
ctx.encoder_pixfmt = V4L2_PIX_FMT_H264;
ctx.bitrate = 2000000;
ctx.ratecontrol = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR;
ctx.profile = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH;
ctx.level = V4L2_MPEG_VIDEO_H264_LEVEL_5_1;
ctx.hw_preset_type = V4L2_ENC_HW_PRESET_FAST;
InitNvEncoder("enc0");
return 0;
}
#endif