I was porting some code I had running in Windows to Linux Mint when I ran into the call to nvEncInitializeEncoder returns NV_ENC_ERR_INVALID_PARAM.
So I got the samples working and, for example, AppEncCuda gives the same error.
I have a 3070 and cuda 13.0
I can’t help you directly, but could you give some more detail on the error and the code you used?
So the NVENC experts might take a look?
Thanks!
Sorry just re-read my post which might have been confusing.
The nvidia samples also fail. I’ve been thrashing this out with ChatGPT but got into a bit of a loop. We’ve tidied up multiple cuda versions being present and insured the driver, nvenc and sdk versions are all the same. Cannot get nvEncInitializeEncoder to succeed, always getting NV_ENC_ERR_INVALID_PARAM. Currently we’re stuck trying to get nvEncGetEncodePresetConfig to work to populate the struct with good value. It’s always returning NV_ENC_ERR_INVALID_DEVICE even though the cucontext etc are all good. I feel at one point this function was working but it hasn’t for many iterations of the conversation.
Apologies for the somewhat messy copy and pasted code. It’s sitting in my programme and written by AI:
#define CHECK_CUDA_ABORT(call) do { \
CUresult __r = (call); \
if (__r != CUDA_SUCCESS) { const char* __s = nullptr; cuGetErrorString(__r, &__s); \
fprintf(stderr, "%s failed (%d): %s\n", #call, __r, __s?__s:"(no msg)"); exit(1); } } while(0)
#define CHECK_NVENC_ABORT(call) do { \
NVENCSTATUS __st = (call); \
if (__st != NV_ENC_SUCCESS) { fprintf(stderr, "%s failed: %s (%d)\n", #call, NvEncStatusToString(__st), (int)__st); exit(1); } } while(0)
int32_t NvVideo::ChatGpt()
{
SLog("ChatGpt starting", SLogLevel::warning);
// 1) CUDA init + create context (cuCtxCreate_v4)
CHECK_CUDA_ABORT(cuInit(0));
CUdevice cuDevice = 0;
CHECK_CUDA_ABORT(cuDeviceGet(&cuDevice, 0));
CUcontext cuContext = nullptr;
CUctxCreateParams params{}; // zero-initialized; adjust if you need affinities
CHECK_CUDA_ABORT(cuCtxCreate_v4(&cuContext, ¶ms, 0, cuDevice));
CHECK_CUDA_ABORT(cuCtxSetCurrent(cuContext));
printf("Created CUcontext %p on device %d\n", (void*)cuContext, (int)cuDevice);
// 2) Load NVENC API function table
NV_ENCODE_API_FUNCTION_LIST nvenc = { NV_ENCODE_API_FUNCTION_LIST_VER };
CHECK_NVENC_ABORT(NvEncodeAPICreateInstance(&nvenc));
printf("Loaded NVENC API, nvenc.nvEncOpenEncodeSessionEx=%p\n", (void*)nvenc.nvEncOpenEncodeSessionEx);
// 3) Open encode session (pass CUcontext)
NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openParams{};
openParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER;
openParams.device = cuContext; // must be CUcontext
openParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
openParams.apiVersion = NVENCAPI_VERSION; // 13.0
void* hEncoder = nullptr;
CHECK_NVENC_ABORT(nvenc.nvEncOpenEncodeSessionEx(&openParams, &hEncoder));
printf("nvEncOpenEncodeSessionEx -> success, hEncoder=%p\n", hEncoder);
// Ensure context current on this thread (very important)
CHECK_CUDA_ABORT(cuCtxSetCurrent(cuContext));
// 4) Query preset using NV_ENC_PRESET_CONFIG (correct type)
NV_ENC_PRESET_CONFIG presetConfig{};
presetConfig.version = NV_ENC_PRESET_CONFIG_VER;
presetConfig.presetCfg.version = NV_ENC_CONFIG_VER;
CUcontext cur = nullptr;
cuCtxGetCurrent(&cur);
printf("Before presetConfig: current=%p expected=%p, hEncoder=%p\n",
(void*)cur, (void*)cuContext, hEncoder);
CHECK_CUDA_ABORT(cuCtxSetCurrent(cuContext));
// Use a conservative preset that's defined in the SDK
// NV_ENC_PRESET_P1_GUID is safe and present in SDK
NVENCSTATUS st = nvenc.nvEncGetEncodePresetConfig(
hEncoder,
NV_ENC_CODEC_H264_GUID,
NV_ENC_PRESET_P1_GUID,
&presetConfig
);
if (st != NV_ENC_SUCCESS) {
fprintf(stderr, "nvEncGetEncodePresetConfig failed: %s (%d)\n", NvEncStatusToString(st), (int)st);
// Dump some minimal debug
CUcontext cur = nullptr; cuCtxGetCurrent(&cur);
printf("Debug: current CUcontext=%p expected=%p, hEncoder=%p, nvEncGetEncodePresetConfig ptr=%p\n",
(void*)cur, (void*)cuContext, hEncoder, (void*)nvenc.nvEncGetEncodePresetConfig);
exit(1);
}
printf("nvEncGetEncodePresetConfig -> success\n");
// 5) Copy presetConfig.presetCfg into NV_ENC_CONFIG and adjust
NV_ENC_CONFIG encodeConfig = presetConfig.presetCfg; // copy
encodeConfig.version = NV_ENC_CONFIG_VER; // re-assert
// tweak a few things to be safe
encodeConfig.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID;
encodeConfig.gopLength = 60;
encodeConfig.frameIntervalP = 1;
// choose a rate-control that matches your needs; CBR here as example
encodeConfig.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
encodeConfig.rcParams.averageBitRate = 4000000; // 4 Mbps
encodeConfig.rcParams.maxBitRate = 4000000;
// vbvBufferSize often expects bytes or bits depending on SDK; set to bitrate as conservative
encodeConfig.rcParams.vbvBufferSize = encodeConfig.rcParams.averageBitRate;
encodeConfig.rcParams.vbvInitialDelay = encodeConfig.rcParams.vbvBufferSize;
// 6) Build init params
NV_ENC_INITIALIZE_PARAMS initParams{};
initParams.version = NV_ENC_INITIALIZE_PARAMS_VER;
initParams.encodeGUID = NV_ENC_CODEC_H264_GUID;
initParams.presetGUID = NV_ENC_PRESET_P1_GUID;
initParams.encodeConfig = &encodeConfig;
initParams.encodeWidth = 1920;
initParams.encodeHeight = 1080;
initParams.darWidth = 1920;
initParams.darHeight = 1080;
initParams.frameRateNum = 30;
initParams.frameRateDen = 1;
initParams.enablePTD = 1;
// Input buffer format - NV12 is usually supported; set explicitly
initParams.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
// Ensure context is current just before init (defensive)
CHECK_CUDA_ABORT(cuCtxSetCurrent(cuContext));
// 7) Initialize encoder
NVENCSTATUS st2 = nvenc.nvEncInitializeEncoder(hEncoder, &initParams);
if (st2 != NV_ENC_SUCCESS) {
fprintf(stderr, "nvEncInitializeEncoder failed: %s (%d)\n", NvEncStatusToString(st2), (int)st2);
// optional: print init/config quick dump
printf("initParams: W x H = %u x %u, bufferFormat=%u, encodeConfig ptr=%p\n",
initParams.encodeWidth, initParams.encodeHeight, (unsigned)initParams.bufferFormat, (void*)initParams.encodeConfig);
exit(1);
}
printf("nvEncInitializeEncoder succeeded ✅\n");
// Clean up
nvenc.nvEncDestroyEncoder(hEncoder);
cuCtxDestroy(cuContext);
I’ve chopped off the end but hopefully that makes sense. Output is:
Created CUcontext 0x5ecb83f3f2f0 on device 0
Loaded NVENC API, nvenc.nvEncOpenEncodeSessionEx=0x719e8bfe6220
nvEncOpenEncodeSessionEx → success, hEncoder=0x5ecb848ff100
Before presetConfig: current=0x5ecb83f3f2f0 expected=0x5ecb83f3f2f0, hEncoder=0x5ecb848ff100
nvEncGetEncodePresetConfig failed: NV_ENC_ERR_INVALID_DEVICE (4)
Debug: current CUcontext=0x5ecb83f3f2f0 expected=0x5ecb83f3f2f0, hEncoder=0x5ecb848ff100, nvEncGetEncodePresetConfig ptr=0x719e8bfe5080
Ok, the nvEncGetEncodePresetConfig error was due to needing nvEncGetEncodePresetConfigEx nowadays. The invalid device error is confusing but at least that’s solved. Back to the nvEncInitializeEncoder NV_ENC_ERR_INVALID_PARAM error. Here is the latest test code:
#include <cuda.h>
#include "nvEncodeAPI.h"
#include <dlfcn.h>
#include <iostream>
#include <vector>
#include <cstring>
#define CHECK_CUDA(call) \
do { \
CUresult _status = call; \
if (_status != CUDA_SUCCESS) { \
const char *errStr; \
cuGetErrorString(_status, &errStr); \
std::cerr << "CUDA error: " << errStr \
<< " at " << __FILE__ << ":" << __LINE__ << std::endl; \
exit(1); \
} \
} while (0)
#define CHECK_NVENC(call) \
do { \
NVENCSTATUS _status = call; \
if (_status != NV_ENC_SUCCESS) { \
std::cerr << #call << " failed: " << _status \
<< " at " << __FILE__ << ":" << __LINE__ << std::endl; \
exit(1); \
} \
} while (0)
int main() {
// ------------------------------------------------------------
// Step 1: CUDA init and context
// ------------------------------------------------------------
CHECK_CUDA(cuInit(0));
CUdevice cuDevice = 0;
CHECK_CUDA(cuDeviceGet(&cuDevice, 0));
CUcontext cuContext = nullptr;
// New API (CUDA 12+), params optional -> pass nullptr
CHECK_CUDA(cuCtxCreate_v4(&cuContext, nullptr, 0, cuDevice));
std::cout << "Created CUcontext " << cuContext << " on device 0" << std::endl;
// ------------------------------------------------------------
// Step 2: Load NVENC API
// ------------------------------------------------------------
void *nvencLib = dlopen("libnvidia-encode.so.1", RTLD_LAZY);
if (!nvencLib) {
std::cerr << "dlopen failed: " << dlerror() << std::endl;
return 1;
}
auto NvEncodeAPICreateInstance =
(NVENCSTATUS (NVENCAPI*)(NV_ENCODE_API_FUNCTION_LIST*))
dlsym(nvencLib, "NvEncodeAPICreateInstance");
if (!NvEncodeAPICreateInstance) {
std::cerr << "Failed to load NvEncodeAPICreateInstance" << std::endl;
return 1;
}
NV_ENCODE_API_FUNCTION_LIST nvenc = {};
nvenc.version = NV_ENCODE_API_FUNCTION_LIST_VER;
CHECK_NVENC(NvEncodeAPICreateInstance(&nvenc));
std::cout << "Loaded NVENC API" << std::endl;
// ------------------------------------------------------------
// Step 3: Open encode session
// ------------------------------------------------------------
NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openParams = {};
openParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER;
openParams.device = cuContext;
openParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
openParams.apiVersion = NVENCAPI_VERSION;
void *hEncoder = nullptr;
CHECK_NVENC(nvenc.nvEncOpenEncodeSessionEx(&openParams, &hEncoder));
std::cout << "Opened encode session: " << hEncoder << std::endl;
// ------------------------------------------------------------
// Step 4: Query a preset config (corrected)
// ------------------------------------------------------------
NV_ENC_PRESET_CONFIG presetConfig = {};
presetConfig.version = NV_ENC_PRESET_CONFIG_VER;
presetConfig.presetCfg.version = NV_ENC_CONFIG_VER;
GUID presetGuid = NV_ENC_PRESET_P3_GUID; // pick P3 for test
NVENCSTATUS st = nvenc.nvEncGetEncodePresetConfigEx(
hEncoder,
NV_ENC_CODEC_H264_GUID,
presetGuid,
NV_ENC_TUNING_INFO_HIGH_QUALITY,
&presetConfig);
if (st != NV_ENC_SUCCESS) {
std::cerr << "nvEncGetEncodePresetConfigEx failed: " << st << std::endl;
return 1;
}
std::cout << "Got preset config (P3, HQ)" << std::endl;
// ------------------------------------------------------------
// Step 5: Initialize encoder
// ------------------------------------------------------------
NV_ENC_INITIALIZE_PARAMS initParams = {};
initParams.version = NV_ENC_INITIALIZE_PARAMS_VER;
initParams.encodeGUID = NV_ENC_CODEC_H264_GUID;
initParams.presetGUID = presetGuid;
initParams.encodeWidth = 1920;
initParams.encodeHeight = 1080;
initParams.darWidth = 1920;
initParams.darHeight = 1080;
initParams.frameRateNum = 30;
initParams.frameRateDen = 1;
initParams.enablePTD = 1;
initParams.encodeConfig = &presetConfig.presetCfg; // <- from preset
CHECK_NVENC(nvenc.nvEncInitializeEncoder(hEncoder, &initParams));
std::cout << "nvEncInitializeEncoder succeeded" << std::endl;
return 0;
}
I “think” I’ve got something working (well the ai did). I haven’t looked to see what change cured it as I’m off to bed.
This is the code:
// test_nvenc_fixed.cpp
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <cuda.h>
#include "nvEncodeAPI.h"
// Simple helpers
static void printGuid(const GUID &g) {
printf("{%08x-%04x-%04x-%02x%02x-", g.Data1, g.Data2, g.Data3,
(unsigned)g.Data4[0], (unsigned)g.Data4[1]);
for (int i = 2; i < 8; ++i) printf("%02x", (unsigned)g.Data4[i]);
printf("}");
}
static void dumpConfig(const NV_ENC_CONFIG &cfg) {
printf("=== NVENC CONFIG DUMP ===\n");
printf("version = 0x%x\n", cfg.version);
printf("profileGUID = "); printGuid(cfg.profileGUID); printf("\n");
printf("gopLength = %u\n", (unsigned)cfg.gopLength);
printf("frameIntervalP = %u\n", (unsigned)cfg.frameIntervalP);
printf("rateControlMode = %u\n", (unsigned)cfg.rcParams.rateControlMode);
printf("constQP: I=%u, P=%u, B=%u\n",
(unsigned)cfg.rcParams.constQP.qpIntra,
(unsigned)cfg.rcParams.constQP.qpInterP,
(unsigned)cfg.rcParams.constQP.qpInterB);
printf("avgBitRate = %u\n", (unsigned)cfg.rcParams.averageBitRate);
printf("maxBitRate = %u\n", (unsigned)cfg.rcParams.maxBitRate);
printf("vbvBufferSize = %u\n", (unsigned)cfg.rcParams.vbvBufferSize);
printf("=========================\n");
}
static void dumpInitParams(const NV_ENC_INITIALIZE_PARAMS &init) {
printf("=== NVENC INIT PARAMS ===\n");
printf("version = 0x%x\n", init.version);
printf("encodeGUID = "); printGuid(init.encodeGUID); printf("\n");
printf("presetGUID = "); printGuid(init.presetGUID); printf("\n");
printf("encodeWidth = %u\n", (unsigned)init.encodeWidth);
printf("encodeHeight = %u\n", (unsigned)init.encodeHeight);
printf("darWidth = %u\n", (unsigned)init.darWidth);
printf("darHeight = %u\n", (unsigned)init.darHeight);
printf("frameRateNum = %u\n", (unsigned)init.frameRateNum);
printf("frameRateDen = %u\n", (unsigned)init.frameRateDen);
printf("enablePTD = %u\n", (unsigned)init.enablePTD);
printf("bufferFormat = %u\n", (unsigned)init.bufferFormat);
printf("maxEncodeWidth = %u\n", (unsigned)init.maxEncodeWidth);
printf("maxEncodeHeight = %u\n", (unsigned)init.maxEncodeHeight);
printf("maxMEHintCountsPerBlock = %p\n", (void*)init.maxMEHintCountsPerBlock);
printf("=========================\n");
}
// Error helpers
static void checkCuda(CUresult r, const char *what) {
if (r != CUDA_SUCCESS) {
const char *name = nullptr;
cuGetErrorName(r, &name);
fprintf(stderr, "CUDA error %s: %d (%s)\n", what, (int)r, name?name:"?");
exit(1);
}
}
static void checkNvenc(NVENCSTATUS s, const char *what) {
if (s != NV_ENC_SUCCESS) {
fprintf(stderr, "NVENC error %s: %d\n", what, (int)s);
// We don't exit here to allow dumps after failures if desired
}
}
int main() {
// ---------- CUDA init + context ----------
checkCuda(cuInit(0), "cuInit");
CUdevice cuDev = 0;
checkCuda(cuDeviceGet(&cuDev, 0), "cuDeviceGet(0)");
CUcontext cuContext = nullptr;
// cuCtxCreate_v4 signature: CUresult cuCtxCreate_v4(CUcontext*, CUctxCreateParams*, unsigned int, CUdevice)
// params is optional on recent CUDA; passing nullptr is fine for a simple context.
checkCuda(cuCtxCreate_v4(&cuContext, nullptr, 0, cuDev), "cuCtxCreate_v4");
checkCuda(cuCtxSetCurrent(cuContext), "cuCtxSetCurrent");
printf("Created CUcontext %p on device 0\n", (void*)cuContext);
// ---------- load NVENC (dlopen + dlsym) ----------
void *lib = dlopen("libnvidia-encode.so.1", RTLD_NOW | RTLD_LOCAL);
if (!lib) {
fprintf(stderr, "dlopen(libnvidia-encode.so.1) failed: %s\n", dlerror());
return 1;
}
using PFN_NvEncodeAPICreateInstance = NVENCSTATUS(NVENCAPI*)(NV_ENCODE_API_FUNCTION_LIST*);
auto pNvEncodeAPICreateInstance = (PFN_NvEncodeAPICreateInstance)dlsym(lib, "NvEncodeAPICreateInstance");
if (!pNvEncodeAPICreateInstance) {
fprintf(stderr, "dlsym(NvEncodeAPICreateInstance) failed: %s\n", dlerror());
return 1;
}
NV_ENCODE_API_FUNCTION_LIST nvenc = { NV_ENCODE_API_FUNCTION_LIST_VER };
NVENCSTATUS st = pNvEncodeAPICreateInstance(&nvenc);
if (st != NV_ENC_SUCCESS) {
fprintf(stderr, "NvEncodeAPICreateInstance -> %d\n", (int)st);
return 1;
}
printf("Loaded NVENC API\n");
// ---------- open session ----------
NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openParams = {};
openParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER;
openParams.device = cuContext;
openParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
#ifdef NVENCAPI_VERSION
openParams.apiVersion = NVENCAPI_VERSION;
#endif
void *hEncoder = nullptr;
st = nvenc.nvEncOpenEncodeSessionEx(&openParams, &hEncoder);
checkNvenc(st, "nvEncOpenEncodeSessionEx");
if (st != NV_ENC_SUCCESS) return 1;
printf("Opened encode session: %p\n", hEncoder);
// assume nvenc, hEncoder, cuContext exist and are valid
// 1) get preset (as you already do)
NV_ENC_PRESET_CONFIG presetCfg;
memset(&presetCfg, 0, sizeof(presetCfg));
presetCfg.version = NV_ENC_PRESET_CONFIG_VER;
presetCfg.presetCfg.version = NV_ENC_CONFIG_VER;
GUID chosenPreset = NV_ENC_PRESET_P3_GUID; // P3 HQ
GUID codec = NV_ENC_CODEC_H264_GUID;
st = nvenc.nvEncGetEncodePresetConfigEx(
hEncoder, codec, chosenPreset, NV_ENC_TUNING_INFO_HIGH_QUALITY, &presetCfg);
if (st != NV_ENC_SUCCESS) {
fprintf(stderr, "nvEncGetEncodePresetConfigEx failed: %d\n", (int)st);
return 1;
}
// 2) sanitize preset: force H264 High profile & force CONSTQP (safe)
presetCfg.presetCfg.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
presetCfg.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP;
presetCfg.presetCfg.rcParams.constQP.qpIntra = 28;
presetCfg.presetCfg.rcParams.constQP.qpInterP = 31;
presetCfg.presetCfg.rcParams.constQP.qpInterB = 31;
presetCfg.presetCfg.gopLength = (uint32_t)250; // or keep as preset
presetCfg.presetCfg.frameIntervalP = 1; // single ref interval
// 3) init params (zeroed)
NV_ENC_INITIALIZE_PARAMS initParams;
memset(&initParams, 0, sizeof(initParams));
initParams.version = NV_ENC_INITIALIZE_PARAMS_VER;
initParams.encodeGUID = codec;
initParams.presetGUID = chosenPreset;
initParams.encodeWidth = 1920;
initParams.encodeHeight = 1080;
initParams.darWidth = 1920;
initParams.darHeight = 1080;
initParams.frameRateNum = 30;
initParams.frameRateDen = 1;
initParams.enablePTD = 1;
initParams.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
initParams.encodeConfig = &presetCfg.presetCfg;
initParams.tuningInfo = NV_ENC_TUNING_INFO_HIGH_QUALITY;
// 4) ensure context is current
cuCtxSetCurrent(cuContext);
// 5) dump and call
dumpConfig(presetCfg.presetCfg);
dumpInitParams(initParams);
st = nvenc.nvEncInitializeEncoder(hEncoder, &initParams);
printf("nvEncInitializeEncoder -> %d\n", (int)st);
// cleanup
nvenc.nvEncDestroyEncoder(hEncoder);
cuCtxDestroy(cuContext);
dlclose(lib);
return 0;
}
With output:
Created CUcontext 0x5c45ee8d8f10 on device 0
Loaded NVENC API
Opened encode session: 0x5c45ef464f80
=== NVENC CONFIG DUMP ===
version = 0xf009000d
profileGUID = {e7cbc309-4f7a-4b89-af2a-d537c92be310}
gopLength = 250
frameIntervalP = 1
rateControlMode = 0
constQP: I=28, P=31, B=31
avgBitRate = 0
maxBitRate = 0
vbvBufferSize = 0
=========================
=== NVENC INIT PARAMS ===
version = 0xf007000d
encodeGUID = {6bc82762-4e63-4ca4-aa85-1e50f321f6bf}
presetGUID = {36850110-3a07-441f-94d5-3670631f91f6}
encodeWidth = 1920
encodeHeight = 1080
darWidth = 1920
darHeight = 1080
frameRateNum = 30
frameRateDen = 1
enablePTD = 1
bufferFormat = 1
maxEncodeWidth = 0
maxEncodeHeight = 0
maxMEHintCountsPerBlock = 0x7ffd823d9a48
=========================
nvEncInitializeEncoder -> 0
This works all the way to producing output. Don’t know why the sdk sample also failed but just assume either v13 or Linux driver is more particular about what it will accept in the parameters. Things to note: nvEncGetEncodePresetConfigEx not nvEncGetEncodePresetConfig, initParams.tuningInfo has to be set. Don’t know what else is significant but here you go:
#include <cuda.h>
#include <dlfcn.h>
#include <nvEncodeAPI.h>
#include <cstdio>
#include <cstring>
#include <vector>
#define CHECK_CUDA(call) do { \
CUresult _status = call; \
if (_status != CUDA_SUCCESS) { \
const char *errStr = nullptr; \
cuGetErrorString(_status, &errStr); \
fprintf(stderr, "CUDA error %d: %s\n", _status, errStr); \
exit(1); \
} \
} while (0)
#define CHECK_NVENC(call) do { \
NVENCSTATUS _status = call; \
if (_status != NV_ENC_SUCCESS) { \
fprintf(stderr, "%s failed: %d\n", #call, _status); \
exit(1); \
} \
} while (0)
static NV_ENCODE_API_FUNCTION_LIST nvenc = { NV_ENCODE_API_FUNCTION_LIST_VER };
#define WIDTH 1920
#define HEIGHT 1080
#define FRAMES 30
void fillNV12Frame(CUdeviceptr devPtr, int pitch, int width, int height, int frameIdx)
{
std::vector<unsigned char> hostFrame(pitch * height * 3 / 2);
// Animate Y with a gradient changing each frame
for (int y = 0; y < height; y++)
for (int x = 0; x < width; x++)
hostFrame[y * pitch + x] = (x + frameIdx * 8) & 0xFF;
unsigned char *uv = hostFrame.data() + pitch * height;
for (int y = 0; y < height / 2; y++)
for (int x = 0; x < width; x += 2) {
uv[y * pitch + x + 0] = 128;
uv[y * pitch + x + 1] = 128;
}
CHECK_CUDA(cuMemcpyHtoD(devPtr, hostFrame.data(), hostFrame.size()));
}
int main()
{
CHECK_CUDA(cuInit(0));
CUdevice cuDevice;
CHECK_CUDA(cuDeviceGet(&cuDevice, 0));
CUcontext cuContext;
CHECK_CUDA(cuCtxCreate_v4(&cuContext, nullptr, 0, cuDevice));
printf("Created CUcontext %p\n", (void*)cuContext);
void *lib = dlopen("libnvidia-encode.so.1", RTLD_NOW);
if (!lib) { fprintf(stderr, "dlopen failed\n"); return 1; }
auto NvEncodeAPICreateInstance =
(decltype(&::NvEncodeAPICreateInstance))dlsym(lib, "NvEncodeAPICreateInstance");
if (!NvEncodeAPICreateInstance) { fprintf(stderr, "NvEncodeAPICreateInstance not found\n"); return 1; }
CHECK_NVENC(NvEncodeAPICreateInstance(&nvenc));
NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openParams = { NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER };
openParams.device = cuContext;
openParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
openParams.apiVersion = NVENCAPI_VERSION;
void *hEncoder = nullptr;
CHECK_NVENC(nvenc.nvEncOpenEncodeSessionEx(&openParams, &hEncoder));
printf("Opened NVENC session: %p\n", hEncoder);
// Preset config
NV_ENC_PRESET_CONFIG presetCfg = { NV_ENC_PRESET_CONFIG_VER };
presetCfg.presetCfg.version = NV_ENC_CONFIG_VER;
GUID chosenPreset = NV_ENC_PRESET_P3_GUID; // P3 HQ
GUID codec = NV_ENC_CODEC_H264_GUID;
CHECK_NVENC(nvenc.nvEncGetEncodePresetConfigEx(
hEncoder, codec, chosenPreset, NV_ENC_TUNING_INFO_HIGH_QUALITY, &presetCfg));
presetCfg.presetCfg.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
presetCfg.presetCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP;
presetCfg.presetCfg.rcParams.constQP.qpIntra = 28;
presetCfg.presetCfg.rcParams.constQP.qpInterP = 31;
presetCfg.presetCfg.rcParams.constQP.qpInterB = 31;
presetCfg.presetCfg.gopLength = (uint32_t)250; // or keep as preset
presetCfg.presetCfg.frameIntervalP = 1; // single ref interval
// Initialize encoder
NV_ENC_INITIALIZE_PARAMS initParams = { NV_ENC_INITIALIZE_PARAMS_VER };
initParams.encodeGUID = codec;
initParams.presetGUID = chosenPreset;
initParams.encodeWidth = WIDTH;
initParams.encodeHeight = HEIGHT;
initParams.darWidth = WIDTH;
initParams.darHeight = HEIGHT;
initParams.frameRateNum = 30;
initParams.frameRateDen = 1;
initParams.enablePTD = 1;
initParams.encodeConfig = &presetCfg.presetCfg;
initParams.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
initParams.tuningInfo = NV_ENC_TUNING_INFO_HIGH_QUALITY;
CHECK_NVENC(nvenc.nvEncInitializeEncoder(hEncoder, &initParams));
printf("Encoder initialized.\n");
// Allocate input buffer
CUdeviceptr devFrame;
size_t frameSize = WIDTH * HEIGHT * 3 / 2;
CHECK_CUDA(cuMemAlloc(&devFrame, frameSize));
NV_ENC_REGISTER_RESOURCE regRes = { NV_ENC_REGISTER_RESOURCE_VER };
regRes.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR;
regRes.resourceToRegister = (void*)devFrame;
regRes.width = WIDTH;
regRes.height = HEIGHT;
regRes.pitch = WIDTH;
regRes.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
CHECK_NVENC(nvenc.nvEncRegisterResource(hEncoder, ®Res));
void *inputHandle = regRes.registeredResource;
// Output bitstream buffer
NV_ENC_CREATE_BITSTREAM_BUFFER bitBuf = { NV_ENC_CREATE_BITSTREAM_BUFFER_VER };
CHECK_NVENC(nvenc.nvEncCreateBitstreamBuffer(hEncoder, &bitBuf));
void *bitstreamBuffer = bitBuf.bitstreamBuffer;
FILE *fp = fopen("out.h264", "wb");
if (!fp) { perror("fopen"); return 1; }
// Encode loop
for (int i = 0; i < FRAMES; i++) {
fillNV12Frame(devFrame, WIDTH, WIDTH, HEIGHT, i);
NV_ENC_MAP_INPUT_RESOURCE mapRes = { NV_ENC_MAP_INPUT_RESOURCE_VER };
mapRes.registeredResource = inputHandle;
CHECK_NVENC(nvenc.nvEncMapInputResource(hEncoder, &mapRes));
NV_ENC_PIC_PARAMS picParams = { NV_ENC_PIC_PARAMS_VER };
picParams.inputBuffer = mapRes.mappedResource;
picParams.bufferFmt = NV_ENC_BUFFER_FORMAT_NV12;
picParams.inputWidth = WIDTH;
picParams.inputHeight = HEIGHT;
picParams.outputBitstream = bitstreamBuffer;
picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME;
picParams.encodePicFlags = (i == 0) ? NV_ENC_PIC_FLAG_FORCEIDR : 0;
CHECK_NVENC(nvenc.nvEncEncodePicture(hEncoder, &picParams));
NV_ENC_LOCK_BITSTREAM lockBit = { NV_ENC_LOCK_BITSTREAM_VER };
lockBit.outputBitstream = bitstreamBuffer;
lockBit.doNotWait = 0;
CHECK_NVENC(nvenc.nvEncLockBitstream(hEncoder, &lockBit));
fwrite(lockBit.bitstreamBufferPtr, 1, lockBit.bitstreamSizeInBytes, fp);
nvenc.nvEncUnlockBitstream(hEncoder, bitstreamBuffer);
nvenc.nvEncUnmapInputResource(hEncoder, mapRes.mappedResource);
printf("Encoded frame %d, %u bytes\n", i, lockBit.bitstreamSizeInBytes);
}
// Flush encoder
NV_ENC_PIC_PARAMS flushParams = { NV_ENC_PIC_PARAMS_VER };
flushParams.encodePicFlags = NV_ENC_PIC_FLAG_EOS;
nvenc.nvEncEncodePicture(hEncoder, &flushParams);
printf("Flushed encoder.\n");
fclose(fp);
printf("Wrote %d frames to out.h264\n", FRAMES);
// Cleanup
nvenc.nvEncDestroyBitstreamBuffer(hEncoder, bitstreamBuffer);
nvenc.nvEncUnregisterResource(hEncoder, inputHandle);
nvenc.nvEncDestroyEncoder(hEncoder);
cuMemFree(devFrame);
printf("Clean shutdown.\n");
return 0;
}