I integrated NVAR into Unreal Engine 5 and at 5.0 it worked well, but there were already some very strange issues, when I upgraded to UE5.1, the same codes didn’t work at all, I thought it was UE that affected it, so I wrapped it into a separate DLL. I think, the code is very simple, this SDK is very easy to use, but its error message is not detailed enough.
I debugged every line of code to make sure they didn’t return an error, and when it go to the line of " NvAR_Run", it return an error status, and I couldn’t known by the return code what the problem was.
bool FaceExpressionSolver::InternalOpen(const FaceExpressionSettings& setting)
{
const unsigned landmarkCount = 126;
_isStatic = setting.IsStaticImage;
_frameWidth = setting.FrameWidth;
_frameHeight = setting.FrameHeight;
LOG_INFO(strf("NvAR Settings:\n FrameWidth: %d\n Frame Height: %d\n Model Folder: %s", _frameWidth, _frameHeight, setting.ModelFolder ? setting.ModelFolder: "NULL").c_str())
if(_featureHandle != nullptr)
{
LOG_IF_ERR(NvAR_Destroy(_featureHandle), "NvAR_Destroy");
_featureHandle = nullptr;
}
AllocImageBuffer(setting.FrameWidth, setting.FrameHeight, setting.FrameAlignment);
RTN_RESULT_IF_ERR(NvAR_Create(NvAR_Feature_FaceExpressions, &_featureHandle), "NvAR_Create");
if(setting.ModelFolder)
{
RTN_RESULT_IF_ERR(NvAR_SetString(_featureHandle, NvAR_Parameter_Config(ModelDir), setting.ModelFolder), "NvAR_SetString(ModelDir)");
}
RTN_RESULT_IF_ERR(NvAR_SetCudaStream(_featureHandle, NvAR_Parameter_Config(CUDAStream), _stream), "NvAR_SetCudaStream");
RTN_RESULT_IF_ERR(NvAR_SetU32(_featureHandle, NvAR_Parameter_Config(Landmarks_Size), landmarkCount), "NvAR_SetU32(Landmarks_Size)");
const auto filter = setting.IsStaticImage ? _staticFilter : _defaultFilter;
RTN_RESULT_IF_ERR(NvAR_SetU32(_featureHandle, NvAR_Parameter_Config(Temporal), filter), "NvAR_SetU32(Filtering)");
RTN_RESULT_IF_ERR(NvAR_SetU32(_featureHandle, NvAR_Parameter_Config(PoseMode), _poseMode), "NvAR_SetU32(PoseMode)");
RTN_RESULT_IF_ERR(NvAR_SetU32(_featureHandle, NvAR_Parameter_Config(Mode), 1), "NvAR_SetU32(Mode)");
RTN_RESULT_IF_ERR(NvAR_SetU32(_featureHandle, NvAR_Parameter_Config(EnableCheekPuff), setting.EnableCheekPuff), "NvAR_SetU32(EnableCheekPuff)");
RTN_RESULT_IF_ERR(NvAR_Load(_featureHandle), "NvAR_Load");
RTN_RESULT_IF_ERR(NvAR_GetU32(_featureHandle, NvAR_Parameter_Config(ExpressionCount), &_expressionCount), "NvAR_GetU32(ExpressionCount)");
_expressions.resize(_expressionCount);
_expressionZeroPoint.resize(_expressionCount, 0.0f);
_expressionScale.resize(_expressionCount, 1.0f);
_expressionExponent.resize(_expressionCount, 1.0f);
RTN_RESULT_IF_ERR(NvAR_SetF32Array(_featureHandle, NvAR_Parameter_Output(ExpressionCoefficients), _expressions.data(), _expressionCount), "NvAR_SetF32Array(ExpressionCoefficients)");
RTN_RESULT_IF_ERR(NvAR_SetObject(_featureHandle, NvAR_Parameter_Input(Image), &_srcGpu, sizeof(NvCVImage)), "NvAR_SetObject(Image)");
// Heuristic for focal length if it is not known
_cameraIntrinsicParams[0] = static_cast<float>(_srcGpu.height); // focal length
_cameraIntrinsicParams[1] = static_cast<float>(_srcGpu.width) / 2.0f; // cx
_cameraIntrinsicParams[2] = static_cast<float>(_srcGpu.height) / 2.0f; // cy
RTN_RESULT_IF_ERR(NvAR_SetF32Array(_featureHandle, NvAR_Parameter_Input(CameraIntrinsicParams), _cameraIntrinsicParams, NUM_CAMERA_INTRINSIC_PARAMS), "NvAR_SetF32Array(CameraIntrinsicParams)");
RTN_RESULT_IF_ERR(NvAR_SetObject(_featureHandle, NvAR_Parameter_Output(Pose), &_headQuaternion, sizeof(NvARQuaternion)), "NvAR_SetObject(Pose)");
return true;
}
solve function:
bool FaceExpressionSolver::SolveR8G8B8A8(void* bytes, FaceExpressionOutput& Output)
{
if(_featureHandle && bytes)
{
NvCVImage cpu;
//RTN_RESULT_IF_ERR(NvCVImage_Alloc(&cpu, _frameWidth, _frameHeight, NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 1), "NvCVImage_Alloc(SrcGPU)");
RTN_RESULT_IF_ERR(NvCVImage_Init(&cpu,
_frameWidth,
_frameHeight,
_frameWidth * 4,
bytes,
NVCV_RGBA,
NVCV_U8,
NVCV_INTERLEAVED, NVCV_CPU), "NvCVImage_Init(RGBA)");
RTN_RESULT_IF_ERR(NvCVImage_Transfer(&cpu, &_srcGpu, 1.0f, _stream, &_staging), "NvCVImage_Transfer(SrcCpu, SrcGpu)");
if(_isStatic)
{
for (float& _expression : _expressions)
{
_expression = 0.0f;
}
}
RTN_RESULT_IF_ERR(NvAR_Run(_featureHandle), "NvAR_Run");
Output.BlendShapesWeights = _expressions.data();
Output.BlendShapeCount = _expressionCount;
Output.HeadRotate.X = _headQuaternion.X;
Output.HeadRotate.Y = _headQuaternion.Y;
Output.HeadRotate.Z = _headQuaternion.Z;
Output.HeadRotate.W = _headQuaternion.W;
return true;
}
return false;
}
Parameter “bytes” is a uint8 array. Please point out what I missed, thanks .
NvAR_Run return NVCV_ERR_MISSINGINPUT (-15).