Trouble with Expression Tracking

AI & Data Science Deep Learning (Training & Inference) AI for Media
1

Hi there! I’m pretty new to the SDK, and I’m just trying to get a minimal facial expression solving working.

I have already gone through the FaceTrack sample and implemented the features I wanted from that and its working great.

However, no matter what I seem to do NvAR_Run always fails for my facial feature handle. im hoping im just missing something small and someone can notice it. I only want to solve for a single frame that i capture at the beginning of this function.

here is the code:

// solve expressions
NvCV_Status expression_err;
NvAR_FeatureHandle expression_handle;
CUstream expression_stream;
std::vector<NvAR_Point2f> _expression_landmarks;
std::vector<NvAR_Rect> _outputBboxData;
NvAR_BBoxes        _outputBboxes;
std::vector<float> _expressions, _expressionZeroPoint, _expressionScale, _expressionExponent, _eigenvalues, _landmark_confidence;
NvCVImage frame_image;
 struct Pose {
	NvAR_Quaternion rotation;
	NvAR_Vector3f translation;
	float* data() { return &rotation.x; }
	const float* data() const { return &rotation.x; }
};            
Pose _pose;

unsigned _filtering = 0;
unsigned _pose_mode = 0;
unsigned _expr_mode = 2;
unsigned _enable_cheek_puff = 0;
unsigned _exprCount;

expression_err = NvAR_Create(NvAR_Feature_FaceExpressions, &expression_handle);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to create facial expression feature");
}

expression_err = NvAR_SetCudaStream(expression_handle, NvAR_Parameter_Config(CUDAStream), expression_stream);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set CUDA stream");
}

expression_err = NvAR_SetU32(expression_handle, NvAR_Parameter_Config(Temporal), _filtering);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set temporal filtering");
}

expression_err = NvAR_SetU32(expression_handle, NvAR_Parameter_Config(PoseMode), _pose_mode);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set pose_mode");
}

expression_err = NvAR_SetU32(expression_handle, NvAR_Parameter_Config(EnableCheekPuff), _enable_cheek_puff);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set cheek puff");
}

expression_err = NvAR_Load(expression_handle);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to load facial expression feature");
} else {
	UtilityFunctions::print("facial expression feature loaded");
}

_outputBboxData.assign(25, { 0.f, 0.f, 0.f, 0.f });
_outputBboxes.boxes = _outputBboxData.data();
_outputBboxes.max_boxes = (uint8_t)_outputBboxData.size();
_outputBboxes.num_boxes = 0;
expression_err = NvAR_SetObject(expression_handle, NvAR_Parameter_Output(BoundingBoxes), &_outputBboxes, sizeof(NvAR_BBoxes));
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set bbox object");
}

_expression_landmarks.resize(126);
expression_err = NvAR_SetObject(expression_handle, NvAR_Parameter_Output(Landmarks), _expression_landmarks.data(), sizeof(NvAR_Point2f));
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set landmarks object");
}

_landmark_confidence.resize(126);
expression_err = NvAR_SetF32Array(expression_handle, NvAR_Parameter_Output(LandmarksConfidence), _landmark_confidence.data(), 126);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set landmark confidence array");
}

expression_err = NvAR_GetU32(expression_handle, NvAR_Parameter_Config(ExpressionCount), &_exprCount);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to get expression count");
}

_expressions.resize(_exprCount);
_expressionZeroPoint.resize(_exprCount, 0.0f);
_expressionScale.resize(_exprCount, 1.0f);
_expressionExponent.resize(_exprCount, 1.0f);

expression_err = NvAR_SetF32Array(expression_handle, NvAR_Parameter_Output(ExpressionCoefficients), _expressions.data(), _exprCount);
if (expression_err!=NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set expression coefficient array");
} else {
	UtilityFunctions::print("successfully set expression coefficient array");
}

ConvertMatToNvCVImage(frame, &frame_image); // Ensure the implementation of this function

expression_err = NvAR_SetObject(expression_handle, NvAR_Parameter_Input(Image), &frame_image, sizeof(frame_image));
if (expression_err != NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set the source image for expression analysis");
}

float cameraIntrinsicParams[3] = {static_cast<float>(inputHeight), inputWidth / 2.0f, inputHeight / 2.0f}; // focal length, cx, cy
expression_err = NvAR_SetF32Array(expression_handle, NvAR_Parameter_Input(CameraIntrinsicParams), cameraIntrinsicParams, 3);
if (expression_err != NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set camera intrinsic parameters");
}

expression_err = NvAR_SetObject(expression_handle, NvAR_Parameter_Output(Pose), &_pose.rotation, sizeof(NvAR_Quaternion));
if (expression_err != NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set pose rotation object");
}

expression_err = NvAR_SetObject(expression_handle, NvAR_Parameter_Output(PoseTranslation), &_pose.translation, sizeof(NvAR_Vector3f));
if (expression_err != NVCV_SUCCESS) {
	UtilityFunctions::print("failed to set pose rotation object");
}

// Run the facial expression feature
expression_err = NvAR_Run(expression_handle);
if (expression_err != NVCV_SUCCESS) {
	UtilityFunctions::print("failed to run facial expression feature");
} else {
	UtilityFunctions::print("facial expression feature run successfully");
}

// Process and print the expression coefficients
// After running the facial expression feature
for (size_t i = 0; i < _expressions.size(); ++i) {
	std::cout << "Expression Coefficient " << i << ": " << _expressions[i] << std::endl;
}

// Cleanup for expression handling
NvAR_Destroy(expression_handle); // Destroy expression feature handle

// Deallocate NvCVImage used for capturing frame
NvCVImage_Dealloc(&frame_image);

// Deallocate std::vectors
_expression_landmarks.clear();
_outputBboxData.clear();
_expressions.clear();
_expressionZeroPoint.clear();
_expressionScale.clear();
_expressionExponent.clear();
_eigenvalues.clear();
_landmark_confidence.clear();

and then here is the function i am trying to use to convert the cv image capture to NvCVImage with

NvCV_Status FaceTrack::ConvertMatToNvCVImage(const cv::Mat& mat, NvCVImage* nvImage) {
  if (!mat.empty() && mat.depth() == CV_8U) {
    NvCV_Status status = NvCVImage_Alloc(nvImage, mat.cols, mat.rows, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 1);
    if (status != NVCV_SUCCESS) return status;

    // Assuming BGR format and contiguous memory in cv::Mat
    memcpy(nvImage->pixels, mat.data, mat.total() * mat.elemSize());
    return NVCV_SUCCESS;
  } else {
    return NVCV_ERR_FEATURENOTFOUND;
  }
}

if anyone has the time to take a look, and tell me what im missing, I would be greatly appreciative! I’ll keep at it in the meantime, but I am taking a break so I figured I’d toss it up here.

also I know it would be better to break this up into more functions. like i said, i’m just trying to find the minimal implementation right now so i can think about how i wanna design the app.

thanks!

2

okay I have kept working on this. I read through the docs more and I realized that you should be able to get the expression coefficients based off the landmark points i am already getting. so this is what I am working with now

    // solve facial expressions
    
    // intialize vars
    NvCV_Status expression_err = NVCV_SUCCESS;
    NvAR_FeatureHandle expression_handle{};
    
    // create feature handle
    expression_err = NvAR_Create(NvAR_Feature_FaceExpressions, &expression_handle);
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to create expressions feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    // set feature config
    expression_err = NvAR_SetString(expression_handle, NvAR_Parameter_Config(ModelDir), modelPath.c_str());
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to set model dir for expressions feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    expression_err = NvAR_SetU32(expression_handle, NvAR_Parameter_Config(PoseMode), 0);
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to set pose mode for expressions feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    expression_err = NvAR_SetU32(expression_handle, NvAR_Parameter_Config(EnableCheekPuff), 0);
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to set cheek puff enable for expressions feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    // load feature
    expression_err = NvAR_Load(expression_handle);
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to load expressions feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    // set feature input
    expression_err = NvAR_SetObject(expression_handle, NvAR_Parameter_Input(Landmarks), facial_landmarks.data(), sizeof(NvAR_Point2f));
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to set landmarks input for expression feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    // set feature outputs
    unsigned int expressionCount;
    expression_err = NvAR_GetU32(expression_handle, NvAR_Parameter_Config(ExpressionCount), &expressionCount);
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to get expression count for expression feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    float* expressionCoeffs = new float[expressionCount];
    expression_err = NvAR_SetF32Array(expression_handle, NvAR_Parameter_Output(ExpressionCoefficients), expressionCoeffs, expressionCount);
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to set expression coefficients output for expression feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    NvAR_Quaternion* pose = new NvAR_Quaternion();
    expression_err = NvAR_SetObject(expression_handle, NvAR_Parameter_Output(Pose), pose, sizeof(NvAR_Quaternion));
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("failed to set pose output for expression feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    }
    
    // run feature
    expression_err = NvAR_Run(expression_handle);
    if (expression_err != NVCV_SUCCESS) {
        UtilityFunctions::print("\nfailed to run expression feature");
        UtilityFunctions::print(NvCV_GetErrorStringFromCode(expression_err));
    } else {
        UtilityFunctions::print("successfully ran expression feature!!!");
    }

I am trying to follow the initialization from the SDK guide for getting the expression from a static image, and the initialization from the ExpressionApp sample.

however, when I run this, I am getting a crash where it tries to access memory at a null pointer address. any ideas? i feel like i have to be close. any tips on how to debug would be appreciated. if i try to step into Nv_Run i get into stuff thats not human readable, i assume bc its inside the dll