Trying to parse output of the engine of DetectNetV2 in TensorRT.
I have TRT engine directly loaded to TensorRT and trying to parse output.
The following codes are used for my inference using TRT enigne.
Load fp16 engine
bool NumberPLate::build()
{
initLibNvInferPlugins(&sample::gLogger.getTRTLogger(), "");
if (mParams.loadEngine.size() > 0)
{
std::vector<char> trtModelStream;
size_t size{0};
std::ifstream file(mParams.loadEngine, std::ios::binary);
if (file.good())
{
file.seekg(0, file.end);
size = file.tellg();
file.seekg(0, file.beg);
trtModelStream.resize(size);
file.read(trtModelStream.data(), size);
file.close();
}
IRuntime* infer = nvinfer1::createInferRuntime(sample::gLogger);
if (mParams.dlaCore >= 0)
{
infer->setDLACore(mParams.dlaCore);
}
mEngine = std::shared_ptr<nvinfer1::ICudaEngine>(infer->deserializeCudaEngine(trtModelStream.data(), size, nullptr), samplesCommon::InferDeleter());
infer->destroy();
sample::gLogInfo << "TRT Engine loaded from: " << mParams.loadEngine << std::endl;
if (!mEngine)
{
return false;
}
}
return true;
}
Then do the inference.
bool NumberPLate::infer()
{
VideoCapture cap;
if(mParams.url!=""){
cap.open(mParams.url);
// Check if camera opened successfully
if(!cap.isOpened()){
cout << "Error opening video stream or file" << endl;
return -1;
}
}
// Create RAII buffer manager object
samplesCommon::BufferManager buffers(mEngine, mParams.batchSize);
auto context = SampleUniquePtr<nvinfer1::IExecutionContext>(mEngine->createExecutionContext());
SimpleProfiler profiler("Numplate performance");
if (mParams.profile)
{
context->setProfiler(&profiler);
}
if (!context)
{
return false;
}
std::vector<float> boxes;
while(1){
Mat frame;
cap >> frame;
if (frame.empty())
break;
//Read the input data into the managed buffers
if (!processInput(buffers, frame))
{
return false;
}
// Memcpy from host input buffers to device input buffers
buffers.copyInputToDevice();
bool status{true};
status = context->execute(mParams.batchSize, buffers.getDeviceBindings().data());
if (!status)
{
return false;
}
if (mParams.profile)
{
std::cout << profiler;
}
// Memcpy from device output buffers to host output buffers
buffers.copyOutputToHost();
// Post-process detections and verify results
if (!verifyOutput(buffers, boxes))
{
return false;
}
boxes.clear();
char c=(char)waitKey(25);
if(c==27)
break;
}
vector<float>().swap(boxes);
cap.release();
return true;
}
Inside this verifyOutput I need to parse.
bool NumberPLate::verifyOutput(const samplesCommon::BufferManager& buffers, std::vector<float> &pred_boxes)
{
const int batchSize = mParams.batchSize;
const float* out_class = static_cast<const float*>(buffers.getHostBuffer("output_bbox/BiasAdd"));
const float* out_reg = static_cast<const float*>(buffers.getHostBuffer("output_cov/Sigmoid"));
const float* out_proposal = static_cast<const float*>(buffers.getHostBuffer(mParams.outputProposalName));
//batch_inverse_transform_classifier(out_proposal, post_nms_top_n, out_class, out_reg, pred_boxes, pred_cls_ids,
// pred_probs, box_num_per_img, batchSize);
//visualize_boxes(batchSize, outputClassSize, pred_boxes, pred_probs, pred_cls_ids, box_num_per_img, ppms);
return true;
}
When I look at buffer, it has three vectors. How can I extract these three vectors?
What I know is DetectNetV2 has only two output blobs, output_bbox/BiasAdd;output_cov/Sigmoid.
(gdb) p buffers
$1 = {static kINVALID_SIZE_VALUE = 18446744073709551615,
mEngine = std::shared_ptr<nvinfer1::ICudaEngine> (use count 2, weak count 0) = {get() = 0x5555e447c0}, mBatchSize = 1,
mManagedBuffers = std::vector of length 3, capacity 4 = {
std::unique_ptr<samplesCommon::ManagedBuffer> = {get() = 0x5567f4f230},
std::unique_ptr<samplesCommon::ManagedBuffer> = {get() = 0x55777ab860},
std::unique_ptr<samplesCommon::ManagedBuffer> = {get() = 0x5577758d40}},
mDeviceBindings = std::vector of length 3, capacity 4 = {0x2122ea000,
0x20dcea000, 0x20dd29c00}}