Realize prediction with .trt model (Jetson Nano 2gb)


I have succeded to extract a prediction score from onnx model on my jetson nano 2gh as follows:

Include libraries

import cv2
import matplotlib.pyplot as plt
import tf2onnx
import numpy as np
import onnxruntime as rt

Convert tensorflow 2 keras 2 onnx model

spec = (tf.TensorSpec((1, 300, 300, 1), tf.float32, name=“input”),)
output_path = + “.onnx”
model_proto, _ = tf2onnx.convert.from_keras(model, input_signature=spec, opset=13, output_path=output_path)
output_names = [ for n in model_proto.graph.output]

load and deploy .onnx model

output_names = [‘dense_1’]
image_shape = (300,300,1)
output_path = “sequential.onnx”
providers = [‘CPUExecutionProvider’]
m = rt.InferenceSession(output_path, providers=providers)

img_path = ‘Test/cast_ok_0_35.jpeg’
img_pred = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
img_pred = cv2.resize(img_pred,(300,300))
img_pred = img_pred / 255 # rescale
onnx_pred =, {“input”: img_pred.reshape(1,*image_shape).astype(np.float32)})

if onnx_pred[0] < 0.5:
predicted_label = “defected”
prob = (1-onnx_pred[0].sum()) * 100
color = (255, 0, 0)
predicted_label = “good”
prob = onnx_pred[0].sum() * 100
color = (0, 255, 0)

img = cv2.imread(img_path)
img = cv2.resize(img,(300,300))
cv2.putText(img=img, text=f"Result : {predicted_label}", org=(10, 30), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.8, color=color, thickness=2)
cv2.putText(img=img, text=f"Probability : {’{:.3f}’.format(prob)}%", org=(10, 280), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.7, color=(0, 0, 255), thickness=2)

after converting onnx model to .trt engine (using trtexec) i cannot find a straightforwad method to mimic the same prediction using tensorrt

convert and check trt engine

import os
import tensorrt as trt

def load_engine(trt_runtime, engine_path):
trt.init_libnvinfer_plugins(None, “”)### Try to add here
with open(engine_path, ‘rb’) as f:
engine_data =
engine = trt_runtime.deserialize_cuda_engine(engine_data)
return engine

TRT_LOGGER = trt.Logger(trt.Logger.WARNING)
trt_runtime = trt.Runtime(TRT_LOGGER)
trt_engine_path = “casting_engine.trt”
trt_engine = load_engine(trt_runtime, trt_engine_path)

if trt_engine is not None:

extract prediction score fron .trt model


any help would be appreciated. thank you


This looks like a Jetson issue. Please refer to the below samples in case useful.

For any further assistance, we will move this post to to Jetson related forum.


All this examples uses pre-trained model like imagenet, there are no inference example to extract prediction score as what i have thanks