Description
I have succeded to extract a prediction score from onnx model on my jetson nano 2gh as follows:
Include libraries
import cv2
import matplotlib.pyplot as plt
import tf2onnx
import numpy as np
import onnxruntime as rt
Convert tensorflow 2 keras 2 onnx model
spec = (tf.TensorSpec((1, 300, 300, 1), tf.float32, name=“input”),)
output_path = model.name + “.onnx”
model_proto, _ = tf2onnx.convert.from_keras(model, input_signature=spec, opset=13, output_path=output_path)
output_names = [n.name for n in model_proto.graph.output]
print(output_names)
load and deploy .onnx model
output_names = [‘dense_1’]
image_shape = (300,300,1)
output_path = “sequential.onnx”
providers = [‘CPUExecutionProvider’]
m = rt.InferenceSession(output_path, providers=providers)
img_path = ‘Test/cast_ok_0_35.jpeg’
img_pred = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
print(img_pred.shape)
img_pred = cv2.resize(img_pred,(300,300))
img_pred = img_pred / 255 # rescale
onnx_pred = m.run(output_names, {“input”: img_pred.reshape(1,*image_shape).astype(np.float32)})
print(onnx_pred)
if onnx_pred[0] < 0.5:
predicted_label = “defected”
prob = (1-onnx_pred[0].sum()) * 100
color = (255, 0, 0)
else:
predicted_label = “good”
prob = onnx_pred[0].sum() * 100
color = (0, 255, 0)
plt.figure(figsize=(32,18))
img = cv2.imread(img_path)
img = cv2.resize(img,(300,300))
cv2.putText(img=img, text=f"Result : {predicted_label}“, org=(10, 30), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.8, color=color, thickness=2)
cv2.putText(img=img, text=f"Probability : {‘{:.3f}’.format(prob)}%”, org=(10, 280), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.7, color=(0, 0, 255), thickness=2)
plt.imshow(img,cmap=‘gray’)
plt.axis(‘off’)
plt.show()
after converting onnx model to .trt engine (using trtexec) i cannot find a straightforwad method to mimic the same prediction using tensorrt
convert and check trt engine
import os
import tensorrt as trt
def load_engine(trt_runtime, engine_path):
trt.init_libnvinfer_plugins(None, “”)### Try to add here
with open(engine_path, ‘rb’) as f:
engine_data = f.read()
engine = trt_runtime.deserialize_cuda_engine(engine_data)
return engine
TRT_LOGGER = trt.Logger(trt.Logger.WARNING)
trt_runtime = trt.Runtime(TRT_LOGGER)
trt_engine_path = “casting_engine.trt”
trt_engine = load_engine(trt_runtime, trt_engine_path)
if trt_engine is not None:
print(“Success”)
else:
print(“Failed”)
extract prediction score fron .trt model
???
any help would be appreciated. thank you