I have a DeepStream Pipeline that receives RTSP streams for inference. I am using resnet18_baseline_att_224x224_A_epoch_249.onnx generated from resnet18_baseline_att_224x224_A_epoch_249.pth using export_for_isaac.py however when integrating the model into the Pipeline it does not generate any inference visualization in the OSD. There is absolutely no information regarding how to receive information from this model in the tiler_sink_pad_buffer_probe() using a Pipeline in Python, is it a detector? classifier? tensor? If it is a tensor, what dimensions does it have? I don’t know how to get the metadata from this model and it’s not showing anything in the OSD. I have 2 cases: 1) I used it as a unique model 2) I used it as a secondary model while the primary is a YoloV8 detector, but the same problems in both cases, how can i got the metada and what kind of data is? tensor? There’s absolutely no information about this situation on internet, just very old codes written in C++ i really think that we need more tutorials, information, codes, etc in order to use in the better way the Nvidia software tools.
I just got this config file searching in internet:
[property]
gpu-id=0
net-scale-factor=0.0174292
offsets=123.675;116.28;103.53
onnx-file=/app/models/trt-pose/resnet18_baseline_att_224x224_A_epoch_249.onnx
model-engine-file=/app/models/trt-pose/resnet18_baseline_att_224x224_A_epoch_249.onnx_b1_gpu0_fp16.engine
#labelfile-path=labels.txt
batch-size=1
process-mode=1
model-color-format=0
network-mode=2
num-detected-classes=4
interval=0
gie-unique-id=1
network-type=100
workspace-size=3000
from this link: Deepstream_pose_estimation
This is my tiler_sink_pad_buffer_probe() using this trt-pose as secondary model in a Python Pipeline:
def tiler_sink_pad_buffer_probe(self, pad: Gst.Pad, info: Gst.PadProbeInfo, u_data: any) -> Gst.PadProbeReturn:
"""Probe function to get and send inference data from buffers going downstream.
Args:
pad (Gst.Pad): Pad where the probe is set
info (Gst.PadProbeInfo): Info going downtream
u_data (any): Custom user data
Returns:
Gst.PadProbeReturn: Always Gst.PadProbeReturn.OK so the buffers can flow downstream
"""
if (info.type & Gst.PadProbeType.BUFFER):
gst_buffer = info.get_buffer()
if not gst_buffer:
logger.warning("Unable to get GstBuffer ")
return
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
# outputNow = datetime.now()
# limitTime = (outputNow - self.last_info).total_seconds()
# if limitTime>=1: # Variable (Interval of seconds to send inferences to broker)
# sendSample = True
# self.last_info = outputNow
# else:
# sendSample = False
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
try:
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
except StopIteration:
break
frame_image = None
camId = self.sources[frame_meta.source_id].external_id
l_obj = frame_meta.obj_meta_list
while l_obj is not None:
try:
# Casting l_obj.data to pyds.NvDsObjectMeta
obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
except StopIteration:
break
l_user_meta = obj_meta.obj_user_meta_list
while l_user_meta is not None:
try:
user_meta = pyds.NvDsUserMeta.cast(l_user_meta.data)
except:
break
tensor_meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
logger.info(tensor_meta)
if tensor_meta.unique_id == self.cfg["inference"]["trt-pose"]["unique-id"] and tensor_meta.num_output_layers > 0:
layer = pyds.get_nvds_LayerInfo(tensor_meta, 0)
logger.info(layer)
#embedding = self.buffer_as_nparray(layer.buffer, (256,))
try:
l_user_meta = l_user_meta.next
except StopIteration:
break
tracker_id = obj_meta.object_id
confidence = obj_meta.confidence
if tracker_id not in self.lista_ids[camId]:
if frame_image is None:
n_frame=pyds.get_nvds_buf_surface(hash(gst_buffer),frame_meta.batch_id)
frame_image = np.array(n_frame,copy=True,order='C')
bbox = [max(int(obj_meta.tracker_bbox_info.org_bbox_coords.left), 0),
max(int(obj_meta.tracker_bbox_info.org_bbox_coords.top), 0),
int(obj_meta.tracker_bbox_info.org_bbox_coords.width),
int(obj_meta.tracker_bbox_info.org_bbox_coords.height)]
#logger.info(f'Object id: {tracker_id}')
# self.send_image({ 'image':{
# 'camId': camId,
# 'id' : tracker_id,
# 'confidence': confidence,
# 'bbox': bbox,
# 'crop' : frame_image[bbox[1]:bbox[1]+bbox[3], bbox[0]:bbox[0]+bbox[2]],
# 'line_crossing': line_crossing
# }})
# self.lista_ids[camId].append(tracker_id)
try:
l_obj = l_obj.next
except StopIteration:
break
# if sendSample: # TODO: Get in when there is at least one person.
# try:
# dict_base64_enc_copy = self.dict_base64_enc.copy()
# # we have to verify that there is base64 encoding in the id dict
# for id_ in list(dict_base64_enc_copy[camId]['person']):
# if not('image' in dict_base64_enc_copy[camId]['person'][id_]) :
# del dict_base64_enc_copy[camId]['person'][id_]
# data = {
# 'source': self.sources[int(frame_meta.source_id)],
# 'detections' : dict_base64_enc_copy[camId]['person']
# }
# self.lista_ids[camId] = [] # clean id's lists visited
# self.dict_base64_enc[camId]['person'] = {} # clean all ids processed for each camera
# except Exception as e:
# data = None
# logger.warning(f"It was not possible to create message with analytics data for source with internal id {frame_meta.source_id}. Error: {e}.")
# if data is None:
# data = {
# 'source': self.sources[int(frame_meta.source_id)],
# 'detections' : []
# }
# self.save_metrics(data)
try:
l_frame = l_frame.next
except StopIteration:
break
return Gst.PadProbeReturn.OK
TensorRT Version : 8.6.1.6
GPU Type : NVIDIA A100-SXM4-40GB
Nvidia Driver Version : 535.161.07
CUDA Version : 12.2
Operating System + Version : ubuntu 22.04
Python Version (if applicable) : 3.10.12
PyTorch Version (if applicable) : 2.3.1+cu121
Relevant Files
Model: resnet18_baseline_att_224x224_A resnet18_baseline_att_224x224_A_epoch_249.pth - Google Drive
trt-pose: GitHub - NVIDIA-AI-IOT/trt_pose: Real-time pose estimation accelerated with NVIDIA TensorRT