• Hardware Platform (Jetson / GPU) : NVIDIA Jetson AGX Orin
• DeepStream Version : 7.0
• JetPack Version (valid for Jetson only) : 6.0
• TensorRT Version : 8.6.2.3
• Issue Type( questions, new requirements, bugs) : question
Hello,
I would like to plug appsink element right after nvinfer in my pipeline.
The extract of my pipeline looks like:
...
arc_pgie = create_pipeline_element("nvinfer", "primary-inference", "Primary Inference")
arc_pgie.set_property("config-file-path", config_path)
arc_appsink = create_pipeline_element("appsink", "appsink", " AppSink")
arc_appsink.set_property("emit-signals", True)
arc_appsink.set_property("sync", False)
arc_appsink.connect("new-sample", on_new_sample) # Connect callback to new-sample signal
...
arc_pgie.link(arc_appsink)
...
pgie_src_pad = arc_pgie.get_static_pad("src")
if not pgie_src_pad:
sys.stderr.write("Unable to get src pad of primary infer\n")
pgie_src_pad.add_probe(Gst.PadProbeType.BUFFER, pgie_src_pad_buffer_probe, 0)
In nvinfer element i perform segmentation so pgie_src_pad_buffer_probe looks like:
def pgie_src_pad_buffer_probe(pad, info, u_data):
gst_buffer = info.get_buffer()
if not gst_buffer:
print("Unable to get GstBuffer")
return Gst.PadProbeReturn.OK
# Retrieve batch metadata from the gst_buffer
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
print(f"Batch size after inference: {batch_meta.num_frames_in_batch}")
while l_frame is not None:
try:
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
except StopIteration:
break
l_obj = frame_meta.obj_meta_list
while l_obj is not None:
try:
obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
except StopIteration:
break
class_meta_list = obj_meta.classifier_meta_list
while class_meta_list is not None:
try:
classifier_meta = pyds.NvDsClassifierMeta.cast(class_meta_list.data)
except StopIteration:
break
label_info_list = classifier_meta.label_info_list
while label_info_list is not None:
try:
label_info = pyds.NvDsLabelInfo.cast(label_info_list.data)
except StopIteration:
break
arcing_prob = label_info.result_prob
is_arcing = arcing_prob > 0.5
if not is_arcing:
continue
ntp_timestamp = int(frame_meta.ntp_timestamp / 1_000_000)
img = get_frame(gst_buffer, frame_meta.batch_id)
alpha = np.ones(img.shape[:-1]).astype(np.uint8) * 255
img = np.dstack((img, alhpa))
out_filepath224 = save_img(img, f"{ntp_timestamp}224")
redis_data = {
"timestamp": ntp_timestamp,
"arcing": int(is_arcing),
"probability": float(arcing_prob),
"files": {
"res224": out_filepath224,
},
}
r_client.zadd(
"arcing-detection", {json.dumps(redis_data): ntp_timestamp}
)
try:
label_info_list = label_info_list.next
except StopIteration:
break
try:
class_meta_list = class_meta_list.next
except StopIteration:
break
try:
l_obj = l_obj.next
except StopIteration:
break
try:
l_frame = l_frame.next
except StopIteration:
break
return Gst.PadProbeReturn.OK
I would like to move this whole operation of saving a file and sending data to Redis:
out_filepath224 = save_img(img, f"{ntp_timestamp}224")
redis_data = {
"timestamp": ntp_timestamp,
"arcing": int(is_arcing),
"probability": float(arcing_prob),
"files": {
"res224": out_filepath224,
},
}
r_client.zadd(
"segmentation", {json.dumps(redis_data): ntp_timestamp}
)
to appsink. How can I achieve this in Python? I tried using something like this:
def pgie_src_pad_buffer_probe(pad, info, u_data):
gst_buffer = info.get_buffer()
if not gst_buffer:
print("Unable to get GstBuffer")
return Gst.PadProbeReturn.OK
# Retrieve batch metadata from the gst_buffer
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
print(f"Batch size after inference: {batch_meta.num_frames_in_batch}")
while l_frame is not None:
try:
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
except StopIteration:
break
l_obj = frame_meta.obj_meta_list
while l_obj is not None:
try:
obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
except StopIteration:
break
class_meta_list = obj_meta.classifier_meta_list
while class_meta_list is not None:
try:
classifier_meta = pyds.NvDsClassifierMeta.cast(class_meta_list.data)
except StopIteration:
break
label_info_list = classifier_meta.label_info_list
while label_info_list is not None:
try:
label_info = pyds.NvDsLabelInfo.cast(label_info_list.data)
except StopIteration:
break
arcing_prob = label_info.result_prob
is_arcing = arcing_prob > 0.5
if not is_arcing:
continue
ntp_timestamp = int(frame_meta.ntp_timestamp / 1_000_000)
img = get_frame(gst_buffer, frame_meta.batch_id)
alpha = np.ones(img.shape[:-1]).astype(np.uint8) * 255
img = np.dstack((img, alhpa))
# ----------Here is the change-----------
# Create a new Gst.Buffer to push to appsink
img_bytes = img.tobytes()
buffer = Gst.Buffer.new_wrapped(img_bytes)
buffer.pts = ntp_timestamp
# Push the buffer to the next element
pad.push(buffer)
# ------------------------
try:
label_info_list = label_info_list.next
except StopIteration:
break
try:
class_meta_list = class_meta_list.next
except StopIteration:
break
try:
l_obj = l_obj.next
except StopIteration:
break
try:
l_frame = l_frame.next
except StopIteration:
break
return Gst.PadProbeReturn.OK
However i received Error that looks like when performing push operation the frame_meta_list is no longer valid and it is None:
l_frame = batch_meta.frame_meta_list
^^^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'frame_meta_list'
What I want to achieve?
I would like to only process and extract image, probability and timestamp in probe function connected to nvinfer element and then perform post-processing of data in appsink. My question is, how can I pass this data from nvinfer element to the next element which is appsink? Is there maybe more efficient way like passing the images that are numpy arrays to filesink element and then adding probe function that saves the data to Redis?