Hi @junshengy,
We have resolved the issue and are now able to get embeddings from ArcFace model. I’ve also created a pipeline outside of DeepStream to store the generated embeddings for training (for using while recognition).
I’ve integrated the recognition model (SGIE) into the DeepStream pipeline, but when trying to extract the embeddings from the SGIE metadata, I am getting None from obj_meta.usr_list_meta. Based on my understanding, when the SGIE (recognition) follows the PGIE (detection), and with process-mode=1 and output-tensor-meta=1 set on the SGIE, the generated embeddings should be stored in the user list of the object. However, when I attempt to extract the embeddings from obj_meta.usr_list_meta, I’m encountering the following issue:
The log from my terminal which states my issue:
Entering PGIE filter function
Processing frame 0
Objects detected in frame 0
Bounding Box: {‘top’: 342, ‘left’: 707, ‘width’: 244, ‘height’: 334}
Face Recognition started
Getting the Face Features
l_user_meta is None
Face feature : None
Entering PGIE filter function
Processing frame 0
Objects detected in frame 0
Bounding Box: {‘top’: 343, ‘left’: 711, ‘width’: 241, ‘height’: 324}
Face Recognition started
Getting the Face Features
l_user_meta is None
Face feature : None
Could you please guide me on where to correctly extract the embeddings generated by the SGIE model within the DeepStream pipeline? Additionally, would using the embeddings generated outside of DeepStream for recognition work, or is it essential to extract them from the pipeline itself?
Here’s the SGIE config file I’m using:
sgie_config_webface.txt (797 Bytes)
And here’s the probe I attached to the SGIE for embedding extraction:
def sgie_feature_extract_probe(pad, info , data):
print("Face Recognition started ")
"""
Probe to extract facial feature from user-meta data and perform recognition.
Args:
pad: GstPad.
info: GstPadProbeInfo.
data: Tuple containing (loaded_faces, threshold, output_dir).
"""
loaded_faces = data # Dictionary of embeddings from the pickle file
threshold = 0.7 # Similarity threshold
#output_dir = data[2] # Directory to save embeddings (optional)
gst_buffer = info.get_buffer()
if not gst_buffer:
print("Unable to get GstBuffer")
return Gst.PadProbeReturn.OK
batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(gst_buffer))
l_frame = batch_meta.frame_meta_list
while l_frame is not None:
try:
frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
except StopIteration:
break
l_obj = frame_meta.obj_meta_list
frame_number = frame_meta.frame_num
while l_obj is not None:
try:
obj_meta = pyds.NvDsObjectMeta.cast(l_obj.data)
except StopIteration:
break
# Extract the face embedding
if obj_meta is None:
print("object meta is None")
face_feature = get_face_feature(obj_meta)
print("Face feature : ", face_feature)
if face_feature is not None:
# Perform similarity scoring
best_match = None
best_score = -1
for key, value in loaded_faces.items():
score = np.dot(face_feature, value.T)[0][0] # Cosine similarity
if score > best_score:
best_score = score
best_match = key
# Display the result if the score exceeds the threshold
if best_score > threshold:
display_meta = pyds.nvds_acquire_display_meta_from_pool(batch_meta)
display_meta.num_labels = 1
py_nvosd_text_params = display_meta.text_params[0]
# Set the display text (name of the matched face)
py_nvosd_text_params.display_text = best_match
# Set the position of the text
py_nvosd_text_params.x_offset = int(obj_meta.rect_params.left)
py_nvosd_text_params.y_offset = int(obj_meta.rect_params.top + obj_meta.rect_params.height)
# Set font properties
py_nvosd_text_params.font_params.font_name = "Serif"
py_nvosd_text_params.font_params.font_size = 20
py_nvosd_text_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0) # White color
# Set text background color
py_nvosd_text_params.set_bg_clr = 1
py_nvosd_text_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0) # Black background
# Add the display meta to the frame
pyds.nvds_add_display_meta_to_frame(frame_meta, display_meta)
try:
l_obj = l_obj.next
except StopIteration:
break
try:
l_frame = l_frame.next
except StopIteration:
break
return Gst.PadProbeReturn.OK
def get_face_feature(obj_meta):
print(“Getting the Face Features”)
“”"Get face feature from user-meta data.
Args:
obj_meta (NvDsObjectMeta): Object metadata.
Returns:
np.array: Normalized face feature.
"""
l_user_meta = obj_meta.obj_user_meta_list
if l_user_meta is None:
print("l_user_meta is None")
#print ( f"l_user_meta: {l_user_meta}")
while l_user_meta:
try:
user_meta = pyds.NvDsUserMeta.cast(l_user_meta.data)
except StopIteration:
break
if user_meta and user_meta.base_meta.meta_type == pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META:
try:
tensor_meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)
except StopIteration:
break
layer = pyds.get_nvds_LayerInfo(tensor_meta, 0)
output = []
for i in range(512): # Assuming the embedding size is 512
output.append(pyds.get_detections(layer.buffer, i))
print("output : ",output)
res = np.reshape(output, (1, -1))
print("result")
norm = np.linalg.norm(res)
normal_array = res / norm # Normalize the embedding
print("Normal array: ", normal_array)
return normal_array
try:
l_user_meta = l_user_meta.next
except StopIteration:
break
return None
