Is it possible to convert the python function to deepstream function?
If so, will the program run faster compared to normal python function call.
My target: face id recognition of unknown person from videos, I have python code for face id recognition using face recognition library, So can I run this code to deepstream ?
Can you elaborate your “python function” and “deeppstream function”?
python function as below:
def detect(video_file, video_id, frame_count):
make_video_directories(video_id)
init() # load model
model = YOLO("weights/yolov8n-face.pt")
results = model.predict(video_file, save=False)
frames = []
for id, frame in enumerate(results):
boxes = []
xyxys = []
try:
for *xyxy, conf, cls in reversed(frame):
x1, y1, x2, y2 = xyxy[0], xyxy[1], xyxy[2], xyxy[3]
boxes.append([int(x1), int(y1), int(x2) - int(x1) , int(y2) - int(y1)])
xyxys.append(xyxy)
frames.append({
'id': id+1,
'boxes': boxes,
'xyxys': xyxys
})
except Exception as e:
# print('reversed(frame) ', reversed(frame))
print('eror1 ', e)
video_cap = cv2.VideoCapture(video_file)
# MP4
annotated_video_path_mp4 = f"files/{video_id}/annotated_video.mp4"
fourcc_mp4 = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
# Webm
annotated_video_path_webm = f"files/{video_id}/annotated_video_webm.webm"
fourcc_webm = cv2.VideoWriter_fourcc('V', 'P', '8', '0')
fps = video_cap.get(cv2.CAP_PROP_FPS)
width = video_cap.get(cv2.CAP_PROP_FRAME_WIDTH)
height = video_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
# Create video writer
vid_writer_mp4 = cv2.VideoWriter(annotated_video_path_mp4, fourcc_mp4, fps, (int(width), int(height)))
vid_writer_webm = cv2.VideoWriter(annotated_video_path_webm, fourcc_webm, fps, (int(width), int(height)))
# Check video is rotated or not
rotate = check_rotation(video_file)
known_face_encodings = []
detect_frames = []
all_faces = []
while video_cap.isOpened():
ret, frame = video_cap.read()
if not ret:
break
current_frame_no = int(video_cap.get(cv2.CAP_PROP_POS_FRAMES))
if rotate:
frame = correct_rotation(frame)
if len(frames) != 0 and len(frames[current_frame_no-1]['boxes']):
# original_frame_path = os.path.join(f"files/{video_id}/original_frames/", str(current_frame_no) + ".jpg")
# cv2.imwrite(original_frame_path, frame)
original_frame_path = ''
imgpath = os.path.join(f"files/{video_id}/frames/", str(current_frame_no) + ".jpg")
resized = cv2.resize(frame, (112, 60), interpolation=cv2.INTER_AREA)
cv2.imwrite(imgpath, resized)
faces = []
found_faces = []
for box_id, box in enumerate(frames[current_frame_no-1]['boxes']):
x1, y1, w, h = box
x2, y2 = x1 + w, y1 + h
# Add checks to ensure valid coordinates
if x1 >= 0 and y1 >= 0 and x2 <= frame.shape[1] and y2 <= frame.shape[0]:
# Find face_id
face_id = 0
try:
face_location = face_recognition.face_locations(np.array(frame[y1:y2, x1:x2]), model="hog")
if len(face_location) == 0:
face_location = face_recognition.face_locations(np.array(frame[y1:y2, x1:x2]), model="cnn")
face_encoding = face_recognition.face_encodings(np.array(frame[y1:y2, x1:x2]), face_location)[0]
matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
if True in matches:
first_match_index = matches.index(True)
face_id = first_match_index+1
else:
known_face_encodings.append(face_encoding)
face_id = len(known_face_encodings)
face_path = f'files/{video_id}/faces/{face_id}.jpg'
cv2.imwrite(face_path, frame[y1:y2, x1:x2])
all_faces.append({
'image': upload_file_to_storage(face_path),
'name': '',
'user_id': face_id
})
except Exception as e:
print('Error ', str(e))
continue
if face_id != 0:
if original_frame_path == "":
original_frame_path = os.path.join(f"files/{video_id}/original_frames/", str(current_frame_no) + ".jpg")
cv2.imwrite(original_frame_path, frame)
emotions = detect_emotion(frame[y1:y2, x1:x2])
label = max(emotions, key=lambda x: emotions[x])
color = emotion_colors[emotions_labels.index(label)]
faces.append({
"box": [int(x1), int(y1), int(x2) - int(x1) , int(y2) - int(y1)],
'emotions': emotions,
'face_id': int(face_id),
'frame_id': current_frame_no,
})
found_faces.append(int(face_id))
plot_one_box(frames[current_frame_no-1]['xyxys'][box_id], frame, label=label, color=color, line_thickness=2)
if len(found_faces) != 0:
detect_frames.append({
'faces': faces,
'found_faces': found_faces,
'frame_id': current_frame_no,
'frame_image': upload_file_to_storage(imgpath),
'note': '',
'original_frame_image': upload_file_to_storage(original_frame_path),
'timestamp': int((current_frame_no / fps ) * 1000)
})
vid_writer_mp4.write(frame)
vid_writer_webm.write(frame)
if current_frame_no % 10 == 0:
update_video_status(video_id, {
u'total_frame_count': frame_count,
u'total_frames_processed': current_frame_no,
u'status': 'Processing',
u'percentage': percentage_processed(current_frame_no, frame_count)
})
#add audio
video_cap.release()
vid_writer_mp4.release()
vid_writer_webm.release()
update_video_status(video_id, {
u'total_frame_count': frame_count,
u'total_frames_processed': frame_count,
u'status': 'Adding Audio',
u'percentage': 100
})
annotated_video_mp4 = add_audio_in_video(video_id, annotated_video_path_mp4, "mp4")
annotated_video_webm = add_audio_in_video(video_id, annotated_video_path_webm)
# Generate summary text, transcription from audio
summary, audio_transcription = '', []
audio_file_location = f"files/{video_id}/audio.mp3"
if os.path.exists(audio_file_location):
start_time_audio = time.perf_counter()
summary, audio_transcription = generate_audio_summary(audio_file_location)
print(f'time taken to generate summary {time.perf_counter() - start_time_audio} seconds',)
return {
'faces': all_faces,
'status' : True,
'annotated_video_mp4': annotated_video_mp4,
'annotated_video_webm': annotated_video_webm,
'summary': summary,
'audio_transcription': audio_transcription,
}, detect_frames
from this code, I need to take face id recognition part code and want to convert deepstream function .
I already created deepstream function code for face detection. Next step is face_recognition
Hope you understand.
Most of the code may not be used in DeepStream. The DeepStream is based on Gstreamer and tensorRT. Your code is based on OpenCV. You can refer to our demo code first deepstream_python_apps. Just change some configuration files and pipeline in the code should meet your needs.
Yes , I know.
But I need face recognition part… Which use face recognition library … Dlib for detector …
In that case, can I convert face recognition code part only to deepstream function
Could you provide a detailed description of your current pipeline and which modules do you want to use Deepstream to handle?
yes.
Yolov8 for face detection and transfer learning model for emotion detection.
Both code working fine with deepstream format. By referring other repo, I done it.
Next step is: face recognition( faceid generation of unknown persons with face recognition library ), this one I try to convert but not working. Thats why i asked how to convert face recognition function code into deepstream function
my code :
Find face_id
face_id = 0
try:
face_location = face_recognition.face_locations(np.array(frame[y1:y2, x1:x2]), model="hog")
if len(face_location) == 0:
face_location = face_recognition.face_locations(np.array(frame[y1:y2, x1:x2]), model="cnn")
face_encoding = face_recognition.face_encodings(np.array(frame[y1:y2, x1:x2]), face_location)[0]
matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
if True in matches:
first_match_index = matches.index(True)
face_id = first_match_index+1
else:
known_face_encodings.append(face_encoding)
face_id = len(known_face_encodings)
face_path = f'files/{video_id}/faces/{face_id}.jpg'
cv2.imwrite(face_path, frame[y1:y2, x1:x2])
all_faces.append({
'image': upload_file_to_storage(face_path),
'name': '',
'user_id': face_id
})
except Exception as e:
print('Error in face id gnrtin : ', str(e))
continue
if face_id != 0:
emotions = detect_emotion(frame[y1:y2, x1:x2])
label = max(emotions, key=lambda x: emotions[x])
color = emotion_colors[emotions_labels.index(label)]
faces.append({
"box": [int(x1), int(y1), int(x2) - int(x1) , int(y2) - int(y1)],
'emotions': emotions,
'face_id': int(face_id),
'frame_id': current_frame_no,
})
found_faces.append(int(face_id))
plot_one_box(frames[current_frame_no-1]['xyxys'][box_id], frame, label=label, color=color, line_thickness=2)
if len(found_faces) != 0:
if first_frame:
original_frame_path = os.path.join(f"files/{video_id}/original_frames/", str(current_frame_no) + ".jpg")
cv2.imwrite(os.path.join(f"files/{video_id}/original_frames/", str(current_frame_no) + ".jpg"), frame)
original_frame_path = upload_file_to_storage(original_frame_path)
first_frame = False
else:
original_frame_path = ''
What do you mean deepstream format? Do you want to replace your face recognition module with DeepStream？I’m still not sure what you want to do with your code.
Could you provide a detailed description of your current pipeline, like source->decode->streammux->pgie->sgie->nvdsosd->filesink?
If you want use your own algorithm in DeepStream to process the inferenced data, you need to use the postprocess plugin to implement that.
It seems like we want to replace our current face recognition module with DeepStream. our goal is to process input videos and generate a unique face ID for each frame containing a face. If a person with a specific ID, say “A,” appears in different frames, we want the face ID to remain constant (i.e., faceID=A). However, if a new person is detected, our aim to generate a new face ID for that individual.
We referring this repo for face detection: GitHub - marcoslucianops/DeepStream-Yolo-Face: NVIDIA DeepStream SDK 6.3 / 6.2 / 6.1.1 / 6.1 / 6.0.1 / 6.0 application for YOLO-Face models
Next, i want to add faceid generation in this code.
Can you please help me to solve this issues?
There is no update from you for a period, assuming this is not an issue anymore.
Hence we are closing this topic. If need further support, please open a new one.
Thanks
We suggest that you have a brief understanding of how DeepStream works. You can refer to some of our samples deepstream-samples.
About your user case, you can achieve this by changing some configurations in the project.
config_infer_primary_yoloV8_face.txt
And impletment your own algorithm here: nvdsparseface_Yolo.cpp.
Please refer to our nvinfer plugin: DS_plugin_gst-nvinfer.html.
This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.