Hi all!! I’m using a Jetson Orin developer kit, a custom implementation of Yolov7 to detect objects and DeepSort for tracking.
The code I’m using is the following:
import logging
from datetime import datetime
from pathlib import Path
import random
from typing import List
import numpy as np
from deep_sort.deep_sort import Deep
from enums.camera.RunningCameraStatus import RunningCameraStatus
from models.CameraRunningContext import CameraRunningContext
from models.measurements.CameraSingleMeasurementValue import CameraSingleMeasurementValue
from object_detection import ObjectDetection
from services.detectors.CameraDetectorService import CameraDetectorService
import cv2
class Yolo7CameraDetectorService(CameraDetectorService):
def __init__(self):
super().__init__()
#Define a dictionary that
# Key : integer
# Value : timestamp of last updated(exit camera timestamp)
self.__detected_items = dict()
"""
Detect items using yolo7
Store items that remains in the current frame in the detected items dictionary
Remove all the other items and write them to camera measurements buffer
"""
def detect(self, camera: CameraRunningContext):
#Call abstract method
super().detect(camera)
# Counting area ROI
yellow_poly_pts = np.array([[0, 150], [1440, 150], [1440, 850], [0, 850]])
#Define codec for video
#fourcc = cv2.VideoWriter_fourcc(*'XVID')
# Load Object Detection
# ATTENTION
# Object detection applies toLower() on path
path_waights = Path(camera.camera.detection_weights_path)
od = ObjectDetection(path_waights)
od.load_detection_model(image_size=1088, confThreshold=0.5, nmsThreshold=0.4)
od.load_class_names(camera.camera.detection_classes_names)
# Load Object Tracking Deep Sort
deep = Deep(max_distance=0.8,
nms_max_overlap=0.3,
n_init=3,
max_age=5,
max_iou_distance=0.7)
tracker = deep.sort_tracker()
# While running camera status is enabled and streaming and detecting
while camera.status is RunningCameraStatus.ENABLED_AND_STREAMING_AND_DETECTING :
try:
now = datetime.utcnow()
# Get current raw image frame
image = camera.get_cur_raw_image()
a=5
# Convert to RGB image
rgb = image.convert("RGB")
# Convert np array
array = rgb.get_numpy_array()
# opencv loads the format bgr so the frame needs to be converted to rgb to process
array = cv2.cvtColor(array, cv2.COLOR_BGR2RGB)
#np.save("/home/christina/Desktop/test.np",array)
logging.info("Get frame and convert to cvtcolor in mseconds=" + str( (datetime.utcnow() - now).total_seconds() * 1000 ))
""" 1. Object Detection """
(class_ids, scores, boxes) = od.detect(array)
logging.info(
"od detect only in mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))
""" 2. Object Tracking """
features = deep.encoder(array, boxes)
detections = deep.Detection(boxes, scores, class_ids, features)
logging.info(
"tracking only in mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))
# Draw yellow polygon for the ROI
cv2.polylines(array, [yellow_poly_pts], True, (0, 255, 255), 2)
tracker.predict()
(class_ids, object_ids, boxes) = tracker.update(detections)
logging.info("Object dtection and tracker in mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))
# Iterate tracker results
for class_id, object_id, box in zip(class_ids, object_ids, boxes):
(x, y, x2, y2) = box
class_name = od.names[class_id]
color = od.colors[class_id]
# Center doubleroll detected
cx = int((x + x2) / 2)
cy = int((y + y2) / 2)
cv2.rectangle(array, (x, y), (x2, y2), (0, 0, 255), 2)
cv2.rectangle(array, (x, y), (x + len(class_name) * 20, y - 30), (0, 0, 255), -1)
cv2.putText(array, class_name + " " + str(object_id), (x, y - 10), 0, 0.75, (255, 255, 255), 2)
cv2.circle(array, (cx, cy), 4, (0, 0, 255), -1)
"""3. Object Counting"""
# Check if object is in the ROI
result = cv2.pointPolygonTest(yellow_poly_pts, (int(cx), int(cy)), False)
#print(result, object_id, datetime.utcnow())
if result > 0:
# Store item to dictionary as camera single measurement value
self.__detected_items[object_id] = CameraSingleMeasurementValue(value=1, timestamp= datetime.utcnow())
print("FOUND ONE!!!", result)
#cv2.imwrite("/home/christina/Desktop/yv7run/Img005.png", array)
print("Done with image")
#imS = cv2.resize(array, (1080, 1080))
#cv2.imshow("Frame", imS)
#Outside iteration
#Remove all items that does not exist in the current object ids list
# The current frame detected items must be added before to the dictionary
camera_measurement_values = self.__remove_and_get_all_except(object_ids)
#Add values to buffers
self.camera_buffered_measurements_service.add_multi(camera.camera.crguid, camera_measurement_values)
logging.info(
"Total mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))
except Exception as ex:
logging.error("Failed to process frame for camera with crguid=" + str(camera.camera.crguid))
logging.error(ex)
"""
Remove and retutn all items ids from
"""
def __remove_and_get_all_except(self, object_ids) -> List[CameraSingleMeasurementValue]:
# Get all dictionary keys that does not exist in the current frame detected objects
keys = [key for key in self.__detected_items.keys() if key not in object_ids]
# Get values for the above keys
values = [self.__detected_items.get(key) for key in keys]
# Remove above keys form dictionary
for key in keys: del self.__detected_items[key]
# Dictionary contains only detected items that are not exited camera and are still trackable
# Return items that exits camera context
return values
The images are coming directly from a machine vision camera and the packages with their versions at the virtual environment I’m working on are the following:
- absl-py 1.4.0
- aiohttp 3.8.1
- aiosignal 1.2.0
- amqp 5.1.1
- asgiref 3.5.1
- asttokens 2.2.1
- astunparse 1.6.3
- async-timeout 4.0.2
- asyncio 3.4.3
- attrs 21.4.0
- autopep8 1.6.0
- awscli 1.25.21
- backcall 0.2.0
- billiard 3.6.4.0
- bleach 5.0.0
- botocore 1.27.21
- cachetools 5.3.0
- certifi 2021.10.8
- cffi 1.15.0
- chardet 3.0.4
- charset-normalizer 2.0.12
- click 8.1.2
- click-didyoumean 0.3.0
- click-plugins 1.1.1
- click-repl 0.2.0
- colorama 0.4.4
- commonmark 0.9.1
- cryptography 40.0.1
- cycler 0.11.0
- decorator 5.1.1
- Deprecated 1.2.13
- docutils 0.16
- executing 1.2.0
- Flask 2.1.2
- flatbuffers 23.3.3
- fonttools 4.33.3
- frozenlist 1.3.0
- gast 0.4.0
- gevent 21.12.0
- google-auth 2.17.0
- google-auth-oauthlib 0.4.6
- google-pasta 0.2.0
- greenlet 1.1.2
- grpcio 1.53.0
- h11 0.8.1
- h2 3.2.0
- h5py 3.8.0
- hpack 3.0.0
- http3 0.6.7
- hyperframe 5.2.0
- idna 2.10
- importlib-metadata 4.11.3
- imutils 0.5.4
- ipython 8.11.0
- itsdangerous 2.1.2
- jedi 0.18.2
- jeepney 0.8.0
- Jinja2 3.1.1
- jmespath 1.0.1
- keras 2.11.0
- keyring 23.6.0
- kiwisolver 1.4.3
- kombu 5.2.4
- libclang 16.0.0
- Markdown 3.4.3
- MarkupSafe 2.1.1
- matplot 0.1.9
- matplotlib 3.5.2
- matplotlib-inline 0.1.6
- mpmath 1.3.0
- multidict 6.0.2
- networkx 3.0
- numpy 1.23.5
- oauthlib 3.2.2
- opencv-python 4.7.0.72
- opt-einsum 3.3.0
- packaging 21.3
- pandas 1.5.3
- parso 0.8.3
- pexpect 4.8.0
- pickleshare 0.7.5
- pika 1.2.0
- Pillow 9.1.1
- pip 23.0.1
- pkginfo 1.8.3
- prompt-toolkit 3.0.38
- protobuf 3.19.5
- psutil 5.9.4
- ptyprocess 0.7.0
- pure-eval 0.2.2
- pyasn1 0.4.8
- pyasn1-modules 0.2.8
- pycodestyle 2.8.0
- pycparser 2.21
- Pygments 2.12.0
- pyloco 0.0.139
- pyparsing 3.0.8
- python-dateutil 2.8.2
- pytz 2022.1
- pywin32-ctypes 0.2.0
- PyYAML 5.4.1
- readme-renderer 35.0
- redis 4.2.2
- requests 2.27.1
- requests-oauthlib 1.3.1
- requests-toolbelt 0.9.1
- rfc3986 1.5.0
- rich 12.4.4
- rsa 4.7.2
- s3transfer 0.6.0
- schedule 1.1.0
- scipy 1.8.0
- seaborn 0.12.2
- SecretStorage 3.3.3
- setuptools 61.0.0
- SimpleWebSocketServer 0.1.1
- six 1.16.0
- stack-data 0.6.2
- sympy 1.11.1
- tensorboard 2.11.2
- tensorboard-data-server 0.6.1
- tensorboard-plugin-wit 1.8.1
- tensorflow 2.11.0
- tensorflow-cpu-aws 2.11.0
- tensorflow-estimator 2.11.0
- tensorflow-io-gcs-filesystem 0.32.0
- termcolor 2.2.0
- thop 0.1.1.post2209072238
- tk 0.1.0
- toml 0.10.2
- torch 1.14.0a0+44dac51c.nv23.2
- torchvision 0.14.1a0+5e8e2f1
- tqdm 4.64.0
- traitlets 5.9.0
- twine 4.0.1
- typing 3.7.4.3
- typing_extensions 4.5.0
- urllib3 1.26.9
- ushlex 0.99.1
- vine 5.0.0
- wcwidth 0.2.5
- webencodings 0.5.1
- websocket-client 1.3.3
- Werkzeug 2.1.1
- wheel 0.37.1
- wrapt 1.14.0
- yarl 1.7.2
- zipp 3.8.0
- zope.event 4.5.0
- zope.interface 5.4.0
The detection is usually more than 1.5 s based on the logging.
2023-03-29 16:06:16.857 INFO Get frame and convert to cvtcolor in mseconds=22.968
2023-03-29 16:06:19.498 INFO od detect only in mseconds=2663.627
2023-03-29 16:06:19.599 INFO tracking only in mseconds=2764.4739999999997
2023-03-29 16:06:19.994 INFO Object dtection and tracker in mseconds=3160.455
FOUND ONE!!! 1.0
Done with image
FOUND ONE!!! 1.0
Done with image
2023-03-29 16:06:20.107 INFO Total mseconds=3272.7369999999996
In addition to this, Orin does no reach max performance.
Any ideas on how to solve this?