Slow detection with Jetson Orin and Yolov7

Hi all!! I’m using a Jetson Orin developer kit, a custom implementation of Yolov7 to detect objects and DeepSort for tracking.

The code I’m using is the following:

import logging
from datetime import datetime
from pathlib import Path
import random
from typing import List


import numpy as np

from deep_sort.deep_sort import Deep
from enums.camera.RunningCameraStatus import RunningCameraStatus
from models.CameraRunningContext import CameraRunningContext
from models.measurements.CameraSingleMeasurementValue import CameraSingleMeasurementValue
from object_detection import ObjectDetection
from services.detectors.CameraDetectorService import CameraDetectorService

import cv2

class Yolo7CameraDetectorService(CameraDetectorService):

    def __init__(self):
        super().__init__()

        #Define a dictionary that
        # Key : integer
        # Value : timestamp of last updated(exit camera timestamp)
        self.__detected_items = dict()

    """
    Detect items using yolo7

    Store items that remains in the current frame in the detected items dictionary
    Remove all the other items and write them to camera measurements buffer
    """

    def detect(self, camera: CameraRunningContext):

        #Call abstract method
        super().detect(camera)

        # Counting area ROI
        yellow_poly_pts = np.array([[0, 150], [1440, 150], [1440, 850], [0, 850]])

        #Define codec for video
        #fourcc = cv2.VideoWriter_fourcc(*'XVID')

        # Load Object Detection

        # ATTENTION
        # Object detection applies toLower() on path
        path_waights = Path(camera.camera.detection_weights_path)

        od = ObjectDetection(path_waights)
        od.load_detection_model(image_size=1088, confThreshold=0.5, nmsThreshold=0.4)
        od.load_class_names(camera.camera.detection_classes_names)

        # Load Object Tracking Deep Sort
        deep = Deep(max_distance=0.8,
                    nms_max_overlap=0.3,
                    n_init=3,
                    max_age=5,
                    max_iou_distance=0.7)
        
        tracker = deep.sort_tracker()

        # While running camera status is enabled and streaming and detecting
        while camera.status is RunningCameraStatus.ENABLED_AND_STREAMING_AND_DETECTING :
            try:
                
                now = datetime.utcnow()

                # Get current raw image frame
                image = camera.get_cur_raw_image()

                a=5
                # Convert to RGB image
                rgb = image.convert("RGB")

                # Convert np array
                array = rgb.get_numpy_array()

                # opencv loads the format bgr so the frame needs to be converted to rgb to process
                array = cv2.cvtColor(array, cv2.COLOR_BGR2RGB)

                #np.save("/home/christina/Desktop/test.np",array)

                logging.info("Get frame and convert to cvtcolor in mseconds=" + str( (datetime.utcnow() - now).total_seconds() * 1000 ))

                """ 1. Object Detection """
                (class_ids, scores, boxes) = od.detect(array)
                logging.info(
                    "od detect only in mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))

                """ 2. Object Tracking """
                features = deep.encoder(array, boxes)
                detections = deep.Detection(boxes, scores, class_ids, features)

                logging.info(
                    "tracking only in mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))

                # Draw yellow polygon for the ROI
                cv2.polylines(array, [yellow_poly_pts], True, (0, 255, 255), 2)

                tracker.predict()
                (class_ids, object_ids, boxes) = tracker.update(detections)

                logging.info("Object dtection and tracker in mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))

                # Iterate tracker results
                for class_id, object_id, box in zip(class_ids, object_ids, boxes):

                    (x, y, x2, y2) = box
                    class_name = od.names[class_id]
                    color = od.colors[class_id]

                    # Center doubleroll detected
                    cx = int((x + x2) / 2)
                    cy = int((y + y2) / 2)

                    cv2.rectangle(array, (x, y), (x2, y2), (0, 0, 255), 2)
                    cv2.rectangle(array, (x, y), (x + len(class_name) * 20, y - 30), (0, 0, 255), -1)
                    cv2.putText(array, class_name + " " + str(object_id), (x, y - 10), 0, 0.75, (255, 255, 255), 2)
                    cv2.circle(array, (cx, cy), 4, (0, 0, 255), -1)

                    """3. Object Counting"""

                    # Check if object is in the ROI
                    result = cv2.pointPolygonTest(yellow_poly_pts, (int(cx), int(cy)), False)
                    #print(result, object_id, datetime.utcnow())


                    if result > 0:
                        # Store item to dictionary as camera single measurement value
                        self.__detected_items[object_id] = CameraSingleMeasurementValue(value=1, timestamp= datetime.utcnow())
                        print("FOUND ONE!!!", result)
                        #cv2.imwrite("/home/christina/Desktop/yv7run/Img005.png", array)
                        print("Done with image")

                #imS = cv2.resize(array, (1080, 1080))
                #cv2.imshow("Frame", imS)

                #Outside iteration
                #Remove all items that does not exist in the current object ids list
                # The current frame detected items must be added before to the dictionary
                camera_measurement_values = self.__remove_and_get_all_except(object_ids)

                #Add values to buffers
                self.camera_buffered_measurements_service.add_multi(camera.camera.crguid, camera_measurement_values)

                logging.info(
                    "Total mseconds=" + str((datetime.utcnow() - now).total_seconds() * 1000))
            except Exception as ex:
                logging.error("Failed to process frame for camera with crguid=" + str(camera.camera.crguid))
                logging.error(ex)


    """
    Remove and retutn all items ids from 
    """
    def __remove_and_get_all_except(self, object_ids) -> List[CameraSingleMeasurementValue]:
        
        # Get all dictionary keys that does not exist in the current frame detected objects
        keys = [key for key in self.__detected_items.keys() if key not in object_ids]

        # Get values for the above keys
        values = [self.__detected_items.get(key) for key in keys]

        # Remove above keys form dictionary
        for key in keys: del self.__detected_items[key]

        # Dictionary contains only detected items that are not exited camera and are still trackable
        # Return items that exits camera context
        return values

The images are coming directly from a machine vision camera and the packages with their versions at the virtual environment I’m working on are the following:

  • absl-py 1.4.0
  • aiohttp 3.8.1
  • aiosignal 1.2.0
  • amqp 5.1.1
  • asgiref 3.5.1
  • asttokens 2.2.1
  • astunparse 1.6.3
  • async-timeout 4.0.2
  • asyncio 3.4.3
  • attrs 21.4.0
  • autopep8 1.6.0
  • awscli 1.25.21
  • backcall 0.2.0
  • billiard 3.6.4.0
  • bleach 5.0.0
  • botocore 1.27.21
  • cachetools 5.3.0
  • certifi 2021.10.8
  • cffi 1.15.0
  • chardet 3.0.4
  • charset-normalizer 2.0.12
  • click 8.1.2
  • click-didyoumean 0.3.0
  • click-plugins 1.1.1
  • click-repl 0.2.0
  • colorama 0.4.4
  • commonmark 0.9.1
  • cryptography 40.0.1
  • cycler 0.11.0
  • decorator 5.1.1
  • Deprecated 1.2.13
  • docutils 0.16
  • executing 1.2.0
  • Flask 2.1.2
  • flatbuffers 23.3.3
  • fonttools 4.33.3
  • frozenlist 1.3.0
  • gast 0.4.0
  • gevent 21.12.0
  • google-auth 2.17.0
  • google-auth-oauthlib 0.4.6
  • google-pasta 0.2.0
  • greenlet 1.1.2
  • grpcio 1.53.0
  • h11 0.8.1
  • h2 3.2.0
  • h5py 3.8.0
  • hpack 3.0.0
  • http3 0.6.7
  • hyperframe 5.2.0
  • idna 2.10
  • importlib-metadata 4.11.3
  • imutils 0.5.4
  • ipython 8.11.0
  • itsdangerous 2.1.2
  • jedi 0.18.2
  • jeepney 0.8.0
  • Jinja2 3.1.1
  • jmespath 1.0.1
  • keras 2.11.0
  • keyring 23.6.0
  • kiwisolver 1.4.3
  • kombu 5.2.4
  • libclang 16.0.0
  • Markdown 3.4.3
  • MarkupSafe 2.1.1
  • matplot 0.1.9
  • matplotlib 3.5.2
  • matplotlib-inline 0.1.6
  • mpmath 1.3.0
  • multidict 6.0.2
  • networkx 3.0
  • numpy 1.23.5
  • oauthlib 3.2.2
  • opencv-python 4.7.0.72
  • opt-einsum 3.3.0
  • packaging 21.3
  • pandas 1.5.3
  • parso 0.8.3
  • pexpect 4.8.0
  • pickleshare 0.7.5
  • pika 1.2.0
  • Pillow 9.1.1
  • pip 23.0.1
  • pkginfo 1.8.3
  • prompt-toolkit 3.0.38
  • protobuf 3.19.5
  • psutil 5.9.4
  • ptyprocess 0.7.0
  • pure-eval 0.2.2
  • pyasn1 0.4.8
  • pyasn1-modules 0.2.8
  • pycodestyle 2.8.0
  • pycparser 2.21
  • Pygments 2.12.0
  • pyloco 0.0.139
  • pyparsing 3.0.8
  • python-dateutil 2.8.2
  • pytz 2022.1
  • pywin32-ctypes 0.2.0
  • PyYAML 5.4.1
  • readme-renderer 35.0
  • redis 4.2.2
  • requests 2.27.1
  • requests-oauthlib 1.3.1
  • requests-toolbelt 0.9.1
  • rfc3986 1.5.0
  • rich 12.4.4
  • rsa 4.7.2
  • s3transfer 0.6.0
  • schedule 1.1.0
  • scipy 1.8.0
  • seaborn 0.12.2
  • SecretStorage 3.3.3
  • setuptools 61.0.0
  • SimpleWebSocketServer 0.1.1
  • six 1.16.0
  • stack-data 0.6.2
  • sympy 1.11.1
  • tensorboard 2.11.2
  • tensorboard-data-server 0.6.1
  • tensorboard-plugin-wit 1.8.1
  • tensorflow 2.11.0
  • tensorflow-cpu-aws 2.11.0
  • tensorflow-estimator 2.11.0
  • tensorflow-io-gcs-filesystem 0.32.0
  • termcolor 2.2.0
  • thop 0.1.1.post2209072238
  • tk 0.1.0
  • toml 0.10.2
  • torch 1.14.0a0+44dac51c.nv23.2
  • torchvision 0.14.1a0+5e8e2f1
  • tqdm 4.64.0
  • traitlets 5.9.0
  • twine 4.0.1
  • typing 3.7.4.3
  • typing_extensions 4.5.0
  • urllib3 1.26.9
  • ushlex 0.99.1
  • vine 5.0.0
  • wcwidth 0.2.5
  • webencodings 0.5.1
  • websocket-client 1.3.3
  • Werkzeug 2.1.1
  • wheel 0.37.1
  • wrapt 1.14.0
  • yarl 1.7.2
  • zipp 3.8.0
  • zope.event 4.5.0
  • zope.interface 5.4.0

The detection is usually more than 1.5 s based on the logging.

2023-03-29 16:06:16.857 INFO     Get frame and convert to cvtcolor in mseconds=22.968
2023-03-29 16:06:19.498 INFO     od detect only in mseconds=2663.627
2023-03-29 16:06:19.599 INFO     tracking only in mseconds=2764.4739999999997
2023-03-29 16:06:19.994 INFO     Object dtection and tracker in mseconds=3160.455
FOUND ONE!!! 1.0
Done with image
FOUND ONE!!! 1.0
Done with image
2023-03-29 16:06:20.107 INFO     Total mseconds=3272.7369999999996

In addition to this, Orin does no reach max performance.

Any ideas on how to solve this?

Hi,

It only uses 5% GPU for inference.
For better optimized, it’s recommended to convert the model into TensorRT.

Thanks.

Could you please recommend a way to convert the existing model to TensorRT?

Hi,

Is your model trained with TensorFlow Object Detection API?
If yes, please check below for a tutorial to convert it into TensorRT:

Thanks.

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.