#!/usr/bin/env python
# coding: utf-8
# # TLT DetectNet_v2 example usecase
#
# This notebook shows an example usecase of Object Detection using DetectNet_v2 in the Transfer Learning Toolkit.
#
# 0. [Set up env variables](#head-0)
# 1. [Prepare dataset and pre-trained model](#head-1)
# 1. [Verify downloaded dataset](#head-1-1)
# 1. [Prepare tfrecords from kitti format dataset](#head-1-2)
# 2. [Download pre-trained model](#head-1-3)
# 2. [Provide training specification](#head-2)
# 3. [Run TLT training](#head-3)
# 4. [Evaluate trained models](#head-4)
# 5. [Prune trained models](#head-5)
# 6. [Retrain pruned models](#head-6)
# 7. [Evaluate retrained model](#head-7)
# 8. [Visualize inferences](#head-8)
# 9. [Deploy](#head-9)
# 1. [Int8 Optimization](#head-9-1)
# 2. [Generate TensorRT engine](#head-9-2)
# 10. [Verify Deployed Model](#head-10)
# 1. [Inference using TensorRT engine](#head-10-1)
# 11. [QAT workflow](#head-11)
# 1. [Convert pruned model to QAT and retrain](#head-11-1)
# 2. [Evaluate QAT converted model](#head-11-2)
# 3. [Export QAT trained model to int8](#head-11-3)
# 4. [Evaluate a QAT trained model using the exported TensorRT engine](#head-11-4)
# 5. [Inference using QAT engine](#head-11-5)
#
# ## 0. Set up env variables
# When using the purpose-built pretrained models from NGC, please make sure to set the `$KEY` environment variable to the key as mentioned in the model overview. Failing to do so, can lead to errors when trying to load them as pretrained models.
#
# *Note: Please make sure to remove any stray artifacts/files from the `$USER_EXPERIMENT_DIR` or `$DATA_DOWNLOAD_DIR` paths as mentioned below, that may have been generated from previous experiments. Having checkpoint files etc may interfere with creating a training graph for a new experiment.*
#
# *Note: This notebook currently is by default set up to run training using 1 GPU. To use more GPU's please update the env variable `$NUM_GPUS` accordingly*
# In[ ]:
# Setting up env variables for cleaner command line commands.
get_ipython().run_line_magic('env', 'KEY=OGQ4ZGw5cXN2M3QwNDJsNGxpbnRsNXJuOHY6OTQ4ZmU2ZTAtZDcyYy00MzE0LTk1ZjEtMTgyMDJkYWFlMDgw')
get_ipython().run_line_magic('env', 'USER_EXPERIMENT_DIR=/tlt-training/detectnet_v2')
get_ipython().run_line_magic('env', 'DATA_DOWNLOAD_DIR=/tlt-training/data')
get_ipython().run_line_magic('env', 'SPECS_DIR=/tlt-training')
get_ipython().run_line_magic('env', 'NUM_GPUS=1')
# ## 1. Prepare dataset and pre-trained model
# We will be using the kitti object detection dataset for this example. To find more details, please visit http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=2d. Please download both, the left color images of the object dataset from [here](http://www.cvlibs.net/download.php?file=data_object_image_2.zip) and, the training labels for the object dataset from [here](http://www.cvlibs.net/download.php?file=data_object_label_2.zip), and place the zip files in `$DATA_DOWNLOAD_DIR`
#
# The data will then be extracted to have
# * training images in `$DATA_DOWNLOAD_DIR/training/image_2`
# * training labels in `$DATA_DOWNLOAD_DIR/training/label_2`
# * testing images in `$DATA_DOWNLOAD_DIR/testing/image_2`
#
# *Note: There are no labels for the testing images, therefore we use it just to visualize inferences for the trained model.*
# ### A. Verify downloaded dataset
# In[ ]:
# Check the dataset is present
#by me
# get_ipython().system('mkdir -p $DATA_DOWNLOAD_DIR')
# get_ipython().system("if [ ! -f $DATA_DOWNLOAD_DIR/data_object_image_2.zip ]; then echo 'Image zip file not found, please download.'; else echo 'Found Image zip file.';fi")
# get_ipython().system("if [ ! -f $DATA_DOWNLOAD_DIR/data_object_label_2.zip ]; then echo 'Label zip file not found, please download.'; else echo 'Found Labels zip file.';fi")
# In[ ]:
# unpack downloaded datasets to $DATA_DOWNLOAD_DIR.
# The training images will be under $DATA_DOWNLOAD_DIR/training/image_2 and
# labels will be under $DATA_DOWNLOAD_DIR/training/label_2.
# The testing images will be under $DATA_DOWNLOAD_DIR/testing/image_2.
# #by me
# get_ipython().system('unzip -u $DATA_DOWNLOAD_DIR/data_object_image_2.zip -d $DATA_DOWNLOAD_DIR')
# get_ipython().system('unzip -u $DATA_DOWNLOAD_DIR/data_object_label_2.zip -d $DATA_DOWNLOAD_DIR')
# In[ ]:
# verify
#by me
import os
# DATA_DIR = os.environ.get('DATA_DOWNLOAD_DIR')
# num_training_images = len(os.listdir(os.path.join(DATA_DIR, "training/image_2")))
# num_training_labels = len(os.listdir(os.path.join(DATA_DIR, "training/label_2")))
# num_testing_images = len(os.listdir(os.path.join(DATA_DIR, "testing/image_2")))
# print("Number of images in the trainval set. {}".format(num_training_images))
# print("Number of labels in the trainval set. {}".format(num_training_labels))
# print("Number of images in the test set. {}".format(num_testing_images))
# In[ ]:
# # Sample kitti label.
# get_ipython().system('cat $DATA_DOWNLOAD_DIR/training/label_2/000110.txt')
# ### B. Prepare tf records from kitti format dataset
#
# * Update the tfrecords spec file to take in your kitti format dataset
# * Create the tfrecords using the tlt-dataset-convert
#
# *Note: TfRecords only need to be generated once.*
# In[ ]:
print("TFrecords conversion spec file for kitti training")
get_ipython().system('cat $SPECS_DIR/detectnet_v2_tfrecords_kitti_trainval.txt')
# In[ ]:
# Creating a new directory for the output tfrecords dump.
print("Converting Tfrecords for kitti trainval dataset")
get_ipython().system('tlt-dataset-convert -d $SPECS_DIR/detectnet_v2_tfrecords_kitti_trainval.txt -o $DATA_DOWNLOAD_DIR/tfrecords/kitti_trainval/kitti_trainval')
# In[ ]:
get_ipython().system('ls -rlt $DATA_DOWNLOAD_DIR/tfrecords/kitti_trainval/')
# ### C. Download pre-trained model
# Download the correct pretrained model from the NGC model registry for your experiment. Please note that for DetectNet_v2, the input is expected to be 0-1 normalized with input channels in RGB order. Therefore, for optimum results please download model templates from `nvidia/tlt_pretrained_detectnet_v2`. The templates are now organizede as version strings. For example, to download a resnet18 model suitable for detectnet please resolve to the ngc object shown as `nvidia/tlt_pretrained_detectnet_v2:resnet18`.
#
# All other models expect input preprocessing with mean subtraction and input channels in BGR order. Thus, using them as pretrained weights may result in suboptimal performance.
# In[ ]:
# List models available in the model registry.
get_ipython().system('ngc registry model list nvidia/tlt_pretrained_detectnet_v2:*')
# In[ ]:
# Create the target destination to download the model.
get_ipython().system('mkdir -p $USER_EXPERIMENT_DIR/pretrained_resnet18/')
# In[ ]:
# Download the pretrained model from NGC
get_ipython().system('ngc registry model download-version nvidia/tlt_pretrained_detectnet_v2:resnet18 --dest $USER_EXPERIMENT_DIR/pretrained_resnet18')
# In[ ]:
get_ipython().system('ls -rlt $USER_EXPERIMENT_DIR/pretrained_resnet18/tlt_pretrained_detectnet_v2_vresnet18')
# ## 2. Provide training specification
# * Tfrecords for the train datasets
# * In order to use the newly generated tfrecords, update the dataset_config parameter in the spec file at `$SPECS_DIR/detectnet_v2_train_resnet18_kitti.txt`
# * Update the fold number to use for evaluation. In case of random data split, please use fold `0` only
# * For sequence-wise split, you may use any fold generated from the dataset convert tool
# * Pre-trained models
# * Augmentation parameters for on the fly data augmentation
# * Other training (hyper-)parameters such as batch size, number of epochs, learning rate etc.
# In[ ]:
get_ipython().system('cat $SPECS_DIR/detectnet_v2_train_resnet18_kitti.txt')
# ## 3. Run TLT training
# * Provide the sample spec file and the output directory location for models
#
# *Note: The training may take hours to complete. Also, the remaining notebook, assumes that the training was done in single-GPU mode. When run in multi-GPU mode, please expect to update the pruning and inference steps with new pruning thresholds and updated parameters in the clusterfile.json accordingly for optimum performance.*
#
# *Detectnet_v2 now supports restart from checkpoint. Incase, the training job is killed prematurely, you may resume training from the closest checkpoint by simply re-running the **same** command line. Please do make sure to use the **same number of GPUs** when restarting the training.*
#
# *When running the training with NUM_GPUs>1, you may need to modify the `batc_size_per_gpu` and `learning_rate` to get similar mAP as a 1GPU training run. In most cases, scaling down the batch-size by a factor of NUM_GPU's or scaling up the learning rate by a factor of NUM_GPU's would be a good place to start.*
# In[ ]:
get_ipython().system('tlt-train detectnet_v2 -e $SPECS_DIR/detectnet_v2_train_resnet18_kitti.txt -r $USER_EXPERIMENT_DIR/experiment_dir_unpruned -k $KEY -n resnet18_detector --gpus $NUM_GPUS')
# In[ ]:
print('Model for each epoch:')
print('---------------------')
get_ipython().system('ls -lh $USER_EXPERIMENT_DIR/experiment_dir_unpruned/weights')
# ## 4. Evaluate the trained model
# In[ ]:
# BY ME
# get_ipython().system('tlt-evaluate detectnet_v2 -e $SPECS_DIR/detectnet_v2_train_resnet18_kitti.txt -m $USER_EXPERIMENT_DIR/experiment_dir_unpruned/weights/resnet18_detector.tlt -k $KEY')
# ## 5. Prune the trained model
# * Specify pre-trained model
# * Equalization criterion (`Applicable for resnets and mobilenets`)
# * Threshold for pruning.
# * A key to save and load the model
# * Output directory to store the model
#
# *Usually, you just need to adjust `-pth` (threshold) for accuracy and model size trade off. Higher `pth` gives you smaller model (and thus higher inference speed) but worse accuracy. The threshold to use is depend on the dataset. A pth value `5.2e-6` is just a start point. If the retrain accuracy is good, you can increase this value to get smaller models. Otherwise, lower this value to get better accuracy.*
#
# *For some internal studies, we have noticed that a pth value of 0.01 is a good starting point for detectnet_v2 models.*
# In[ ]:
# Create an output directory if it doesn't exist.
get_ipython().system('mkdir -p $USER_EXPERIMENT_DIR/experiment_dir_pruned')
# In[ ]:
get_ipython().system('tlt-prune -pm $USER_EXPERIMENT_DIR/experiment_dir_unpruned/weights/resnet18_detector.tlt -o $USER_EXPERIMENT_DIR/experiment_dir_pruned -eq union -pth 0.0000052 -k $KEY')
# In[ ]:
get_ipython().system('ls -rlt $USER_EXPERIMENT_DIR/experiment_dir_pruned/')
# ## 6. Retrain the pruned model
# * Model needs to be re-trained to bring back accuracy after pruning
# * Specify re-training specification with pretrained weights as pruned model.
#
# *Note: For retraining, please set the `load_graph` option to `true` in the model_config to load the pruned model graph. Also, if after retraining, the model shows some decrease in mAP, it could be that the originally trained model, was pruned a little too much. Please try reducing the pruning threshold, thereby reducing the pruning ratio, and use the new model to retrain.*
#
# *Note: DetectNet_v2 now supports Quantization Aware Training, to help with optmizing the model. By default the training in the cell below doesn't run the model with QAT enabled. For information on training a model with QAT please refer to the cells under [section 11](#head-11)*
# In[ ]:
# Printing the retrain experiment file.
# Note: We have updated the experiment file to include the
# newly pruned model as a pretrained weights and, the
# load_graph option is set to true
get_ipython().system('cat $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti.txt')
# In[ ]:
# Retraining using the pruned model as pretrained weights
get_ipython().system('tlt-train detectnet_v2 -e $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti.txt -r $USER_EXPERIMENT_DIR/experiment_dir_retrain -k $KEY -n resnet18_detector_pruned --gpus $NUM_GPUS')
# In[ ]:
# Listing the newly retrained model.
get_ipython().system('ls -rlt $USER_EXPERIMENT_DIR/experiment_dir_retrain/weights')
# ## 7. Evaluate the retrained model
# This section evaluates the pruned and retrained model, using `tlt-evaluate`.
# In[ ]:
# BY ME
# get_ipython().system('tlt-evaluate detectnet_v2 -e $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti.txt -m $USER_EXPERIMENT_DIR/experiment_dir_retrain/weights/resnet18_detector_pruned.tlt -k $KEY')
# ## 8. Visualize inferences
# In this section, we run the `tlt-infer` tool to generate inferences on the trained models. To render bboxes from more classes, please edit the spec file `detectnet_v2_inference_kitti_tlt.txt` to include all the classes you would like to visualize and edit the rest of the file accordingly.
# In[ ]:
# Running inference for detection on n images
get_ipython().system('tlt-infer detectnet_v2 -e $SPECS_DIR/detectnet_v2_inference_kitti_tlt.txt -o $USER_EXPERIMENT_DIR/tlt_infer_testing -i $DATA_DOWNLOAD_DIR/training/image_2 -k $KEY')
# The `tlt-infer` tool produces two outputs.
# 1. Overlain images in `$USER_EXPERIMENT_DIR/tlt_infer_testing/images_annotated`
# 2. Frame by frame bbox labels in kitti format located in `$USER_EXPERIMENT_DIR/tlt_infer_testing/labels`
#
# *Note: To run inferences for a single image, simply replace the path to the -i flag in `tlt-infer` command with the path to the image.*
# In[ ]:
# Simple grid visualizer
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
import os
from math import ceil
valid_image_ext = ['.jpg', '.png', '.jpeg', '.ppm']
def visualize_images(image_dir, num_cols=4, num_images=10):
output_path = os.path.join(os.environ['USER_EXPERIMENT_DIR'], image_dir)
num_rows = int(ceil(float(num_images) / float(num_cols)))
f, axarr = plt.subplots(num_rows, num_cols, figsize=[80,30])
f.tight_layout()
a = [os.path.join(output_path, image) for image in os.listdir(output_path)
if os.path.splitext(image)[1].lower() in valid_image_ext]
for idx, img_path in enumerate(a[:num_images]):
col_id = idx % num_cols
row_id = idx // num_cols
img = plt.imread(img_path)
axarr[row_id, col_id].imshow(img)
# In[ ]:
# Visualizing the first 12 images.
OUTPUT_PATH = 'tlt_infer_testing/images_annotated' # relative path from $USER_EXPERIMENT_DIR.
COLS = 4 # number of columns in the visualizer grid.
IMAGES = 12 # number of images to visualize.
visualize_images(OUTPUT_PATH, num_cols=COLS, num_images=IMAGES)
# ## 9. Deploy!
# In[ ]:
get_ipython().system('mkdir -p $USER_EXPERIMENT_DIR/experiment_dir_final')
# Removing a pre-existing copy of the etlt if there has been any.
import os
output_file=os.path.join(os.environ['USER_EXPERIMENT_DIR'],
"experiment_dir_final/resnet18_detector.etlt")
if os.path.exists(output_file):
os.system("rm {}".format(output_file))
get_ipython().system('tlt-export detectnet_v2 -m $USER_EXPERIMENT_DIR/experiment_dir_retrain/weights/resnet18_detector_pruned.tlt -o $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector.etlt -k $KEY')
# In[ ]:
print('Exported model:')
print('------------')
get_ipython().system('ls -lh $USER_EXPERIMENT_DIR/experiment_dir_final')
## by me
# # ### A. Int8 Optimization
# # DetectNet_v2 model supports int8 inference mode in TRT. In order to use int8 mode, we must calibrate the model to run 8-bit inferences. This involves 2 steps
# #
# # * Generate calibration tensorfile from the training data using tlt-int8-tensorfile
# # * Use tlt-export to generate int8 calibration table.
# #
# # *Note: For this example, we generate a calibration tensorfile containing 10 batches of training data.
# # Ideally, it is best to use atleast 10-20% of the training data to calibrate the model. The more data provided during calibration, the closer int8 inferences are to fp32 inferences.*
# #
# # *Note: If the model was trained with QAT nodes available, please refrain from using the post training int8 optimization as mentioned below. Please export the model in int8 mode (using the arg `--data_type int8`) with just the path to the calibration cache file (using the argument `--cal_cache_file`)*
# # In[ ]:
# get_ipython().system('tlt-int8-tensorfile detectnet_v2 -e $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti.txt -m 10 -o $USER_EXPERIMENT_DIR/experiment_dir_final/calibration.tensor')
# # In[ ]:
# get_ipython().system('rm -rf $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector.etlt')
# get_ipython().system('rm -rf $USER_EXPERIMENT_DIR/experiment_dir_final/calibration.bin')
# get_ipython().system('tlt-export detectnet_v2 -m $USER_EXPERIMENT_DIR/experiment_dir_retrain/weights/resnet18_detector_pruned.tlt -o $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector.etlt -k $KEY --cal_data_file $USER_EXPERIMENT_DIR/experiment_dir_final/calibration.tensor --data_type int8 --batches 10 --batch_size 4 --max_batch_size 4 --engine_file $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector.trt.int8 --cal_cache_file $USER_EXPERIMENT_DIR/experiment_dir_final/calibration.bin --verbose')
# # ### B. Generate TensorRT engine
# # Verify engine generation using the `tlt-converter` utility included with the docker.
# #
# # The `tlt-converter` produces optimized tensorrt engines for the platform that it resides on. Therefore, to get maximum performance, please instantiate this docker and execute the `tlt-converter` command, with the exported `.etlt` file and calibration cache (for int8 mode) on your target device. The converter utility included in this docker only works for x86 devices, with discrete NVIDIA GPU's.
# #
# # For the jetson devices, please download the converter for jetson from the dev zone link [here](https://developer.nvidia.com/tlt-converter).
# #
# # If you choose to integrate your model into deepstream directly, you may do so by simply copying the exported `.etlt` file along with the calibration cache to the target device and updating the spec file that configures the `gst-nvinfer` element to point to this newly exported model. Usually this file is called `config_infer_primary.txt` for detection models and `config_infer_secondary_*.txt` for classification models.
# # In[ ]:
# get_ipython().system('tlt-converter $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector.etlt -k $KEY -c $USER_EXPERIMENT_DIR/experiment_dir_final/calibration.bin -o output_cov/Sigmoid,output_bbox/BiasAdd -d 3,384,1248 -i nchw -m 64 -t int8 -e $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector.trt -b 4')
# # ## 10. Verify Deployed Model
# # Verify the exported model by visualizing inferences on TensorRT.
# # In addition to running inference on a `.tlt` model in [step 8](#head-8), the `tlt-infer` tool is also capable of consuming the converted `TensorRT engine` from [step 9.B](#head-9-2).
# #
# # *If after int-8 calibration the accuracy of the int-8 inferences seem to degrade, it could be because the there wasn't enough data in the calibration tensorfile used to calibrate thee model or, the training data is not entirely representative of your test images, and the calibration maybe incorrect. Therefore, you may either regenerate the calibration tensorfile with more batches of the training data, and recalibrate the model, or calibrate the model on a few images from the test set. This may be done using `--cal_image_dir` flag in the `tlt-export` tool. For more information, please follow the instructions in the USER GUIDE.
# # ### A. Inference using TensorRT engine
# # In[ ]:
# get_ipython().system('tlt-infer detectnet_v2 -e $SPECS_DIR/detectnet_v2_inference_kitti_etlt.txt -o $USER_EXPERIMENT_DIR/etlt_infer_testing -i $DATA_DOWNLOAD_DIR/testing/image_2 -k $KEY')
# # In[ ]:
# # visualize the first 12 inferenced images.
# OUTPUT_PATH = 'etlt_infer_testing/images_annotated' # relative path from $USER_EXPERIMENT_DIR.
# COLS = 4 # number of columns in the visualizer grid.
# IMAGES = 12 # number of images to visualize.
# visualize_images(OUTPUT_PATH, num_cols=COLS, num_images=IMAGES)
# # ## 11. QAT workflow
# # This section delves into the newly enabled Quantization Aware Training feature with DetectNet_v2. The workflow defined below converts a pruned model from section [5](#head-5).
# # ### A. Convert pruned model to QAT and retrain
# # All detectnet models, unpruned and pruned models can be converted to QAT models by setting the `enable_qat` parameter in the `training_config` component of the spec file to `true`.
# # In[ ]:
# # Printing the retrain experiment file.
# # Note: We have updated the experiment file to convert the
# # pretrained model to qat mode by setting the enable_qat
# # parameter.
# get_ipython().system('cat $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti_qat.txt')
# # In[ ]:
# get_ipython().system('tlt-train detectnet_v2 -e $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti_qat.txt -r $USER_EXPERIMENT_DIR/experiment_dir_retrain_qat -k $KEY -n resnet18_detector_pruned_qat --gpus $NUM_GPUS')
# # In[ ]:
# get_ipython().system('ls -rlt $USER_EXPERIMENT_DIR/experiment_dir_retrain_qat/weights')
# # ### B. Evaluate QAT converted model
# # This section evaluates a QAT enabled pruned retrained model. The mAP of this model should be comparable to that of the pruned retrained model without QAT. However, due to quantization, it is possible sometimes to see a drop in the mAP value for certain datasets.
# # In[ ]:
# get_ipython().system('tlt-evaluate detectnet_v2 -e $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti_qat.txt -m $USER_EXPERIMENT_DIR/experiment_dir_retrain_qat/weights/resnet18_detector_pruned_qat.tlt -k $KEY -f tlt')
# # ### C. Export QAT trained model to int8
# # Export a QAT trained model to TensorRT parsable model. This command generates an .etlt file from the trained model and the serializes corresponding int8 scales as a TRT readable calibration cache file.
# # In[ ]:
# get_ipython().system('rm -rf $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector_qat.etlt')
# get_ipython().system('rm -rf $USER_EXPERIMENT_DIR/experiment_dir_final/calibration_qat.bin')
# get_ipython().system('tlt-export detectnet_v2 -m $USER_EXPERIMENT_DIR/experiment_dir_retrain_qat/weights/resnet18_detector_pruned_qat.tlt -o $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector_qat.etlt -k $KEY --data_type int8 --batch_size 64 --max_batch_size 64 --engine_file $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector_qat.trt.int8 --cal_cache_file $USER_EXPERIMENT_DIR/experiment_dir_final/calibration_qat.bin --verbose')
# # ### D. Evaluate a QAT trained model using the exported TensorRT engine
# # This section evaluates a QAT enabled pruned retrained model using the TensorRT int8 engine that was exported in [Section C](#head-11-3). Please note that there maybe a slight difference (~0.1-0.5%) in the mAP from [Section B](#head-11-2), oweing to some differences in the implementation of quantization in TensorRT.
# #
# # *Note: The TRT evaluator might be slightly slower than the TLT evaluator here, because the evaluation dataloader is pinned to the CPU to avoid any clashes between TRT and TLT instances in the GPU. Please note that this tool was not intended and has not been developed for profiling the model. It is just a means to qualitatively analyse the model.*
# #
# # *Please use native TensorRT or DeepStream for the most optimized inferences.*
# # In[ ]:
# get_ipython().system('tlt-evaluate detectnet_v2 -e $SPECS_DIR/detectnet_v2_retrain_resnet18_kitti_qat.txt -m $USER_EXPERIMENT_DIR/experiment_dir_final/resnet18_detector_qat.trt.int8 -f tensorrt')
# # ### D. Inference using QAT engine
# # Run inference and visualize detections on test images, using the exported TensorRT engine from [Section C](#head-11-3).
# # In[ ]:
# get_ipython().system('tlt-infer detectnet_v2 -e $SPECS_DIR/detectnet_v2_inference_kitti_etlt_qat.txt -o $USER_EXPERIMENT_DIR/tlt_infer_testing_qat -i $DATA_DOWNLOAD_DIR/testing/image_2 -k $KEY')
# # In[ ]:
# # visualize the first 12 inferenced images.
# OUTPUT_PATH = 'tlt_infer_testing_qat/images_annotated' # relative path from $USER_EXPERIMENT_DIR.
# COLS = 4 # number of columns in the visualizer grid.
# IMAGES = 12 # number of images to visualize.
# visualize_images(OUTPUT_PATH, num_cols=COLS, num_images=IMAGES)