Hello, first time using TAO Toolkit, I’m trying to train a custom model for object detection on aerial imagery from a drone. I took the dataset from here in TFRecord format, and followed the instructions from the tutorial, however when I run the command tao detectnet_v2 train -k custom_model -e ~/specs.txt I get the following error:
Traceback (most recent call last):
File "</usr/local/lib/python3.6/dist-packages/iva/detectnet_v2/scripts/train.py>", line 3, in <module>
File "<frozen iva.detectnet_v2.scripts.train>", line 1022, in <module>
File "<frozen iva.detectnet_v2.scripts.train>", line 1011, in <module>
File "<decorator-gen-117>", line 2, in main
File "<frozen iva.detectnet_v2.utilities.timer>", line 46, in wrapped_fn
File "<frozen iva.detectnet_v2.scripts.train>", line 994, in main
File "<frozen iva.detectnet_v2.scripts.train>", line 853, in run_experiment
File "<frozen iva.detectnet_v2.scripts.train>", line 625, in train_gridbox
File "<frozen iva.detectnet_v2.dataloader.build_dataloader>", line 273, in build_dataloader
File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 491, in __init__
File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 548, in _construct_data_sources
File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 398, in __init__
File "<frozen iva.detectnet_v2.dataloader.drivenet_dataloader>", line 439, in _get_max_image_size
IndexError: list index (0) out of range
Only solution I found was in this thread, but when I tried uploading the dataset to CVAT it stuck at 81%. Does anyone know what else I can try?
This is my spec file:
# Sample model config for to instantiate a resnet18 model with pretrained weights and freeze blocks 0, 1
# with all shortcuts having projection layers.
model_config {
arch: "mobilenet_v2"
pretrained_model_file: ''
freeze_blocks: 0
freeze_blocks: 1
all_projections: False
num_layers: 18
use_pooling: False
use_batch_norm: True
dropout_rate: 0.0
objective_set: {
cov {}
bbox {
scale: 35.0
offset: 0.5
}
}
}
################################################################################################################################################################
# Sample rasterizer configs to instantiate a 3 class bbox rasterizer
bbox_rasterizer_config {
target_class_config {
key: "airport"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "baseball-diamond"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "basketball-court"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "bridge"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "container-crane"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "ground-track-field"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "harbor"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "helicopter"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "helipad"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "large-vehicle"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "plane"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "roundabout"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "ship"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "small-vehicle"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "soccer-ball-field"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "storage-tank"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "swimming-pool"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
target_class_config {
key: "tennis-court"
value: {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4
cov_radius_y: 0.4
bbox_min_radius: 1.0
}
}
deadzone_radius: 0.67
}
################################################################################################################################################################
postprocessing_config {
target_class_config {
key: "airport"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "baseball-diamond"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "basketball-court"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "bridge"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "container-crane"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "ground-track-field"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "harbor"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "helicopter"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "helipad"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "large-vehicle"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "plane"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "roundabout"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "ship"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "small-vehicle"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "soccer-ball-field"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "storage-tank"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "swimming-pool"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "tennis-court"
value: {
clustering_config {
coverage_threshold: 0.005
dbscan_eps: 0.15
dbscan_min_samples: 0.05
minimum_bounding_box_height: 20
}
}
}
}
################################################################################################################################################################
cost_function_config {
target_classes {
name: "airport"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "baseball-diamond"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "basketball-court"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "bridge"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "container-crane"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "ground-track-field"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "harbor"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "helicopter"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "helipad"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "large-vehicle"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "plane"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "roundabout"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "ship"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "small-vehicle"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "soccer-ball-field"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "storage-tank"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "swimming-pool"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "tennis-court"
class_weight: 1.0
coverage_foreground_weight: 0.05
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
enable_autoweighting: True
max_objective_weight: 0.9999
min_objective_weight: 0.0001
}
################################################################################################################################################################
training_config {
batch_size_per_gpu: 8
num_epochs: 120
learning_rate {
soft_start_annealing_schedule {
min_learning_rate: 5e-6
max_learning_rate: 5e-4
soft_start: 0.1
annealing: 0.7
}
}
regularizer {
type: L1
weight: 3e-9
}
optimizer {
adam {
epsilon: 1e-08
beta1: 0.9
beta2: 0.999
}
}
cost_scaling {
enabled: False
initial_exponent: 20.0
increment: 0.005
decrement: 1.0
}
visualizer {
enabled: true
num_images: 3
scalar_logging_frequency: 10
infrequent_logging_frequency: 1
target_class_config {
key: "airport"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "baseball-diamond"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "basketball-court"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "bridge"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "container-crane"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "ground-track-field"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "harbor"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "helicopter"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "helipad"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "large-vehicle"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "plane"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "roundabout"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "ship"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "small-vehicle"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "soccer-ball-field"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "storage-tank"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "swimming-pool"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "tennis-court"
value: {
coverage_threshold: 0.005
}
}
}
}
################################################################################################################################################################
# Sample augementation config for
augmentation_config {
preprocessing {
output_image_width: 640
output_image_height: 480
output_image_channel: 3
min_bbox_width: 0.0
min_bbox_height: 0.0
}
spatial_augmentation {
hflip_probability: 0.5
vflip_probability: 0.0
zoom_min: 1.0
zoom_max: 1.0
translate_max_x: 8.0
translate_max_y: 8.0
}
color_augmentation {
color_shift_stddev: 0.0
hue_rotation_max: 25.0
saturation_shift_max: 0.2
contrast_scale_max: 0.1
contrast_center: 0.5
}
}
################################################################################################################################################################
# Sample evaluation config to run evaluation in integrate mode for the given 3 class model,
# at every 10th epoch starting from the epoch 1.
evaluation_config {
average_precision_mode: INTEGRATE
validation_period_during_training: 10
first_validation_epoch: 1
minimum_detection_ground_truth_overlap {
key: "airport"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "baseball-diamond"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "basketball-court"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "bridge"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "container-crane"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "ground-trank-field"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "harbor"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "helicopter"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "helipad"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "large-vehicle"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "plane"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "roundabout"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "ship"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "small-vehicle"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "soccer-ball-field"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "storage-tank"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "swimming-pool"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "tennis-court"
value: 0.5
}
evaluation_box_config {
key: "airport"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "baseball-diamond"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "basketball-court"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "bridge"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "container-crane"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "ground-trank-field"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "harbor"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "helicopter"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "helipad"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "large-vehicle"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "plane"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "roundabout"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "ship"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "small-vehicle"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "soccer-ball-field"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "storage-tank"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "swimming-pool"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "tennis-court"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
}
################################################################################################################################################################
dataset_config {
data_sources: {
tfrecords_path: "/home/rp-hyperx/Downloads/DOTA v2.0 --easy.v7-small_object.tfrecord/train/no.tfrecord"
image_directory_path: "/home/rp-hyperx/Downloads/DOTA v2.0 --easy.v7-small_object.tfrecord/train/images/"
}
image_extension: "jpg"
target_class_mapping {
key: "airport"
value: "airport"
}
target_class_mapping {
key: "baseball-diamond"
value: "baseball-diamond"
}
target_class_mapping {
key: "basketball-court"
value: "basketball-court"
}
target_class_mapping {
key: "bridge"
value: "bridge"
}
target_class_mapping {
key: "container-crane"
value: "container-crane"
}
target_class_mapping {
key: "ground-track-field"
value: "ground-track-field"
}
target_class_mapping {
key: "harbor"
value: "harbor"
}
target_class_mapping {
key: "helicopter"
value: "helicopter"
}
target_class_mapping {
key: "helipad"
value: "helipad"
}
target_class_mapping {
key: "large-vehicle"
value: "large-vehicle"
}
target_class_mapping {
key: "plane"
value: "plane"
}
target_class_mapping {
key: "roundabout"
value: "roundabout"
}
target_class_mapping {
key: "ship"
value: "ship"
}
target_class_mapping {
key: "small-vehicle"
value: "small-vehicle"
}
target_class_mapping {
key: "soccer-ball-field"
value: "soccer-ball-field"
}
target_class_mapping {
key: "storage-tank"
value: "storage-tank"
}
target_class_mapping {
key: "swimming-pool"
value: "swimming-pool"
}
target_class_mapping {
key: "tennis-court"
value: "tennis-court"
}
validation_fold: 0
}
Thank you in advance