Please provide the following information when requesting support.
• Hardware : RTX 2070
• Network Type (Detectnet_v2)
• TLT Version (Please run “tlt info --verbose” and share “docker_tag” here)
• Training spec file(If have, please share here)
• How to reproduce the issue ? (This is for errors. Please share the command line and the detailed log here.)
I am trying to train Detectnet v2 with the following configuration:
random_seed: 42
dataset_config {
data_sources {
tfrecords_path: "/workspace/tao-experiments/data/tfrecords/kitti_trainval/*"
image_directory_path: "/workspace/tao-experiments/data/train"
}
image_extension: "jpg"
target_class_mapping {
key: "sedan"
value: "sedan"
}
target_class_mapping {
key: "midtruck"
value: "midtruck"
}
target_class_mapping {
key: "motorbike"
value: "motorbike"
}
target_class_mapping {
key: "threewheeler"
value: "threewheeler"
}
target_class_mapping {
key: "bicycle"
value: "bicycle"
}
target_class_mapping {
key: "minibus"
value: "minibus"
}
target_class_mapping {
key: "lighttruck"
value: "lighttruck"
}
target_class_mapping {
key: "microbus"
value: "microbus"
}
target_class_mapping {
key: "bigbus"
value: "bigbus"
}
target_class_mapping {
key: "heavytruck"
value: "heavytruck"
}
target_class_mapping {
key: "utility"
value: "utility"
}
target_class_mapping {
key: "nmt"
value: "nmt"
}
target_class_mapping {
key: "person"
value: "person"
}
validation_fold: 0
}
augmentation_config {
preprocessing {
output_image_width: 1920
output_image_height: 1088
min_bbox_width: 1.0
min_bbox_height: 1.0
output_image_channel: 3
enable_auto_resize: true
}
spatial_augmentation {
hflip_probability: 0.5
zoom_min: 1.0
zoom_max: 1.0
translate_max_x: 8.0
translate_max_y: 8.0
}
color_augmentation {
hue_rotation_max: 25.0
saturation_shift_max: 0.20000000298
contrast_scale_max: 0.10000000149
contrast_center: 0.5
}
}
postprocessing_config {
target_class_config {
key: "sedan"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "midtruck"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "motorbike"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "heavytruck"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "microbus"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "bicycle"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "threewheeler"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "lighttruck"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "minibus"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "bigbus"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "utility"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "nmt"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "person"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
}
model_config {
pretrained_model_file: "/workspace/tao-experiments/detectnet_v2/pretrained_resnet18/pretrained_detectnet_v2_vresnet18/resnet18.hdf5"
num_layers: 18
use_batch_norm: true
objective_set {
bbox {
scale: 35.0
offset: 0.5
}
cov {
}
}
arch: "resnet"
}
evaluation_config {
validation_period_during_training: 2
first_validation_epoch: 5
minimum_detection_ground_truth_overlap {
key: "sedan"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "midtruck"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "motorbike"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "threewheeler"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "bicycle"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "minibus"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "lighttruck"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "microbus"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "bigbus"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "heavytruck"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "utility"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "nmt"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "person"
value: 0.5
}
evaluation_box_config {
key: "microbus"
value {
minimum_height: 40
maximum_height: 500
minimum_width: 30
maximum_width: 500
}
}
evaluation_box_config {
key: "heavytruck"
value {
minimum_height: 40
maximum_height: 500
minimum_width: 40
maximum_width: 500
}
}
evaluation_box_config {
key: "motorbike"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "midtruck"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "bicycle"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "threewheeler"
value {
minimum_height: 20
maximum_height: 500
minimum_width: 20
maximum_width: 500
}
}
evaluation_box_config {
key: "lighttruck"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "minibus"
value {
minimum_height: 40
maximum_height: 500
minimum_width: 40
maximum_width: 500
}
}
evaluation_box_config {
key: "bigbus"
value {
minimum_height: 40
maximum_height: 500
minimum_width: 40
maximum_width: 500
}
}
evaluation_box_config {
key: "sedan"
value {
minimum_height: 30
maximum_height: 500
minimum_width: 40
maximum_width: 500
}
}
evaluation_box_config {
key: "utility"
value {
minimum_height: 30
maximum_height: 500
minimum_width: 30
maximum_width: 500
}
}
evaluation_box_config {
key: "nmt"
value {
minimum_height: 20
maximum_height: 500
minimum_width: 20
maximum_width: 500
}
}
evaluation_box_config {
key: "person"
value {
minimum_height: 20
maximum_height: 500
minimum_width: 10
maximum_width: 500
}
}
average_precision_mode: INTEGRATE
}
cost_function_config {
target_classes {
name: "sedan"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "midtruck"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "motorbike"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "threewheeler"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "bicycle"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "minibus"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "lighttruck"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "microbus"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "bigbus"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "heavytruck"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "utility"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "nmt"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "person"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
enable_autoweighting: true
max_objective_weight: 0.999899983406
min_objective_weight: 9.99999974738e-05
}
training_config {
batch_size_per_gpu: 4
num_epochs: 120
learning_rate {
soft_start_annealing_schedule {
min_learning_rate: 5e-06
max_learning_rate: 5e-04
soft_start: 0.10000000149
annealing: 0.699999988079
}
}
regularizer {
type: L1
weight: 3.00000002618e-09
}
optimizer {
adam {
epsilon: 9.99999993923e-09
beta1: 0.899999976158
beta2: 0.999000012875
}
}
cost_scaling {
initial_exponent: 20.0
increment: 0.005
decrement: 1.0
}
visualizer{
enabled: true
num_images: 3
scalar_logging_frequency: 50
infrequent_logging_frequency: 5
target_class_config {
key: "sedan"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "midtruck"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "motorbike"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "threewheeler"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "bicycle"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "minibus"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "lighttruck"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "microbus"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "bigbus"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "heavytruck"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "utility"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "nmt"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "person"
value: {
coverage_threshold: 0.005
}
}
clearml_config{
project: "TAO Toolkit ClearML Demo"
task: "detectnet_v2_resnet18_clearml"
tags: "detectnet_v2"
tags: "training"
tags: "resnet18"
tags: "unpruned"
}
wandb_config{
project: "TAO Toolkit Wandb Demo"
name: "detectnet_v2_resnet18_wandb"
tags: "detectnet_v2"
tags: "training"
tags: "resnet18"
tags: "unpruned"
}
}
checkpoint_interval: 1
}
bbox_rasterizer_config {
target_class_config {
key: "sedan"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "midtruck"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "motorbike"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "threewheeler"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "bicycle"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "minibus"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "lighttruck"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "microbus"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "bigbus"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "heavytruck"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "utility"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "nmt"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "person"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
deadzone_radius: 0.400000154972
}
But the training precision is very low or zero for most of the classes
2023-04-15 08:48:07,624 [INFO] tensorflow: epoch = 4.999688360255544, learning_rate = 3.406052e-05, loss = 2.735026e-06, step = 770072 (5.578 sec)
INFO:tensorflow:epoch = 4.999753285202306, learning_rate = 3.4061362e-05, loss = 4.4776903e-06, step = 770082 (5.593 sec)
2023-04-15 08:48:13,216 [INFO] tensorflow: epoch = 4.999753285202306, learning_rate = 3.4061362e-05, loss = 4.4776903e-06, step = 770082 (5.593 sec)
INFO:tensorflow:epoch = 4.999818210149067, learning_rate = 3.4062206e-05, loss = 4.467456e-06, step = 770092 (5.577 sec)
2023-04-15 08:48:18,794 [INFO] tensorflow: epoch = 4.999818210149067, learning_rate = 3.4062206e-05, loss = 4.467456e-06, step = 770092 (5.577 sec)
2023-04-15 08:48:20,468 [INFO] modulus.hooks.sample_counter_hook: Train Samples / sec: 7.163
INFO:tensorflow:global_step/sec: 1.79057
2023-04-15 08:48:21,026 [INFO] tensorflow: global_step/sec: 1.79057
INFO:tensorflow:epoch = 4.999883135095829, learning_rate = 3.4063087e-05, loss = 3.6539363e-06, step = 770102 (5.600 sec)
2023-04-15 08:48:24,394 [INFO] tensorflow: epoch = 4.999883135095829, learning_rate = 3.4063087e-05, loss = 3.6539363e-06, step = 770102 (5.600 sec)
INFO:tensorflow:epoch = 4.999948060042591, learning_rate = 3.406393e-05, loss = 2.757089e-06, step = 770112 (5.609 sec)
2023-04-15 08:48:30,003 [INFO] tensorflow: epoch = 4.999948060042591, learning_rate = 3.406393e-05, loss = 2.757089e-06, step = 770112 (5.609 sec)
Epoch 5/120
=========================
Validation cost: 0.000006
Mean average_precision (in %): 1.5064
class name average precision (in %)
------------ --------------------------
bicycle 0
bigbus 4.82811
heavytruck 0
lighttruck 0
microbus 0
midtruck 4.05732
minibus 0
motorbike 5.69286
nmt 0
person 0
sedan 3.48447
threewheeler 1.52022
utility 0
I am trying to find out the problems with the config, but couldn’t. Most of the training image resolutions are 1920x1080. Also I would like to simplify the training config files. Is it possible to ignore the class specific configurations as all the classes are treated as same in the training dataset.