Things I have confirmed from the previous issues
image extensions: jpg
image height,width: multiple of 16
batch size : low
update in tfrecords
Command run :
tao detectnet_v2 train -k 7221 -r /home/usr/train/task2_detectNet_train.cfg
error :
2022-07-05 11:38:27,519 [INFO] root: Sampling mode of the dataloader was set to user_defined.
Traceback (most recent call last):
File "/root/.cache/bazel/_bazel_root/b770f990bb7b9e2db5771981fb3a38b4/execroot/ai_infra/bazel-out/k8-fastbuild/bin/magnet/packages/iva/build_wheel.runfiles/ai_infra/iva/detectnet_v2/scripts/train.py", line 917, in <module>
File "/root/.cache/bazel/_bazel_root/b770f990bb7b9e2db5771981fb3a38b4/execroot/ai_infra/bazel-out/k8-fastbuild/bin/magnet/packages/iva/build_wheel.runfiles/ai_infra/iva/detectnet_v2/scripts/train.py", line 906, in <module>
File "<decorator-gen-2>", line 2, in main
File "/root/.cache/bazel/_bazel_root/b770f990bb7b9e2db5771981fb3a38b4/execroot/ai_infra/bazel-out/k8-fastbuild/bin/magnet/packages/iva/build_wheel.runfiles/ai_infra/iva/detectnet_v2/utilities/timer.py", line 46, in wrapped_fn
File "/root/.cache/bazel/_bazel_root/b770f990bb7b9e2db5771981fb3a38b4/execroot/ai_infra/bazel-out/k8-fastbuild/bin/magnet/packages/iva/build_wheel.runfiles/ai_infra/iva/detectnet_v2/scripts/train.py", line 893, in main
File "/root/.cache/bazel/_bazel_root/b770f990bb7b9e2db5771981fb3a38b4/execroot/ai_infra/bazel-out/k8-fastbuild/bin/magnet/packages/iva/build_wheel.runfiles/ai_infra/iva/detectnet_v2/scripts/train.py", line 757, in run_experiment
File "/root/.cache/bazel/_bazel_root/b770f990bb7b9e2db5771981fb3a38b4/execroot/ai_infra/bazel-out/k8-fastbuild/bin/magnet/packages/iva/build_wheel.runfiles/ai_infra/iva/detectnet_v2/scripts/train.py", line 638, in train_gridbox
File "/root/.cache/bazel/_bazel_root/b770f990bb7b9e2db5771981fb3a38b4/execroot/ai_infra/bazel-out/k8-fastbuild/bin/magnet/packages/iva/build_wheel.runfiles/ai_infra/iva/detectnet_v2/cost_function/cost_auto_weight_hook.py", line 26, in build_cost_auto_weight_hook
ValueError: steps_per_epoch must be > 0
2022-07-05 11:38:29,408 [INFO] tlt.components.docker_handler.docker_handler: Stopping container.
Spec file :
dataset_config {
data_sources {
tfrecords_path: "~/dataset/tf/-fold*"
image_directory_path: "~/dataset/data/train/images"
}
image_extension: "jpg"
target_class_mapping {
key: "bicycle"
value: "bicycle"
}
target_class_mapping {
key: "pedestrian"
value: "pedestrian"
}
validation_fold: 0
}
augmentation_config {
preprocessing {
output_image_width: 1088
output_image_height: 544
min_bbox_width: 1.0
min_bbox_height: 1.0
output_image_channel: 3
}
spatial_augmentation {
hflip_probability: 0.5
zoom_min: 1.0
zoom_max: 1.0
translate_max_x: 8.0
translate_max_y: 8.0
}
color_augmentation {
hue_rotation_max: 25.0
saturation_shift_max: 0.20000000298023224
contrast_scale_max: 0.10000000149011612
contrast_center: 0.5
}
}
postprocessing_config {
target_class_config {
key: "bicycle"
value {
clustering_config {
coverage_threshold: 0.004999999888241291
minimum_bounding_box_height: 20
dbscan_eps: 0.15000000596046448
dbscan_min_samples: 0.05000000074505806
}
}
}
target_class_config {
key: "pedestrian"
value {
clustering_config {
coverage_threshold: 0.004999999888241291
minimum_bounding_box_height: 20
dbscan_eps: 0.15000000596046448
dbscan_min_samples: 0.05000000074505806
}
}
}
}
model_config {
num_layers: 18
use_batch_norm: true
objective_set {
bbox {
scale: 35.0
offset: 0.5
}
cov {
}
}
arch: "resnet"
load_graph: true
all_projections: true
}
evaluation_config {
validation_period_during_training: 20
first_validation_epoch: 5
minimum_detection_ground_truth_overlap {
key: "bicycle"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "pedestrian"
value: 0.699999988079071
}
evaluation_box_config {
key: "bicycle"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
evaluation_box_config {
key: "pedestrian"
value {
minimum_height: 4
maximum_height: 9999
minimum_width: 4
maximum_width: 9999
}
}
average_precision_mode: INTEGRATE
}
cost_function_config {
target_classes {
name: "pedestrian"
class_weight: 1.0
coverage_foreground_weight: 0.05000000074505806
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "bicycle"
class_weight: 1.0
coverage_foreground_weight: 0.05000000074505806
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 1.0
}
}
enable_autoweighting: true
max_objective_weight: 0.9998999834060669
min_objective_weight: 9.999999747378752e-05
}
training_config {
batch_size_per_gpu: 4
num_epochs: 20
learning_rate {
soft_start_annealing_schedule {
min_learning_rate: 4.999999873689376e-06
max_learning_rate: 0.0005000000237487257
soft_start: 0.10000000149011612
annealing: 0.699999988079071
}
}
regularizer {
weight: 3.000000026176508e-09
}
optimizer {
adam {
epsilon: 9.99999993922529e-09
beta1: 0.8999999761581421
beta2: 0.9990000128746033
}
}
cost_scaling {
initial_exponent: 20.0
increment: 0.005
decrement: 1.0
}
}
bbox_rasterizer_config {
target_class_config {
key: "bicycle"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4000000059604645
cov_radius_y: 0.4000000059604645
bbox_min_radius: 1.0
}
}
target_class_config {
key: "pedestrian"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 0.4000000059604645
cov_radius_y: 0.4000000059604645
bbox_min_radius: 1.0
}
}
deadzone_radius: 0.6700000166893005
}