Please provide the following information when requesting support.
• Hardware RTX 2060
• Network Type (Detectnet_v2)
• TLT Version (4.0.0)
I am trying to train Detectnet v2 with the following specs file:
random_seed: 42
dataset_config {
data_sources {
tfrecords_path: "/workspace/tao-experiments/data/tfrecords/kitti_trainval/*"
image_directory_path: "/workspace/tao-experiments/data/train"
}
image_extension: "jpg"
target_class_mapping {
key: "sedan"
value: "sedan"
}
target_class_mapping {
key: "midtruck"
value: "midtruck"
}
target_class_mapping {
key: "motorbike"
value: "motorbike"
}
target_class_mapping {
key: "threewheeler"
value: "threewheeler"
}
target_class_mapping {
key: "bicycle"
value: "bicycle"
}
target_class_mapping {
key: "minibus"
value: "minibus"
}
target_class_mapping {
key: "lighttruck"
value: "lighttruck"
}
target_class_mapping {
key: "microbus"
value: "microbus"
}
target_class_mapping {
key: "bigbus"
value: "bigbus"
}
target_class_mapping {
key: "heavytruck"
value: "heavytruck"
}
target_class_mapping {
key: "utility"
value: "utility"
}
target_class_mapping {
key: "nmt"
value: "nmt"
}
target_class_mapping {
key: "person"
value: "person"
}
validation_fold: 0
}
augmentation_config {
preprocessing {
output_image_width: 1248
output_image_height: 384
min_bbox_width: 1.0
min_bbox_height: 1.0
output_image_channel: 3
}
spatial_augmentation {
hflip_probability: 0.5
zoom_min: 1.0
zoom_max: 1.0
translate_max_x: 8.0
translate_max_y: 8.0
}
color_augmentation {
hue_rotation_max: 25.0
saturation_shift_max: 0.20000000298
contrast_scale_max: 0.10000000149
contrast_center: 0.5
}
}
postprocessing_config {
target_class_config {
key: "sedan"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "midtruck"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "motorbike"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "heavytruck"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "microbus"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "bicycle"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "threewheeler"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "lighttruck"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "minibus"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "bigbus"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "utility"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "nmt"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
target_class_config {
key: "person"
value {
clustering_config {
clustering_algorithm: DBSCAN
dbscan_confidence_threshold: 0.9
coverage_threshold: 0.00499999988824
dbscan_eps: 0.20000000298
dbscan_min_samples: 0.0500000007451
minimum_bounding_box_height: 20
}
}
}
}
model_config {
pretrained_model_file: "/workspace/tao-experiments/detectnet_v2/pretrained_resnet18/pretrained_detectnet_v2_vresnet18/resnet18.hdf5"
num_layers: 18
use_batch_norm: true
objective_set {
bbox {
scale: 35.0
offset: 0.5
}
cov {
}
}
arch: "resnet"
}
evaluation_config {
validation_period_during_training: 10
first_validation_epoch: 5
minimum_detection_ground_truth_overlap {
key: "sedan"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "midtruck"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "motorbike"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "threewheeler"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "bicycle"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "minibus"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "lighttruck"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "microbus"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "bigbus"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "heavytruck"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "utility"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "nmt"
value: 0.5
}
minimum_detection_ground_truth_overlap {
key: "person"
value: 0.5
}
evaluation_box_config {
key: "microbus"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "heavytruck"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "motorbike"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "midtruck"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "bicycle"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "threewheeler"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "lighttruck"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "minibus"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "bigbus"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "sedan"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "utility"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "nmt"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
evaluation_box_config {
key: "person"
value {
minimum_height: 20
maximum_height: 9999
minimum_width: 10
maximum_width: 9999
}
}
average_precision_mode: INTEGRATE
}
cost_function_config {
target_classes {
name: "sedan"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "midtruck"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "motorbike"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "threewheeler"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "bicycle"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "minibus"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "lighttruck"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "microbus"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "bigbus"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "heavytruck"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "utility"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "nmt"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
target_classes {
name: "person"
class_weight: 1.0
coverage_foreground_weight: 0.0500000007451
objectives {
name: "cov"
initial_weight: 1.0
weight_target: 1.0
}
objectives {
name: "bbox"
initial_weight: 10.0
weight_target: 10.0
}
}
enable_autoweighting: true
max_objective_weight: 0.999899983406
min_objective_weight: 9.99999974738e-05
}
training_config {
batch_size_per_gpu: 8
num_epochs: 120
learning_rate {
soft_start_annealing_schedule {
min_learning_rate: 5e-06
max_learning_rate: 5e-04
soft_start: 0.10000000149
annealing: 0.699999988079
}
}
regularizer {
type: L1
weight: 3.00000002618e-09
}
optimizer {
adam {
epsilon: 9.99999993923e-09
beta1: 0.899999976158
beta2: 0.999000012875
}
}
cost_scaling {
initial_exponent: 20.0
increment: 0.005
decrement: 1.0
}
visualizer{
enabled: true
num_images: 3
scalar_logging_frequency: 50
infrequent_logging_frequency: 5
target_class_config {
key: "sedan"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "midtruck"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "motorbike"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "threewheeler"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "bicycle"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "minibus"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "lighttruck"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "microbus"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "bigbus"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "heavytruck"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "utility"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "nmt"
value: {
coverage_threshold: 0.005
}
}
target_class_config {
key: "person"
value: {
coverage_threshold: 0.005
}
}
clearml_config{
project: "TAO Toolkit ClearML Demo"
task: "detectnet_v2_resnet18_clearml"
tags: "detectnet_v2"
tags: "training"
tags: "resnet18"
tags: "unpruned"
}
wandb_config{
project: "TAO Toolkit Wandb Demo"
name: "detectnet_v2_resnet18_wandb"
tags: "detectnet_v2"
tags: "training"
tags: "resnet18"
tags: "unpruned"
}
}
checkpoint_interval: 1
}
bbox_rasterizer_config {
target_class_config {
key: "sedan"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "midtruck"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "motorbike"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "threewheeler"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "bicycle"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "minibus"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "lighttruck"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "microbus"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "bigbus"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "heavytruck"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "utility"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "nmt"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
target_class_config {
key: "person"
value {
cov_center_x: 0.5
cov_center_y: 0.5
cov_radius_x: 1.0
cov_radius_y: 1.0
bbox_min_radius: 1.0
}
}
deadzone_radius: 0.400000154972
}
But after first epoch, it returns the error:
INFO:tensorflow:epoch = 0.9673596746485292, learning_rate = 7.247637e-06, loss = 0.00015355347, step = 82776 (5.518 sec)
2023-03-02 17:39:51,154 [INFO] tensorflow: epoch = 0.9673596746485292, learning_rate = 7.247637e-06, loss = 0.00015355347, step = 82776 (5.518 sec)
INFO:tensorflow:epoch = 0.9675934041533732, learning_rate = 7.2482867e-06, loss = 0.0001721511, step = 82796 (5.522 sec)
2023-03-02 17:39:56,676 [INFO] tensorflow: epoch = 0.9675934041533732, learning_rate = 7.2482867e-06, loss = 0.0001721511, step = 82796 (5.522 sec)
2023-03-02 17:39:57,510 [INFO] modulus.hooks.sample_counter_hook: Train Samples / sec: 28.931
INFO:tensorflow:epoch = 0.9678271336582173, learning_rate = 7.2489365e-06, loss = 0.00016435228, step = 82816 (5.530 sec)
2023-03-02 17:40:02,206 [INFO] tensorflow: epoch = 0.9678271336582173, learning_rate = 7.2489365e-06, loss = 0.00016435228, step = 82816 (5.530 sec)
2023-03-02 17:40:04,421 [INFO] modulus.hooks.sample_counter_hook: Train Samples / sec: 28.943
INFO:tensorflow:epoch = 0.9680608631630614, learning_rate = 7.2495864e-06, loss = 0.00016470911, step = 82836 (5.530 sec)
2023-03-02 17:40:07,736 [INFO] tensorflow: epoch = 0.9680608631630614, learning_rate = 7.2495864e-06, loss = 0.00016470911, step = 82836 (5.530 sec)
2023-03-02 17:40:11,334 [INFO] modulus.hooks.sample_counter_hook: Train Samples / sec: 28.934
Input file read error
Input file read error
INFO:tensorflow:epoch = 0.9682945926679054, learning_rate = 7.250236e-06, loss = 0.00014266044, step = 82856 (5.533 sec)
2023-03-02 17:40:13,269 [INFO] tensorflow: epoch = 0.9682945926679054, learning_rate = 7.250236e-06, loss = 0.00014266044, step = 82856 (5.533 sec)
2023-03-02 17:40:15,236 [INFO] root: Saving trained model.
2023-03-02 17:40:16,195 [INFO] root: Model saved.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py", line 1365, in _do_call
return fn(*args)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py", line 1350, in _run_fn
target_list, run_metadata)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py", line 1443, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: {{function_node __inference_Dataset_map__map_func_set_random_wrapper_5585}} Invalid JPEG data or crop window, data size 786432
[[{{node AssetLoader/DecodeJpeg}}]]
[[data_loader_out]]
(1) Invalid argument: {{function_node __inference_Dataset_map__map_func_set_random_wrapper_5585}} Invalid JPEG data or crop window, data size 786432
[[{{node AssetLoader/DecodeJpeg}}]]
[[data_loader_out]]
[[NotEqual_1/_5117]]
0 successful operations.
0 derived errors ignored.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "</usr/local/lib/python3.6/dist-packages/iva/detectnet_v2/scripts/train.py>", line 3, in <module>
File "<frozen iva.detectnet_v2.scripts.train>", line 1022, in <module>
File "<frozen iva.detectnet_v2.scripts.train>", line 1011, in <module>
File "<decorator-gen-117>", line 2, in main
File "<frozen iva.detectnet_v2.utilities.timer>", line 46, in wrapped_fn
File "<frozen iva.detectnet_v2.scripts.train>", line 994, in main
File "<frozen iva.detectnet_v2.scripts.train>", line 853, in run_experiment
File "<frozen iva.detectnet_v2.scripts.train>", line 728, in train_gridbox
File "<frozen iva.detectnet_v2.scripts.train>", line 200, in run_training_loop
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/monitored_session.py", line 754, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/monitored_session.py", line 1360, in run
raise six.reraise(*original_exc_info)
File "/usr/local/lib/python3.6/dist-packages/six.py", line 696, in reraise
raise value
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/monitored_session.py", line 1345, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/monitored_session.py", line 1418, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/monitored_session.py", line 1176, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py", line 956, in run
run_metadata_ptr)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py", line 1359, in _do_run
run_metadata)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: Invalid JPEG data or crop window, data size 786432
[[{{node AssetLoader/DecodeJpeg}}]]
[[data_loader_out]]
(1) Invalid argument: Invalid JPEG data or crop window, data size 786432
[[{{node AssetLoader/DecodeJpeg}}]]
[[data_loader_out]]
[[NotEqual_1/_5117]]
0 successful operations.
0 derived errors ignored.
Telemetry data couldn't be sent, but the command ran successfully.
[WARNING]: <urlopen error [Errno -2] Name or service not known>
Execution status: FAIL
2023-03-02 23:40:39,885 [INFO] tlt.components.docker_handler.docker_handler: Stopping container.
print('Model for each epoch:')
print('---------------------')
I have prepared the TFrecords with the following specs:
kitti_config {
root_directory_path: "/workspace/tao-experiments/data/train"
image_dir_name: "images"
label_dir_name: "labels"
image_extension: ".jpg"
partition_mode: "random"
num_partitions: 2
val_split: 14
num_shards: 10
}
image_directory_path: "/workspace/tao-experiments/data/train"
I have manually checked the images and labels but found no corrupt images. Is there any way to check TFrecords data integrity?
My label files are formatted in KITTI annotation format like the following:
bicycle 0.0 0 0.0 1335.178955078125 596.6967163085938 1361.0630187988281 636.173641204834 0.0 0.0 0.0 0.0 0.0 0.0 0.0
Please help me resolve the issue.