Hi,
I train faster-rcnn model with darknet53.hdf5 in tlt-2.0GA-py3 version, and get the error at the begin of my train.
Epoch 1/300
2020-08-06 03:48:05.807093: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2020-08-06 03:48:06.294723: I tensorflow/core/kernels/cuda_solvers.cc:159] Creating CudaSolver handles for stream 0x6756f20
2020-08-06 03:48:06.295019: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2020-08-06 03:48:06.832669: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2020-08-06 03:48:06.834126: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-08-06 03:48:11.523281: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2020-08-06 03:48:11.970626: I tensorflow/core/kernels/cuda_solvers.cc:159] Creating CudaSolver handles for stream 0x6cddab0
2020-08-06 03:48:11.970983: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2020-08-06 03:48:12.504427: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2020-08-06 03:48:12.506128: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2020-08-06 03:48:16.509319: W tensorflow/core/common_runtime/bfc_allocator.cc:239] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.73GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
2020-08-06 03:48:16.509429: W tensorflow/core/common_runtime/bfc_allocator.cc:239] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.73GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
70/559 [==>...........................] - ETA: 6:44 - loss: 207.3303 - rpn_out_class_loss: 0.2214 - rpn_out_regress_loss: 205.1718 - dense_class_td_loss: 0. 71/559 [==>...........................] - ETA: 5:59 - loss: nan - rpn_out_class_loss: 0.2705 - rpn_out_regress_loss: 457.3021 - dense_class_td_loss: 0.4129 - dense_regress_td_loss: nan - dense_class_td_acc: 0.9530 Batch 70: Invalid loss, terminating traininglass_td_acc: 0.975059 [==>...................... 71/559 [==>...........................] - ETA: 7:40 - loss: 32.1663 - rpn_out_class_loss: 0.2307 - rpn_out_regress_loss: 13.8539 - dense_class_td_loss: 0.4125 - dense_regress_td_loss: 0.1514 - dense_class_td_acc: 0.9522Batch 70: Invalid loss, terminating training_class_td_acc: 0.4883
71/559 [==>...........................] - ETA: 6:41 - loss: 221.3829 - rpn_out_class_loss: 0.2279 - rpn_out_regress_loss: 202.2821 - dense_class_td_loss: 0.3179 - dense_regress_td_loss: 1.0371 - dense_class_td_acc: 0.9570
Here is my specs, can you help me check whether has problem, thanks!
# Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
random_seed: 42
enc_key: 'YjlxOTRkaHRjYWI2Z2NxN2cwOXBlZjh1OTQ6ZTE2YjdkN********'
verbose: True
network_config {
input_image_config {
image_type: RGB
image_channel_order: 'bgr'
size_height_width {
height: 720
width: 1280
}
image_channel_mean {
key: 'b'
value: 103.939
}
image_channel_mean {
key: 'g'
value: 116.779
}
image_channel_mean {
key: 'r'
value: 123.68
}
image_scaling_factor: 1.0
max_objects_num_per_image: 100
}
feature_extractor: "darknet:53"
anchor_box_config {
scale: 64.0
scale: 128.0
scale: 256.0
ratio: 1.0
ratio: 0.5
ratio: 2.0
}
freeze_bn: True
freeze_blocks: 0
freeze_blocks: 1
roi_mini_batch: 256
rpn_stride: 16
conv_bn_share_bias: True
roi_pooling_config {
pool_size: 7
pool_size_2x: False
}
all_projections: True
use_pooling:False
enable_qat: False
}
training_config {
kitti_data_config {
data_sources: {
tfrecords_path: "/workspace/tlt-experiments/tfrecords/kitti_trainval/kitti_trainval*"
image_directory_path: "/workspace/tlt-experiments/data/training"
}
image_extension: 'jpg'
target_class_mapping {
key: 'person'
value: 'person'
}
validation_fold: 0
}
data_augmentation {
preprocessing {
output_image_width: 1280
output_image_height: 720
output_image_channel: 3
min_bbox_width: 1.0
min_bbox_height: 1.0
}
spatial_augmentation {
hflip_probability: 0.5
vflip_probability: 0.0
zoom_min: 1.0
zoom_max: 1.0
translate_max_x: 0
translate_max_y: 0
}
color_augmentation {
hue_rotation_max: 0.0
saturation_shift_max: 0.0
contrast_scale_max: 0.0
contrast_center: 0.5
}
}
enable_augmentation: True
batch_size_per_gpu: 1
num_epochs: 300
#checkpoint_interval: 5
pretrained_weights: "/workspace/tlt-experiments/darknet_53.hdf5"
#resume_from_model: "/workspace/tlt-experiments/data/faster_rcnn/resnet18.epoch2.tlt"
output_model: "/workspace/tlt-experiments/data/faster_rcnn/frcnn_kitti_darknet53.tlt"
rpn_min_overlap: 0.3
rpn_max_overlap: 0.7
classifier_min_overlap: 0.0
classifier_max_overlap: 0.5
gt_as_roi: False
std_scaling: 1.0
classifier_regr_std {
key: 'x'
value: 10.0
}
classifier_regr_std {
key: 'y'
value: 10.0
}
classifier_regr_std {
key: 'w'
value: 5.0
}
classifier_regr_std {
key: 'h'
value: 5.0
}
rpn_mini_batch: 256
rpn_pre_nms_top_N: 12000
rpn_nms_max_boxes: 2000
rpn_nms_overlap_threshold: 0.7
reg_config {
type: L2
weight: 1e-4
}
optimizer {
sgd {
lr: 0.02
momentum: 0.9
decay: 0.0
nesterov: False
}
}
lr_scheduler {
soft_start {
base_lr: 0.02
start_lr: 0.002
soft_start: 0.1
annealing_points: 0.8
annealing_points: 0.9
annealing_divider: 10.0
}
}
lambda_rpn_regr: 1.0
lambda_rpn_class: 1.0
lambda_cls_regr: 1.0
lambda_cls_class: 1.0
inference_config {
images_dir: '/workspace/tlt-experiments/data/testing/image_2'
model: '/workspace/tlt-experiments/data/faster_rcnn/frcnn_kitti_darknet53.epoch12.tlt'
batch_size: 1
detection_image_output_dir: '/workspace/tlt-experiments/data/faster_rcnn/inference_results_imgs'
labels_dump_dir: '/workspace/tlt-experiments/data/faster_rcnn/inference_dump_labels'
rpn_pre_nms_top_N: 6000
rpn_nms_max_boxes: 300
rpn_nms_overlap_threshold: 0.7
bbox_visualize_threshold: 0.6
classifier_nms_max_boxes: 300
classifier_nms_overlap_threshold: 0.3
}
evaluation_config {
model: '/workspace/tlt-experiments/data/faster_rcnn/frcnn_kitti_darknet53.epoch12.tlt'
batch_size: 1
validation_period_during_training: 5
labels_dump_dir: '/workspace/tlt-experiments/data/faster_rcnn/test_dump_labels'
rpn_pre_nms_top_N: 6000
rpn_nms_max_boxes: 300
rpn_nms_overlap_threshold: 0.7
classifier_nms_max_boxes: 300
classifier_nms_overlap_threshold: 0.3
object_confidence_thres: 0.0001
use_voc07_11point_metric:False
}
}