IndexError: index 6 is out of bounds for axis 1 with size 6 while training by using FasterRCNN.

random_seed: 42
enc_key: "API"
verbose: True
network_config {
input_image_config {
image_type: RGB
image_channel_order: 'bgr'
    size_height_width {
height: 544
width: 1024
}
    image_channel_mean {
        key: 'b'
        value: 103.939
}
    image_channel_mean {
        key: 'g'
        value: 116.779
}
    image_channel_mean {
        key: 'r'
        value: 123.68
}
    image_scaling_factor: 1.0
}
feature_extractor: "resnet:50"
anchor_box_config {
scale: 128.0
scale: 256.0
scale: 512.0
ratio: 1.0
ratio: 0.5
ratio: 2.0
}
freeze_bn: True
freeze_blocks: 1
freeze_blocks: 2
roi_mini_batch: 256
rpn_stride: 16
conv_bn_share_bias: True
roi_pooling_config {
pool_size: 7
pool_size_2x: True
}
}
training_config {
kitti_data_config {
images_dir: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/images'
labels_dir: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/labels'
}
training_data_parser: 'raw_kitti'
data_augmentation {
use_augmentation: True
spatial_augmentation {
hflip_probability: 0.5
vflip_probability: 0.0
zoom_min: 1.0
zoom_max: 1.0
translate_max_x: 0
translate_max_y: 0
}
color_augmentation {
color_shift_stddev: 0.0
hue_rotation_max: 0.0
saturation_shift_max: 0.0
contrast_scale_max: 0.0
contrast_center: 0.5
}
}
num_epochs: 50
class_mapping {
key: 'Platforms'
value: 0
}
class_mapping {
key: "Ladders"
value: 1
}
class_mapping {
key: "Stack-shells"
value: 2
}
class_mapping {
key: "Stack-tip"
value: 3
}
class_mapping {
key: "Closed-column-tip"
value: 4
}
class_mapping {
key: "Stack-diameter-change-zone"
value: 5
}
class_mapping {
key: "Flare-tip"
value: 6
}
class_mapping {
key: "background"
value: 7
}

pretrained_model: ""
pretrained_weights: "/home/samjth/NVIDIA_Transfer_Learning _Toolkit/tlt_resnet50_faster_rcnn_v1/resnet14.h5"
output_weights: "/home/samjth/NVIDIA_Transfer_Learning _Toolkit/faster_rcnn_result/frcnn_kitti.tltw"
output_model: "/home/samjth/NVIDIA_Transfer_Learning _Toolkit/faster_rcnn_result/frcnn_kitti.tlt"
rpn_min_overlap: 0.3
rpn_max_overlap: 0.7
classifier_min_overlap: 0.0
classifier_max_overlap: 0.5
gt_as_roi: False
std_scaling: 1.0
classifier_regr_std {
key: 'x'
value: 10.0
}
classifier_regr_std {
key: 'y'
value: 10.0
}
classifier_regr_std {
key: 'w'
value: 5.0
}
classifier_regr_std {
key: 'h'
value: 5.0
}

rpn_mini_batch: 256
rpn_pre_nms_top_N: 12000
rpn_nms_max_boxes: 2000
rpn_nms_overlap_threshold: 0.7
reg_config {
reg_type: 'L2'
weight_decay: 1e-4
}

optimizer {
adam {
lr: 0.00001
beta_1: 0.9
beta_2: 0.999
decay: 0.0
}
}

lr_scheduler {
step {
base_lr: 0.00001
gamma: 1.0
step_size: 30
}
}

lambda_rpn_regr: 1.0
lambda_rpn_class: 1.0
lambda_cls_regr: 1.0
lambda_cls_class: 1.0

inference_config {
images_dir: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/images'
model: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/faster_rcnn_result/frcnn_kitti.epoch14.tlt'
detection_image_output_dir: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/images'
labels_dump_dir: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/labels'
rpn_pre_nms_top_N: 6000
rpn_nms_max_boxes: 300
rpn_nms_overlap_threshold: 0.7
bbox_visualize_threshold: 0.6
classifier_nms_max_boxes: 300
classifier_nms_overlap_threshold: 0.3
}
evaluation_config {
dataset {
images_dir : '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/images'
labels_dir: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/labels'
}
data_parser: 'raw_kitti'
model: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/faster_rcnn_result/frcnn_kitti.epoch14.tlt'
labels_dump_dir: '/home/samjth/NVIDIA_Transfer_Learning _Toolkit/dataset/labels'
rpn_pre_nms_top_N: 6000
rpn_nms_max_boxes: 300
rpn_nms_overlap_threshold: 0.7
classifier_nms_max_boxes: 300
classifier_nms_overlap_threshold: 0.3
object_confidence_thres: 0.0001
use_voc07_11point_metric:True
}
}

Is there any issues with the training spec file? (eg: labels_dump_dir)
Actually i have observed that, no issues after generating the tfrecords. Extra classes are added into my label file after the training or evaluation .

Hi samjith888,
labels_dump_dir is from inference_config or evaluation_config. It is the path to the dumped labels directory when you run “tlt-infer” or “tlt-evaluate”. So, they are not your original dataset’s label text files.

More, would you please refer to the spec showing in the Jupyter notebook? There is a spec file(frcnn_kitti_spec.txt) under examples/faster_rcnn/spec/
After checking, there are several different places from yours. You can have an experiment with that spec.

Also, I would like to suggest you trigger Faster-rcnn Jupyter notebook as a start point. It download KITTI dataset and train three classes(Car,Cyclist,Pedestrian). You can cross check and verify.

More, if it is possible, you can train with multi-gpu to solve OOM issue.