I have trained a ssd mobilenet model using latest object detection api and while converting to tensorrt engine i am facing the following error
[libprotobuf FATAL /home/erisuser/p4sw/sw/gpgpu/MachineLearning/DIT/externals/protobuf/x86_64/10.0/include/google/protobuf/repeated_field.h:1408] CHECK failed: (index) < (current_size_)
After debugging a lot i found that error is occurring in grid anchor plugin
I was able to successfully create tensorrt engine from pretrained model available in https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md
Following is the config file i use
import graphsurgeon as gs
import tensorflow as tf
Input = gs.create_node("Input",
op="Placeholder",
dtype=tf.float32,
shape=[1, 3, 300, 300])
PriorBox = gs.create_plugin_node(name="GridAnchor", op="GridAnchor_TRT",
numLayers=6,
minSize=0.2,
maxSize=0.95,
aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33],
variance=[0.1,0.1,0.2,0.2],
featureMapShapes=[19, 10, 5, 3, 2, 1])
NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT",
shareLocation=1,
varianceEncodedInTarget=0,
backgroundLabelId=0,
confidenceThreshold=1e-8,
nmsThreshold=0.6,
topK=100,
keepTopK=100,
numClasses=2,
inputOrder=[0, 2, 1],
confSigmoid=1,
isNormalized=1,
scoreConverter="SIGMOID")
concat_priorbox = gs.create_node(name="concat_priorbox", op="ConcatV2", dtype=tf.float32, axis=2)
concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0)
concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0)
namespace_plugin_map = {
"MultipleGridAnchorGenerator": PriorBox,
#"Postprocessor": NMS,
"Preprocessor": Input,
# "ToFloat": Input,
# "image_tensor": Input,
"Concatenate": concat_priorbox,
"concat": concat_box_loc,
"concat_1": concat_box_conf
}
def preprocess(dynamic_graph):
# remove the unrelated or error layers
dynamic_graph.remove(dynamic_graph.find_nodes_by_path(namespace_remove), remove_exclusive_dependencies=False)
# Now create a new graph by collapsing namespaces
dynamic_graph.collapse_namespaces(namespace_plugin_map)
# Remove the outputs, so we just have a single output node (NMS).
dynamic_graph.remove(dynamic_graph.graph_outputs, remove_exclusive_dependencies=False)
# Remove the Squeeze to avoid "Assertion `isPlugin(layerName)' failed"
Squeeze = dynamic_graph.find_node_inputs_by_name(dynamic_graph.graph_outputs[0], 'Squeeze')
dynamic_graph.forward_inputs(Squeeze)
The command i used to generate uff file from tensorflow frozen graph is as follows
python3 convert_to_uff.py <model_file> -p config_ssd_mobilenet.py -O concat_priorbox
By changing the output node names i found that when i give output node name as concat_box_conf or concat_box_loc i am able to generate engine without any error.
But when i give output node name as NMS i got the above mentioned error. To find out whether input to NMS plugin node is proper or not i tried giving the output node name as concat_priorbox which resulted in the above error
when i use the same command for the pretrained model available in tensorflow repo the engine creation is success and following is the pbtxt file generated for pretrained model when the output node name is concat_priorbox
version: 1
descriptor_core_version: 1
descriptors {
id: "tensorflow_extension"
version: 1
}
descriptors {
id: "custom"
version: 1
}
graphs {
id: "main"
nodes {
id: "concat_priorbox"
inputs: "GridAnchor"
operation: "Concat"
fields {
key: "axis"
value {
i: 2
}
}
}
nodes {
[b] id: "GridAnchor"
inputs: "Const"
operation: "_GridAnchor_TRT"
fields {
key: "aspectRatios_u_flist"
value {
d_list {
val: 1.0
val: 2.0
val: 0.5
val: 3.0
val: 0.33000001311302185
}
}
}[/b]
fields {
key: "featureMapShapes_u_ilist"
value {
i_list {
val: 19
val: 10
val: 5
val: 3
val: 2
val: 1
}
}
}
fields {
key: "maxSize_u_float"
value {
d: 0.949999988079071
}
}
fields {
key: "minSize_u_float"
value {
d: 0.20000000298023224
}
}
fields {
key: "numLayers_u_int"
value {
i: 6
}
}
fields {
key: "variance_u_flist"
value {
d_list {
val: 0.10000000149011612
val: 0.10000000149011612
val: 0.20000000298023224
val: 0.20000000298023224
}
}
}
}
nodes {
id: "Const"
operation: "Const"
fields {
key: "dtype"
value {
dtype: DT_FLOAT32
}
}
fields {
key: "shape"
value {
i_list {
val: 2
}
}
}
fields {
key: "values"
value {
ref: "weights_Const"
}
}
}
nodes {
id: "MarkOutput_0"
inputs: "concat_priorbox"
operation: "MarkOutput"
}
}
referenced_data {
key: "weights_Const"
value {
blob: "
version: 1
descriptor_core_version: 1
descriptors {
id: “tensorflow_extension”
version: 1
}
descriptors {
id: “custom”
version: 1
}
graphs {
id: “main”
nodes {
id: “concat_priorbox”
inputs: “GridAnchor”
operation: “Concat”
fields {
key: “axis”
value {
i: 2
}
}
}
nodes {
id: “GridAnchor”
inputs: “Const”
operation: “_GridAnchor_TRT”
fields {
key: “aspectRatios_u_flist”
value {
d_list {
val: 1.0
val: 2.0
val: 0.5
val: 3.0
val: 0.33000001311302185
}
}
}
fields {
key: “featureMapShapes_u_ilist”
value {
i_list {
val: 19
val: 10
val: 5
val: 3
val: 2
val: 1
}
}
}
fields {
key: “maxSize_u_float”
value {
d: 0.949999988079071
}
}
fields {
key: “minSize_u_float”
value {
d: 0.20000000298023224
}
}
fields {
key: “numLayers_u_int”
value {
i: 6
}
}
fields {
key: “variance_u_flist”
value {
d_list {
val: 0.10000000149011612
val: 0.10000000149011612
val: 0.20000000298023224
val: 0.20000000298023224
}
}
}
}
nodes {
id: “Const”
operation: “Const”
fields {
key: “dtype”
value {
dtype: DT_FLOAT32
}
}
fields {
key: “shape”
value {
i_list {
val: 2
}
}
}
fields {
key: “values”
value {
ref: “weights_Const”
}
}
}
nodes {
id: “MarkOutput_0”
inputs: “concat_priorbox”
operation: “MarkOutput”
}
}
referenced_data {
key: “weights_Const”
value {
blob: “\000\000\200?\000\000\200?”
}
}
00
version: 1
descriptor_core_version: 1
descriptors {
id: “tensorflow_extension”
version: 1
}
descriptors {
id: “custom”
version: 1
}
graphs {
id: “main”
nodes {
id: “concat_priorbox”
inputs: “GridAnchor”
operation: “Concat”
fields {
key: “axis”
value {
i: 2
}
}
}
nodes {
id: “GridAnchor”
inputs: “Const”
operation: “_GridAnchor_TRT”
fields {
key: “aspectRatios_u_flist”
value {
d_list {
val: 1.0
val: 2.0
val: 0.5
val: 3.0
val: 0.33000001311302185
}
}
}
fields {
key: “featureMapShapes_u_ilist”
value {
i_list {
val: 19
val: 10
val: 5
val: 3
val: 2
val: 1
}
}
}
fields {
key: “maxSize_u_float”
value {
d: 0.949999988079071
}
}
fields {
key: “minSize_u_float”
value {
d: 0.20000000298023224
}
}
fields {
key: “numLayers_u_int”
value {
i: 6
}
}
fields {
key: “variance_u_flist”
value {
d_list {
val: 0.10000000149011612
val: 0.10000000149011612
val: 0.20000000298023224
val: 0.20000000298023224
}
}
}
}
nodes {
id: “Const”
operation: “Const”
fields {
key: “dtype”
value {
dtype: DT_FLOAT32
}
}
fields {
key: “shape”
value {
i_list {
val: 2
}
}
}
fields {
key: “values”
value {
ref: “weights_Const”
}
}
}
nodes {
id: “MarkOutput_0”
inputs: “concat_priorbox”
operation: “MarkOutput”
}
}
referenced_data {
key: “weights_Const”
value {
blob: “\000\000\200?\000\000\200?”
}
}
0000?
version: 1
descriptor_core_version: 1
descriptors {
id: “tensorflow_extension”
version: 1
}
descriptors {
id: “custom”
version: 1
}
graphs {
id: “main”
nodes {
id: “concat_priorbox”
inputs: “GridAnchor”
operation: “Concat”
fields {
key: “axis”
value {
i: 2
}
}
}
nodes {
id: “GridAnchor”
inputs: “Const”
operation: “_GridAnchor_TRT”
fields {
key: “aspectRatios_u_flist”
value {
d_list {
val: 1.0
val: 2.0
val: 0.5
val: 3.0
val: 0.33000001311302185
}
}
}
fields {
key: “featureMapShapes_u_ilist”
value {
i_list {
val: 19
val: 10
val: 5
val: 3
val: 2
val: 1
}
}
}
fields {
key: “maxSize_u_float”
value {
d: 0.949999988079071
}
}
fields {
key: “minSize_u_float”
value {
d: 0.20000000298023224
}
}
fields {
key: “numLayers_u_int”
value {
i: 6
}
}
fields {
key: “variance_u_flist”
value {
d_list {
val: 0.10000000149011612
val: 0.10000000149011612
val: 0.20000000298023224
val: 0.20000000298023224
}
}
}
}
nodes {
id: “Const”
operation: “Const”
fields {
key: “dtype”
value {
dtype: DT_FLOAT32
}
}
fields {
key: “shape”
value {
i_list {
val: 2
}
}
}
fields {
key: “values”
value {
ref: “weights_Const”
}
}
}
nodes {
id: “MarkOutput_0”
inputs: “concat_priorbox”
operation: “MarkOutput”
}
}
referenced_data {
key: “weights_Const”
value {
blob: “\000\000\200?\000\000\200?”
}
}
00
version: 1
descriptor_core_version: 1
descriptors {
id: “tensorflow_extension”
version: 1
}
descriptors {
id: “custom”
version: 1
}
graphs {
id: “main”
nodes {
id: “concat_priorbox”
inputs: “GridAnchor”
operation: “Concat”
fields {
key: “axis”
value {
i: 2
}
}
}
nodes {
id: “GridAnchor”
inputs: “Const”
operation: “_GridAnchor_TRT”
fields {
key: “aspectRatios_u_flist”
value {
d_list {
val: 1.0
val: 2.0
val: 0.5
val: 3.0
val: 0.33000001311302185
}
}
}
fields {
key: “featureMapShapes_u_ilist”
value {
i_list {
val: 19
val: 10
val: 5
val: 3
val: 2
val: 1
}
}
}
fields {
key: “maxSize_u_float”
value {
d: 0.949999988079071
}
}
fields {
key: “minSize_u_float”
value {
d: 0.20000000298023224
}
}
fields {
key: “numLayers_u_int”
value {
i: 6
}
}
fields {
key: “variance_u_flist”
value {
d_list {
val: 0.10000000149011612
val: 0.10000000149011612
val: 0.20000000298023224
val: 0.20000000298023224
}
}
}
}
nodes {
id: “Const”
operation: “Const”
fields {
key: “dtype”
value {
dtype: DT_FLOAT32
}
}
fields {
key: “shape”
value {
i_list {
val: 2
}
}
}
fields {
key: “values”
value {
ref: “weights_Const”
}
}
}
nodes {
id: “MarkOutput_0”
inputs: “concat_priorbox”
operation: “MarkOutput”
}
}
referenced_data {
key: “weights_Const”
value {
blob: “\000\000\200?\000\000\200?”
}
}
0000?"
}
}
Following is the pbtxt generated for the model i have trained using latest object detection api with output node name as concat_priorbox
version: 1
descriptor_core_version: 1
descriptors {
id: "tensorflow_extension"
version: 1
}
descriptors {
id: "custom"
version: 1
}
graphs {
id: "main"
nodes {
id: "concat_priorbox"
inputs: "GridAnchor"
operation: "Concat"
fields {
key: "axis"
value {
i: 2
}
}
}
[b]nodes {
id: "GridAnchor"
operation: "_GridAnchor_TRT"
fields {
key: "aspectRatios_u_flist"
value {
d_list {
val: 1.0
val: 2.0
val: 0.5
val: 3.0
val: 0.33000001311302185
}
}
}[/b]
fields {
key: "featureMapShapes_u_ilist"
value {
i_list {
val: 19
val: 10
val: 5
val: 3
val: 2
val: 1
}
}
}
fields {
key: "maxSize_u_float"
value {
d: 0.949999988079071
}
}
fields {
key: "minSize_u_float"
value {
d: 0.20000000298023224
}
}
fields {
key: "numLayers_u_int"
value {
i: 6
}
}
fields {
key: "variance_u_flist"
value {
d_list {
val: 0.10000000149011612
val: 0.10000000149011612
val: 0.20000000298023224
val: 0.20000000298023224
}
}
}
}
nodes {
id: "MarkOutput_0"
inputs: "concat_priorbox"
operation: "MarkOutput"
}
}
From both the pbtxt the difference is in the pretrained model i have 4 nodes in the graph whereas in my model i have 3 nodes.
In the pbtxt generated for my model for the node GridAnchor Inputs is missing.
If somebody have experienced the similar issue or any other suggestions to resolve this issue as it is critical for my project
Pretrained model is trained using tensorflow 1.12.0 and my model is trained using tensorflow 1.13.1