I will try using the gst-nvinfer dump plugin. I already tried saving the buffer post inference to a jpg file in both deepstream 6.1 and 6.2 and compared them with a frame diff and found they are the exact same. This is the RGBA image extracted post nvinfer
My belief is that TensorRT is affecting detection. Is that a possibility from your end? Again, the difference on the same video + model is better extreme across different deepstream versions. For example
DS 6.1.1 = 18188 detections
DS 6.1 = 18181
DS 6.2 = 16237 detections
Also, .engine file sizes are different although they had the same pgie config
DS 6.1 = 99.4 mb
DS 6.1.1 = 94.6 mb
DS 6.2 = 94.4 mb
this is the model architecture
Layer Input Shape Output Shape WeightPtr
(0) conv_silu [3, 544, 960] [64, 272, 480] 7168
(1) conv_silu [64, 272, 480] [128, 136, 240] 81408
(2) conv_silu [128, 136, 240] [64, 136, 240] 89856
(3) route: 1 - [128, 136, 240] -
(4) conv_silu [128, 136, 240] [64, 136, 240] 98304
(5) conv_silu [64, 136, 240] [64, 136, 240] 102656
(6) conv_silu [64, 136, 240] [64, 136, 240] 139776
(7) shortcut_add_linear: 4 [64, 136, 240] [64, 136, 240] -
(8) conv_silu [64, 136, 240] [64, 136, 240] 144128
(9) conv_silu [64, 136, 240] [64, 136, 240] 181248
(10) shortcut_add_linear: 7 [64, 136, 240] [64, 136, 240] -
(11) conv_silu [64, 136, 240] [64, 136, 240] 185600
(12) conv_silu [64, 136, 240] [64, 136, 240] 222720
(13) shortcut_add_linear: 10 [64, 136, 240] [64, 136, 240] -
(14) route: 13, 2 - [128, 136, 240] -
(15) conv_silu [128, 136, 240] [128, 136, 240] 239616
(16) conv_silu [128, 136, 240] [256, 68, 120] 535552
(17) conv_silu [256, 68, 120] [128, 68, 120] 568832
(18) route: 16 - [256, 68, 120] -
(19) conv_silu [256, 68, 120] [128, 68, 120] 602112
(20) conv_silu [128, 68, 120] [128, 68, 120] 619008
(21) conv_silu [128, 68, 120] [128, 68, 120] 766976
(22) shortcut_add_linear: 19 [128, 68, 120] [128, 68, 120] -
(23) conv_silu [128, 68, 120] [128, 68, 120] 783872
(24) conv_silu [128, 68, 120] [128, 68, 120] 931840
(25) shortcut_add_linear: 22 [128, 68, 120] [128, 68, 120] -
(26) conv_silu [128, 68, 120] [128, 68, 120] 948736
(27) conv_silu [128, 68, 120] [128, 68, 120] 1096704
(28) shortcut_add_linear: 25 [128, 68, 120] [128, 68, 120] -
(29) conv_silu [128, 68, 120] [128, 68, 120] 1113600
(30) conv_silu [128, 68, 120] [128, 68, 120] 1261568
(31) shortcut_add_linear: 28 [128, 68, 120] [128, 68, 120] -
(32) conv_silu [128, 68, 120] [128, 68, 120] 1278464
(33) conv_silu [128, 68, 120] [128, 68, 120] 1426432
(34) shortcut_add_linear: 31 [128, 68, 120] [128, 68, 120] -
(35) conv_silu [128, 68, 120] [128, 68, 120] 1443328
(36) conv_silu [128, 68, 120] [128, 68, 120] 1591296
(37) shortcut_add_linear: 34 [128, 68, 120] [128, 68, 120] -
(38) route: 37, 17 - [256, 68, 120] -
(39) conv_silu [256, 68, 120] [256, 68, 120] 1657856
(40) conv_silu [256, 68, 120] [512, 34, 60] 2839552
(41) conv_silu [512, 34, 60] [256, 34, 60] 2971648
(42) route: 40 - [512, 34, 60] -
(43) conv_silu [512, 34, 60] [256, 34, 60] 3103744
(44) conv_silu [256, 34, 60] [256, 34, 60] 3170304
(45) conv_silu [256, 34, 60] [256, 34, 60] 3761152
(46) shortcut_add_linear: 43 [256, 34, 60] [256, 34, 60] -
(47) conv_silu [256, 34, 60] [256, 34, 60] 3827712
(48) conv_silu [256, 34, 60] [256, 34, 60] 4418560
(49) shortcut_add_linear: 46 [256, 34, 60] [256, 34, 60] -
(50) conv_silu [256, 34, 60] [256, 34, 60] 4485120
(51) conv_silu [256, 34, 60] [256, 34, 60] 5075968
(52) shortcut_add_linear: 49 [256, 34, 60] [256, 34, 60] -
(53) conv_silu [256, 34, 60] [256, 34, 60] 5142528
(54) conv_silu [256, 34, 60] [256, 34, 60] 5733376
(55) shortcut_add_linear: 52 [256, 34, 60] [256, 34, 60] -
(56) conv_silu [256, 34, 60] [256, 34, 60] 5799936
(57) conv_silu [256, 34, 60] [256, 34, 60] 6390784
(58) shortcut_add_linear: 55 [256, 34, 60] [256, 34, 60] -
(59) conv_silu [256, 34, 60] [256, 34, 60] 6457344
(60) conv_silu [256, 34, 60] [256, 34, 60] 7048192
(61) shortcut_add_linear: 58 [256, 34, 60] [256, 34, 60] -
(62) conv_silu [256, 34, 60] [256, 34, 60] 7114752
(63) conv_silu [256, 34, 60] [256, 34, 60] 7705600
(64) shortcut_add_linear: 61 [256, 34, 60] [256, 34, 60] -
(65) conv_silu [256, 34, 60] [256, 34, 60] 7772160
(66) conv_silu [256, 34, 60] [256, 34, 60] 8363008
(67) shortcut_add_linear: 64 [256, 34, 60] [256, 34, 60] -
(68) conv_silu [256, 34, 60] [256, 34, 60] 8429568
(69) conv_silu [256, 34, 60] [256, 34, 60] 9020416
(70) shortcut_add_linear: 67 [256, 34, 60] [256, 34, 60] -
(71) route: 70, 41 - [512, 34, 60] -
(72) conv_silu [512, 34, 60] [512, 34, 60] 9284608
(73) conv_silu [512, 34, 60] [1024, 17, 30] 14007296
(74) conv_silu [1024, 17, 30] [512, 17, 30] 14533632
(75) route: 73 - [1024, 17, 30] -
(76) conv_silu [1024, 17, 30] [512, 17, 30] 15059968
(77) conv_silu [512, 17, 30] [512, 17, 30] 15324160
(78) conv_silu [512, 17, 30] [512, 17, 30] 17685504
(79) shortcut_add_linear: 76 [512, 17, 30] [512, 17, 30] -
(80) conv_silu [512, 17, 30] [512, 17, 30] 17949696
(81) conv_silu [512, 17, 30] [512, 17, 30] 20311040
(82) shortcut_add_linear: 79 [512, 17, 30] [512, 17, 30] -
(83) conv_silu [512, 17, 30] [512, 17, 30] 20575232
(84) conv_silu [512, 17, 30] [512, 17, 30] 22936576
(85) shortcut_add_linear: 82 [512, 17, 30] [512, 17, 30] -
(86) route: 85, 74 - [1024, 17, 30] -
(87) conv_silu [1024, 17, 30] [1024, 17, 30] 23989248
(88) conv_silu [1024, 17, 30] [512, 17, 30] 24515584
(89) maxpool [512, 17, 30] [512, 17, 30] -
(90) maxpool [512, 17, 30] [512, 17, 30] -
(91) maxpool [512, 17, 30] [512, 17, 30] -
(92) route: 88, 89, 90, 91 - [2048, 17, 30] -
(93) conv_silu [2048, 17, 30] [1024, 17, 30] 26616832
(94) conv_silu [1024, 17, 30] [512, 17, 30] 27143168
(95) upsample [512, 17, 30] [512, 34, 60] -
(96) route: 95, 72 - [1024, 34, 60] -
(97) conv_silu [1024, 34, 60] [256, 34, 60] 27406336
(98) route: 96 - [1024, 34, 60] -
(99) conv_silu [1024, 34, 60] [256, 34, 60] 27669504
(100) conv_silu [256, 34, 60] [256, 34, 60] 27736064
(101) conv_silu [256, 34, 60] [256, 34, 60] 28326912
(102) conv_silu [256, 34, 60] [256, 34, 60] 28393472
(103) conv_silu [256, 34, 60] [256, 34, 60] 28984320
(104) conv_silu [256, 34, 60] [256, 34, 60] 29050880
(105) conv_silu [256, 34, 60] [256, 34, 60] 29641728
(106) route: 105, 97 - [512, 34, 60] -
(107) conv_silu [512, 34, 60] [512, 34, 60] 29905920
(108) conv_silu [512, 34, 60] [256, 34, 60] 30038016
(109) upsample [256, 34, 60] [256, 68, 120] -
(110) route: 109, 39 - [512, 68, 120] -
(111) conv_silu [512, 68, 120] [128, 68, 120] 30104064
(112) route: 110 - [512, 68, 120] -
(113) conv_silu [512, 68, 120] [128, 68, 120] 30170112
(114) conv_silu [128, 68, 120] [128, 68, 120] 30187008
(115) conv_silu [128, 68, 120] [128, 68, 120] 30334976
(116) conv_silu [128, 68, 120] [128, 68, 120] 30351872
(117) conv_silu [128, 68, 120] [128, 68, 120] 30499840
(118) conv_silu [128, 68, 120] [128, 68, 120] 30516736
(119) conv_silu [128, 68, 120] [128, 68, 120] 30664704
(120) route: 119, 111 - [256, 68, 120] -
(121) conv_silu [256, 68, 120] [256, 68, 120] 30731264
(122) conv_silu [256, 68, 120] [256, 34, 60] 31322112
(123) route: 122, 108 - [512, 34, 60] -
(124) conv_silu [512, 34, 60] [256, 34, 60] 31454208
(125) route: 123 - [512, 34, 60] -
(126) conv_silu [512, 34, 60] [256, 34, 60] 31586304
(127) conv_silu [256, 34, 60] [256, 34, 60] 31652864
(128) conv_silu [256, 34, 60] [256, 34, 60] 32243712
(129) conv_silu [256, 34, 60] [256, 34, 60] 32310272
(130) conv_silu [256, 34, 60] [256, 34, 60] 32901120
(131) conv_silu [256, 34, 60] [256, 34, 60] 32967680
(132) conv_silu [256, 34, 60] [256, 34, 60] 33558528
(133) route: 132, 124 - [512, 34, 60] -
(134) conv_silu [512, 34, 60] [512, 34, 60] 33822720
(135) conv_silu [512, 34, 60] [512, 17, 30] 36184064
(136) route: 135, 94 - [1024, 17, 30] -
(137) conv_silu [1024, 17, 30] [512, 17, 30] 36710400
(138) route: 136 - [1024, 17, 30] -
(139) conv_silu [1024, 17, 30] [512, 17, 30] 37236736
(140) conv_silu [512, 17, 30] [512, 17, 30] 37500928
(141) conv_silu [512, 17, 30] [512, 17, 30] 39862272
(142) conv_silu [512, 17, 30] [512, 17, 30] 40126464
(143) conv_silu [512, 17, 30] [512, 17, 30] 42487808
(144) conv_silu [512, 17, 30] [512, 17, 30] 42752000
(145) conv_silu [512, 17, 30] [512, 17, 30] 45113344
(146) route: 145, 137 - [1024, 17, 30] -
(147) conv_silu [1024, 17, 30] [1024, 17, 30] 46166016
(148) route: 121 - [256, 68, 120] -
(149) conv_logistic [256, 68, 120] [21, 68, 120] 46171413
(150) yolo [21, 68, 120] - -
(151) route: 134 - [512, 34, 60] -
(152) conv_logistic [512, 34, 60] [21, 34, 60] 46182186
(153) yolo [21, 34, 60] - -
(154) route: 147 - [1024, 17, 30] -
(155) conv_logistic [1024, 17, 30] [21, 17, 30] 46203711
(156) yolo [21, 17, 30] - -
DS 6.1 Config
[property]
gpu-id = 0
model-color-format = 0
labelfile-path = /labels.txt
uff-input-blob-name = input_image
process-mode = 1
num-detected-classes = 2
interval = 0
batch-size = 1
gie-unique-id = 1
is-classifier = 0
maintain-aspect-ratio = 1
network-mode = 2
workspace-size = 9000
net-scale-factor = .0039215697906911373
cluster-mode = 2
offsets = 0;0;0
force-implicit-batch-dim = 1
infer-dims = 3;544;960
custom-network-config = /best_ap.cfg
model-file=/best_ap.wts
model-engine-file = /fp16.engine
parse-bbox-func-name = NvDsInferParseYolo
custom-lib-path = /opt/nvidia/deepstream/deepstream-6.1/sources/nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so
engine-create-func-name = NvDsInferYoloCudaEngineGet
[class-attrs-0]
#class = P
post-cluster-threshold = 0.83
[class-attrs-1]
#class = R
post-cluster-threshold = 0.85
DS 6.1.1 Config
[property]
gpu-id = 0
model-color-format = 0
labelfile-path = /labels.txt
uff-input-blob-name = input_image
process-mode = 1
num-detected-classes = 2
interval = 0
batch-size = 1
gie-unique-id = 1
is-classifier = 0
maintain-aspect-ratio = 1
network-mode = 2
workspace-size = 9000
net-scale-factor = .0039215697906911373
cluster-mode = 2
offsets = 0;0;0
force-implicit-batch-dim = 1
infer-dims = 3;544;960
custom-network-config = /best_ap.cfg
model-file=/best_ap.wts
model-engine-file = /fp16.engine
parse-bbox-func-name = NvDsInferParseYolo
custom-lib-path = /opt/nvidia/deepstream/deepstream-6.1/sources/nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so
engine-create-func-name = NvDsInferYoloCudaEngineGet
[class-attrs-0]
post-cluster-threshold = 0.83
[class-attrs-1]
post-cluster-threshold = 0.85
[property]
gpu-id = 0
model-color-format = 0
labelfile-path = /labels.txt
uff-input-blob-name = input_image
process-mode = 1
num-detected-classes = 2
interval = 0
batch-size = 1
gie-unique-id = 1
is-classifier = 0
maintain-aspect-ratio = 1
network-mode = 2
workspace-size = 9000
net-scale-factor = .0039215697906911373
cluster-mode = 2
offsets = 0;0;0
force-implicit-batch-dim = 1
infer-dims = 3;544;960
custom-network-config = /best_ap.cfg
model-engine-file = /fp16.engine
parse-bbox-func-name = NvDsInferParseYolo
model-file=/best_ap.wts
custom-lib-path = /opt/nvidia/deepstream/deepstream-6.2/sources/nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so
engine-create-func-name = NvDsInferYoloCudaEngineGet
[class-attrs-0]
post-cluster-threshold = 0.87
[class-attrs-1]
post-cluster-threshold = 0.9
each deepstream is running in its own docker container separate to the other