Trying to unmarshal NvDsBatchMeta in GOLANG, some help required

Please provide complete information as applicable to your setup.

Hardware Platform GPU
DeepStream Version 6.4
JetPack Version n.a.
TensorRT Version 8.6.1
NVIDIA GPU Driver Version (valid for GPU only) 535.104.12

Hi,

I’m having ported the deepstream-rtsp-in-rtsp-out.py sample to GOLANG utilizing the great work of the contributors of GitHub - go-gst/go-gst: Gstreamer bindings and utilities for golang.

Inference works w/o problems, but one thing remains unsolved so far: The examination/extraction of batch/frame metadata from a probe.

I’m applying the probe an nvtracker component (this is not initially part of the sample, but it doesn’t matter at all I guess).

The probe code so far looks like so:

	tracker, err := gst.NewElementWithProperties("nvtracker", trackerProperties)
	if err != nil {
		return pipeline, err
	}

	trackerSrcPad := tracker.GetStaticPad("src")
	if trackerSrcPad == nil {
		return pipeline, errors.New("cannot obtain tracker src pad")
	}

	trackerSrcPad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn {
		buffer := info.GetBuffer()
		
		mapInfo := buffer.Map(gst.MapRead)
		defer buffer.Unmap()

		samples := mapInfo.AsInt8Slice()
		if len(samples) == 0 {
			return gst.PadProbeOK
		}
		Info.Printf("buffer %d bytes\n", len(samples))
		for _, sample := range samples {
			fmt.Printf("%02X ", uint8(sample))
		}
		fmt.Println()
		return gst.PadProbeOK
	})

The Info.Printf("buffer %d bytes\n", len(samples)) gives 64 bytes with content shown below (matches in size and kind with Python):

As data:

00 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 F0 05 62 E0 83 73 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

From Python I know it is supposed to be a NvDsBatchMeta thingy, but - with the exception of max_frames_in_batch and num_frames_in_batch I cannot really see a full match with that structure. Declaration can be found at several locations on the web, not at last here includes/nvdsmeta.h · develop · PAMokhlyakov / Deepstream_Triton_LPDR · GitLab

For sure the first elements max_frames_in_batch and num_frames_in_batch are correct; I tested with a batch-size of 3 and that was reflected correcty in max_frames_in_batch. But how do the other elements match to the expected structure?

Anybody having an idea?

TIA

Anybody?

Due to the differences in the object models of python and go, they cannot be directly compared.
If you want to use the go language, you need to add bindings yourself. I added some of the bindings of DeepStream as a sample

put it to go-gst/examples/deepstream-pad-probes, then cd deepstream-pad-probes and go build.

package main

/*
#cgo pkg-config: gstreamer-1.0
#cgo CFLAGS: -I/opt/nvidia/deepstream/deepstream/sources/includes
#cgo LDFLAGS: -L/opt/nvidia/deepstream/deepstream/lib/ -lnvds_meta -lnvdsgst_meta

#include <gstnvdsmeta.h>
*/
import "C"

import (
	"errors"
	"fmt"
	"unsafe"

	"github.com/go-gst/go-glib/glib"
	"github.com/go-gst/go-gst/examples"
	"github.com/go-gst/go-gst/gst"
)

type NvBboxCoords struct {
	left float32
	top float32
	width float32
	height float32
}

type NvDsBatchMeta struct {
	ptr *C.NvDsBatchMeta
}

func wrapNvDsBatchMeta(buf *C.NvDsBatchMeta) *NvDsBatchMeta {
	return &NvDsBatchMeta{ptr: buf}
}

func (p *NvDsBatchMeta) frameMetaList() []*NvDsFrameMeta {
	glist := p.ptr.frame_meta_list
	if glist == nil {
		return nil
	}
	goList := glib.WrapList(unsafe.Pointer(glist))
	out := make([]*NvDsFrameMeta, 0)
	goList.Foreach(func(item interface{}) {
		pt := item.(unsafe.Pointer)
		var rawPoint *C.NvDsFrameMeta = (*C.NvDsFrameMeta)(unsafe.Pointer(pt))
		out = append(out, wrapNvDsFrameMeta(rawPoint))
	})
	return out
}

func wrapNvDsFrameMeta(buf *C.NvDsFrameMeta) *NvDsFrameMeta {
	return &NvDsFrameMeta{ptr: buf}
}

type NvDsFrameMeta struct {
	ptr *C.NvDsFrameMeta
}

func (p *NvDsFrameMeta) objMetaList() []*NvDsObjectMeta {
	glist := p.ptr.obj_meta_list
	if glist == nil {
		return nil
	}
	goList := glib.WrapList(unsafe.Pointer(glist))
	out := make([]*NvDsObjectMeta, 0)
	goList.Foreach(func(item interface{}) {
		pt := item.(unsafe.Pointer)
		var rawPoint *C.NvDsObjectMeta = (*C.NvDsObjectMeta)(unsafe.Pointer(pt))
		out = append(out, wrapNvDsObjectMeta(rawPoint))
	})
	return out
}

func wrapNvDsObjectMeta(buf *C.NvDsObjectMeta) *NvDsObjectMeta {
	return &NvDsObjectMeta{ptr: buf}
}

type NvDsObjectMeta struct {
	ptr *C.NvDsObjectMeta
}

func (p *NvDsObjectMeta) classId() uint32 { return uint32(p.ptr.class_id) }

func (p *NvDsObjectMeta) objBboxCoords() *NvBboxCoords {
	bbox := new(NvBboxCoords)
	bbox.left = float32(p.ptr.detector_bbox_info.org_bbox_coords.left)
	bbox.top = float32(p.ptr.detector_bbox_info.org_bbox_coords.top)
	bbox.width = float32(p.ptr.detector_bbox_info.org_bbox_coords.width)
	bbox.height = float32(p.ptr.detector_bbox_info.org_bbox_coords.height)
	return bbox
}

var frameNum int = 0

const PGIE_CLASS_ID_VEHICLE = 0
const PGIE_CLASS_ID_PERSON = 2

func padProbes(mainLoop *glib.MainLoop) error {
	gst.Init(nil)

	// Parse the pipeline we want to probe from a static in-line string.
	// Here we give our audiotestsrc a name, so we can retrieve that element
	// from the resulting pipeline.
	pipeline, err := gst.NewPipelineFromString(
		"nvstreammux name=mux batch-size=1 width=1280 height=720 ! nvinfer name=infer config-file-path=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test1/dstest1_pgie_config.yml ! nvvideoconvert ! video/x-raw(memory:NVMM), format=RGBA ! nvdsosd ! fakesink filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.h264 ! h264parse ! nvv4l2decoder ! queue ! mux.sink_0",
	)

	if err != nil {
		return err
	}

	src, err := pipeline.GetElementByName("infer")
	if err != nil {
		return err
	}
	// Get the audiotestsrc's src-pad.
	srcPad := src.GetStaticPad("src")
	if srcPad == nil {
		return errors.New("src pad on src element was nil")
	}

	// Add a probe handler on the audiotestsrc's src-pad.
	// This handler gets called for every buffer that passes the pad we probe.
	srcPad.AddProbe(gst.PadProbeTypeBuffer, func(self *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn {
		// Interpret the data sent over the pad as a buffer. We know to expect this because of
		// the probe mask defined above.
		buffer := info.GetBuffer()
		var buf *C.GstBuffer = (*C.GstBuffer)(unsafe.Pointer(buffer.Instance()))
		// NvDsBatchMeta
		batchMeta := wrapNvDsBatchMeta(C.gst_buffer_get_nvds_batch_meta(buf))
		frameMetaList := batchMeta.frameMetaList()
		if (frameMetaList == nil) {
			fmt.Println("no frame")
			return gst.PadProbePass
		}

		for _, frameMeta := range frameMetaList {
			objMetaList := frameMeta.objMetaList()
			if (objMetaList == nil) {
				fmt.Println("no obj")
				return gst.PadProbePass
			}
			var num_rects int = 0;
			var vehicle_count int = 0
			var person_count int = 0
			for _, objMeta := range objMetaList {
				if (objMeta.classId() == PGIE_CLASS_ID_VEHICLE) {
					vehicle_count++
					num_rects++;
				}
				if (objMeta.classId() == PGIE_CLASS_ID_PERSON) {
					person_count++
					num_rects++
				}
				//objMeta.objBboxCoords()
			}
			s := fmt.Sprintf("Frame Number = %d Number of objects = %d Vehicle Count = %d Person Count = %d",
							 frameNum, num_rects, vehicle_count, person_count)
			fmt.Println(s)
			frameNum++
		}
		return gst.PadProbeOK
	})

	// Start the pipeline
	pipeline.SetState(gst.StatePlaying)

	// Block on messages coming in from the bus instead of using the main loop
	for {
		msg := pipeline.GetPipelineBus().TimedPop(gst.ClockTimeNone)
		if msg == nil {
			break
		}
		if err := handleMessage(msg); err != nil {
			return err
		}
	}

	return nil
}

func handleMessage(msg *gst.Message) error {
	defer msg.Unref()
	switch msg.Type() {
	case gst.MessageEOS:
		return errors.New("end-of-stream")
	case gst.MessageError:
		return msg.ParseError()
	}
	return nil
}

func main() {
	examples.RunLoop(padProbes)
}

Cool. Thanks. Will test that out

Hmm. It’s missing a shared library libnvdsgst_meta.so

/tmp/go-build1553760086/b001/exe/main: error while loading shared libraries: libnvdsgst_meta.so: cannot open shared object file: No such file or directory
exit status 127

Wait, I missed the CGO instructions

No, unfortunately no success:

GOOS=linux GOARCH=amd64 CGO_CFLAGS=-I/opt/nvidia/deepstream/deepstream/sources/includes CGO_LDFLAGS="-L/opt/nvidia/deepstream/deepstream/lib/ -lnvds_meta -lnvdsgst_meta" go run main.go
/tmp/go-build1412282274/b001/exe/main: error while loading shared libraries: libnvdsgst_meta.so: cannot open shared object file: No such file or directory
exit status 127

This is not a code problem. You’d better check that your go-gst and deepstream are installed correctly.

In addition, the official does not provide support for go bindings. The above is just an example.

cd go-gst
./buildAll.sh 

I am using go1.22.2

go version go1.22.2 linux/amd64

Yepp, did it again. Seems to be a runtime problem. The lib is there /opt/nvidia/deepstream/deepstream/lib/libnvdsgst_meta.so, but there seems to be an issue with the path.

I can compile it, the problem arises at runtime.

go build
ubuntu@ai:~/go-gst/examples/deepstream-pad-probes$ ls
deepstream-pad-probes  main  main.go
ubuntu@ai:~/go-gst/examples/deepstream-pad-probes$ ./main 
./main: error while loading shared libraries: libnvdsgst_meta.so: cannot open shared object file: No such file or directory
ubuntu@ai:~/go-gst/examples/deepstream-pad-probes$ 

I know, this is “just for fun” :)

Added the NVIDIA libs to the library path now:

export LD_LIBRARY_PATH=/opt/nvidia/deepstream/deepstream/lib:$LD_LIBRARY_PATH

and it works :)

Frame Number = 1429 Number of objects = 23 Vehicle Count = 20 Person Count = 3
Frame Number = 1430 Number of objects = 23 Vehicle Count = 20 Person Count = 3
Frame Number = 1431 Number of objects = 24 Vehicle Count = 20 Person Count = 4
Frame Number = 1432 Number of objects = 24 Vehicle Count = 20 Person Count = 4
Frame Number = 1433 Number of objects = 24 Vehicle Count = 20 Person Count = 4

Perfect. Thank you very much, this was helpful

And it works in my code too now. Great stuff…

OK, some “cons”:

It runs - until the end of days, IF I DO NOT PROBE

If I probe, it dies sooner or later with a mutex problem (unmarshaling nvdsbatchmeta · Issue #81 · go-gst/go-gst · GitHub -

The error is always:

g_mutex_clear() called on uninitialised or locked mutex

The “unprobed” and re-streamed annotated inference results can be observed here

ffplay -fflags nobuffer -flags low_delay  rtsp://ai.votix.com:8554/test-inference-annotated

Model is yolov7-tiny, cars and persons filtered.

EDIT: Input is from file, a NY walk video

And if it comes to configure_source_for_ntp_sync: What would be the proper binding for this? TIA

This is what I’m doing inside the probe, and it crashes, sooner or later.

I’m just trying to set border, background and font color as well as descriptive text.

	trackerSrcPad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn {
		// Interpret the data sent over the pad as a buffer. We know to expect this because of
		// the probe mask defined above.
		buffer := info.GetBuffer()
		var buf *C.GstBuffer = (*C.GstBuffer)(unsafe.Pointer(buffer.Instance()))
		// NvDsBatchMeta
		batchMeta := wrapNvDsBatchMeta(C.gst_buffer_get_nvds_batch_meta(buf))
		frameMetaList := batchMeta.frameMetaList()
		if frameMetaList == nil {
			Error.Println("no frame")
			return gst.PadProbePass
		}
		for _, frameMeta := range frameMetaList {
			//timestamp := frameMeta.timestamp()
			objMetaList := frameMeta.objMetaList()
			if objMetaList == nil {
				Error.Println("no obj")
				return gst.PadProbePass
			}
			//objCounter := make([]int, len(labels))

			for _, objMeta := range objMetaList {
				
				classID := int(objMeta.classId())
				// //objCounter[classID] += 1

				objName := labels[classID]
				cDisplayText := C.CString(fmt.Sprintf("%s - %.2f%%", objName, ((objMeta.ptr.tracker_confidence+objMeta.ptr.confidence)/2)*100))
				objMeta.ptr.text_params.display_text = cDisplayText

				colorIndex := classID % len(classColors)
				color := classColors[colorIndex]
				backcolor := backColors[colorIndex]

				objMeta.ptr.text_params.font_params.font_color.red = C.double(color.red)
				objMeta.ptr.text_params.font_params.font_color.green = C.double(color.green)
				objMeta.ptr.text_params.font_params.font_color.blue = C.double(color.blue)
				objMeta.ptr.text_params.font_params.font_color.alpha = 1.0

				objMeta.ptr.text_params.text_bg_clr.red = C.double(backcolor.red)
				objMeta.ptr.text_params.text_bg_clr.green = C.double(backcolor.green)
				objMeta.ptr.text_params.text_bg_clr.blue = C.double(backcolor.blue)
				objMeta.ptr.text_params.text_bg_clr.alpha = 0.5

				objMeta.ptr.rect_params.border_color.red = C.double(color.red)
				objMeta.ptr.rect_params.border_color.green = C.double(color.green)
				objMeta.ptr.rect_params.border_color.blue = C.double(color.blue)
				objMeta.ptr.rect_params.border_color.alpha = 0.5

				// bBox := objMeta.objBboxCoords()

				// Debug.Printf("Idx: %d, TS: %d, Object: %s, L: %.0f, T: %.0f W: %.0f, H: %.0f, dc: %.2f, tc: %.2f\n",
				//  	frameMeta.ptr.pad_index,
				//  	timestamp,
				//  	objName,
				// 	bBox.left,
				// 	bBox.top,
				// 	bBox.width,
				// 	bBox.height,
				// 	objMeta.ptr.confidence,
				// 	objMeta.ptr.tracker_confidence)
				// bBox = nil
				//C.free(unsafe.Pointer(cDisplayText))
			}
		}
		return gst.PadProbeOK

You cannot modify a char array in C language as above, The following is correct

There is a difference in memory layout between arrays and pointers

func modifyObjLabel(objMeta *NvDsObjectMeta, label string) {
	// Ensure the new label size does not exceed MAX_LABEL_SIZE
	if len(label) >= C.MAX_LABEL_SIZE {
		label = label[:C.MAX_LABEL_SIZE-1]
	}

	// Convert Go string to C string
	cLabel := C.CString(label)
	defer C.free(unsafe.Pointer(cLabel))

	// Copy the new label into the obj_label field
	C.strcpy(&objMeta.ptr.obj_label[0], cLabel)
}

Thanks. I thought I also had tested to omit any change of anything and it crashed anyway.

With your suggestion it looks like so now. It survives one minute… Most likely I can also not assign any colour like so.

I think I’ll stick with Python.


func modifyObjLabel(objMeta *NvDsObjectMeta, label string) {
	// Ensure the new label size does not exceed MAX_LABEL_SIZE
	if len(label) >= C.MAX_LABEL_SIZE {
		label = label[:C.MAX_LABEL_SIZE-1]
	}

	// Convert Go string to C string
	cLabel := C.CString(label)
	defer C.free(unsafe.Pointer(cLabel))

	// Copy the new label into the obj_label field
	C.strcpy(&objMeta.ptr.obj_label[0], cLabel)
}

	trackerSrcPad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn {
		// Interpret the data sent over the pad as a buffer. We know to expect this because of
		// the probe mask defined above.
		buffer := info.GetBuffer()
		var buf *C.GstBuffer = (*C.GstBuffer)(unsafe.Pointer(buffer.Instance()))
		// NvDsBatchMeta
		batchMeta := wrapNvDsBatchMeta(C.gst_buffer_get_nvds_batch_meta(buf))
		frameMetaList := batchMeta.frameMetaList()
		if frameMetaList == nil {
			Error.Println("no frame")
			return gst.PadProbePass
		}
		for _, frameMeta := range frameMetaList {
			//timestamp := frameMeta.timestamp()
			objMetaList := frameMeta.objMetaList()
			if objMetaList == nil {
				Error.Println("no obj")
				return gst.PadProbePass
			}

			for _, objMeta := range objMetaList {

				classID := int(objMeta.classId())

				objName := labels[classID]
				displayText := fmt.Sprintf("%s - %.2f%%", objName, ((objMeta.ptr.tracker_confidence+objMeta.ptr.confidence)/2)*100)
				modifyObjLabel(objMeta, displayText)

				colorIndex := classID % len(classColors)
				color := classColors[colorIndex]
				backcolor := backColors[colorIndex]

				objMeta.ptr.text_params.font_params.font_color.red = C.double(color.red)
				objMeta.ptr.text_params.font_params.font_color.green = C.double(color.green)
				objMeta.ptr.text_params.font_params.font_color.blue = C.double(color.blue)
				objMeta.ptr.text_params.font_params.font_color.alpha = 1.0

				objMeta.ptr.text_params.text_bg_clr.red = C.double(backcolor.red)
				objMeta.ptr.text_params.text_bg_clr.green = C.double(backcolor.green)
				objMeta.ptr.text_params.text_bg_clr.blue = C.double(backcolor.blue)
				objMeta.ptr.text_params.text_bg_clr.alpha = 0.5

				objMeta.ptr.rect_params.border_color.red = C.double(color.red)
				objMeta.ptr.rect_params.border_color.green = C.double(color.green)
				objMeta.ptr.rect_params.border_color.blue = C.double(color.blue)
				objMeta.ptr.rect_params.border_color.alpha = 0.5
			}
		}
		return gst.PadProbeOK
	})

I must still be doing something wrong. Not only that it crashes, the changed caption doesn’t appear in the video:

for _, frameMeta := range frameMetaList {
			objMetaList := frameMeta.objMetaList()
			if objMetaList == nil {
				Error.Println("no obj")
				return gst.PadProbePass
			}

			for _, objMeta := range objMetaList {
				modifyObjLabel(objMeta, "test")
            }
}

I now took your original script and adjusted it to my use case by changing the pipeline:

pipeline, err := gst.NewPipelineFromString(
		"nvstreammux name=mux batch-size=1 width=1280 height=720 ! nvinfer name=infer config-file-path=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test1/dstest1_pgie_config.yml ! nvvideoconvert ! video/x-raw(memory:NVMM), format=RGBA ! nvdsosd ! fakesink nvurisrcbin uri=file:///home/ubuntu/ny.mp4 file-loop=true type=1 cudadec-memtype=0 ! queue ! mux.sink_0",
	)

The video input file is here ny.mp4

It crashes all the time, with various problems.

But it is not crashing at all, if the probing is NOT done.

The strange thing: It seems to run pretty good with an RTSP source as input…