VPI - DCF Tracker error

Autonomous Machines Jetson & Embedded Systems Jetson Orin NX
1

Hello!

I’m trying to use DCF tracker in combination with an object detector for object tracking.
I always receive an error (VPIStatus 2, i.e. VPI_ERROR_INVALID_ARGUMENT) during the Update and Localize stage as soon as I have at least one tracked object.
I have checked that all the inputs are correct according to what is mentioned in the docs, i.e.

 * @retval #VPI_ERROR_INVALID_ARGUMENT     \p stream is NULL.
 * @retval #VPI_ERROR_INVALID_ARGUMENT     \p inPatches or trackedObjects are NULL.
 * @retval #VPI_ERROR_INVALID_ARGUMENT     \p payload not created by vpiCreateDCFTracker.
 * @retval #VPI_ERROR_INVALID_ARGUMENT     \p inPatches dimensions outside valid range.
 * @retval #VPI_ERROR_INVALID_ARGUMENT     Unsupported array type in \p trackedObjects.
 * @retval #VPI_ERROR_INVALID_ARGUMENT     Parameter(s) in \p params outside valid range.

however, the inputs seem to be normal.

Here is the snippet of my code:
initialization:

VPI_CHECK(vpiCreateCropScaler(VPI_BACKEND_CUDA, 1, max_det, &cropPayload));

VPIDCFTrackerCreationParams dcfTrackerCreateParams;
VPI_CHECK(vpiInitDCFTrackerCreationParams(&dcfTrackerCreateParams));

VPI_CHECK(vpiCreateDCFTracker(VPI_BACKEND_CUDA, 1, max_det, &dcfTrackerCreateParams, &dcfPayload));
VPI_CHECK(vpiInitDCFTrackerParams(&dcfTrackerParams));
VPI_CHECK(vpiArrayCreate(max_det, VPI_ARRAY_TYPE_DCF_TRACKED_BOUNDING_BOX, VPI_BACKEND_CUDA|VPI_BACKEND_CPU, &trackedObjects));
VPI_CHECK(vpiArrayCreate(max_det, VPI_ARRAY_TYPE_F32, VPI_BACKEND_CUDA|VPI_BACKEND_CPU, &trackedObjectsCorrelationResponses));
patchH = patchW = dcfTrackerCreateParams.featurePatchSize * dcfTrackerCreateParams.hogCellSize;
VPI_CHECK(vpiImageCreate(patchW,
							 patchH * max_det,
							 VPI_IMAGE_FORMAT_RGBA8,
							 VPI_BACKEND_CUDA,
							 &objPatches));

loop:

...
...
VPIImage vpiImg; // U8
VPI_CHECK(vpiImageCreateWrapperOpenCVMat(cvImg, VPI_BACKEND_CUDA, &vpiImg));

VPIImage vpiImgRGBA8; // RGBA8
VPI_CHECK(vpiImageCreate(cvImg.cols, cvImg.rows, VPI_IMAGE_FORMAT_RGBA8, VPI_BACKEND_CUDA, &vpiImgRGBA8));
VPI_CHECK(vpiSubmitConvertImageFormat(vpiStream, VPI_BACKEND_CUDA, vpiImg, vpiImgRGBA8, nullptr));

...
...

// Track
// predict and visualy localize
int nTrackedObjects;
VPI_CHECK(vpiArrayGetSize(trackedObjects, &nTrackedObjects));
if (nTrackedObjects > 0)
{
	VPI_CHECK(vpiSubmitCropScalerBatch(vpiStream, VPI_BACKEND_CUDA, cropPayload,
													 &vpiImgRGBA8, 1, trackedObjects,
													 patchW, patchH, objPatches));

	VPI_CHECK(vpiSubmitDCFTrackerLocalizeBatch(vpiStream, VPI_BACKEND_CUDA, dcfPayload, nullptr, 1,
	                                 nullptr, objPatches, trackedObjects, trackedObjects, nullptr,
	                                 trackedObjectsCorrelationResponses, &dcfTrackerParams));

	VPI_CHECK(vpiStreamSync(vpiStream));
}


// update objects
if (nTrackedObjects > 0 or numDets[0] > 0)
{
	VPIArrayData trackedObjectsData;
	VPI_CHECK(vpiArrayLockData(trackedObjects, VPI_LOCK_READ_WRITE, VPI_ARRAY_BUFFER_HOST_AOS, &trackedObjectsData));

...
...
	 
	// add new objects (not associated)
	VPI_CHECK(vpiArrayGetSize(trackedObjects, &nTrackedObjects));
	VPI_CHECK(vpiArraySetSize(trackedObjects, nTrackedObjects + nNewObjects));
	for (int i = 0; i < nNewObjects; ++i)
	{
		int idxBbox = newObjects[i];
		int idxObject = nTrackedObjects + i;
		VPIAxisAlignedBoundingBoxF32 newObjectBbox{(float)bboxes[0][idxBbox].x, (float)bboxes[0][idxBbox].y, (float)bboxes[0][idxBbox].width, (float)bboxes[0][idxBbox].height};
		VPIDCFTrackedBoundingBox newObject{newObjectBbox, VPI_TRACKING_STATE_NEW, 0, filterLR, filterChannelWeightsLR, new trackedObjectInfo{0}};
		((VPIDCFTrackedBoundingBox*)trackedObjectsData.buffer.aos.data)[idxObject] = newObject;
	}

	VPI_CHECK(vpiArrayUnlock(trackedObjects));
}

// update tracker
VPI_CHECK(vpiArrayGetSize(trackedObjects, &nTrackedObjects));
if (nTrackedObjects > 0)
{
	// Extract from the current frame the image patches of each object given its refined bounding box.
	VPI_CHECK(vpiSubmitCropScalerBatch(vpiStream, VPI_BACKEND_CUDA, cropPayload,
	                         &vpiImgRGBA8, 1, trackedObjects,
	                         patchW, patchH, objPatches));

	// Update tracking information
	VPI_CHECK(vpiSubmitDCFTrackerUpdateBatch(vpiStream, VPI_BACKEND_CUDA, dcfPayload,
																					 nullptr, 1,
																					 nullptr, nullptr,
																					 objPatches, trackedObjects, &dcfTrackerParams));

	VPI_CHECK(vpiStreamSync(vpiStream));

	VPIArrayData trackedObjectsData;
	VPI_CHECK(vpiArrayLockData(trackedObjects, VPI_LOCK_READ_WRITE, VPI_ARRAY_BUFFER_HOST_AOS, &trackedObjectsData));

	for (int i = 0; i < nTrackedObjects; ++i) {
		if (((VPIDCFTrackedBoundingBox *) trackedObjectsData.buffer.aos.data)[i].state != VPI_TRACKING_STATE_LOST)
			((VPIDCFTrackedBoundingBox *) trackedObjectsData.buffer.aos.data)[i].state = VPI_TRACKING_STATE_TRACKED;
	}

	VPI_CHECK(vpiArrayUnlock(trackedObjects));
}
3

Hi,

Could you share where the VPI_ERROR_INVALID_ARGUMENT triggers?
To check it further, could you share a reproducible sample for us to check?

Thanks.

4

Hi!

The error is triggered by vpiSubmitDCFTrackerUpdateBatch, and if I do not throw an exception, then also by vpiSubmitDCFTrackerLocalizeBatch (in the next iteration).

To reproduce, you can use this code (single iteration). You need to take some image as an input, which has at least the dimensions of the bounding box:

#include <vpi/Stream.h>
#include <vpi/OpenCVInterop.hpp>
#include <vpi/Pyramid.h>
#include <vpi/algo/DCFTracker.h>
#include <vpi/algo/CropScaler.h>
#include <vpi/Array.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/Status.h>

#include <opencv2/imgcodecs.hpp>

void VPI_CHECK(VPIStatus status, const std::string& errorMessage = "") { if (status) throw std::runtime_error("VPIStatus code: " + std::to_string(status) + ". " + errorMessage); }

struct trackedObjectInfo {
	int nNotTracked = 0;
};

int main() {

	VPIStream vpiStream;
	VPIPayload cropPayload;
	VPIPayload dcfPayload;
	VPIDCFTrackerParams dcfTrackerParams;
	VPIArray trackedObjects;
	VPIArray trackedObjectsCorrelationResponses;
	VPIImage objPatches;
	int patchW;
	int patchH;

	int max_det = 10;
	float filterLR = 0.5f;
	float filterChannelWeightsLR = 0.5f;

	vpiStreamCreate(0, &vpiStream);
	VPI_CHECK(vpiCreateCropScaler(VPI_BACKEND_CUDA, 1, max_det, &cropPayload));

	VPIDCFTrackerCreationParams dcfTrackerCreateParams;
	VPI_CHECK(vpiInitDCFTrackerCreationParams(&dcfTrackerCreateParams));

	VPI_CHECK(vpiCreateDCFTracker(VPI_BACKEND_CUDA, 1, max_det, &dcfTrackerCreateParams, &dcfPayload));
	VPI_CHECK(vpiInitDCFTrackerParams(&dcfTrackerParams));
	VPI_CHECK(vpiArrayCreate(max_det, VPI_ARRAY_TYPE_DCF_TRACKED_BOUNDING_BOX, VPI_BACKEND_CUDA|VPI_BACKEND_CPU, &trackedObjects));
	VPI_CHECK(vpiArrayCreate(max_det, VPI_ARRAY_TYPE_F32, VPI_BACKEND_CUDA|VPI_BACKEND_CPU, &trackedObjectsCorrelationResponses));
	patchH = patchW = dcfTrackerCreateParams.featurePatchSize * dcfTrackerCreateParams.hogCellSize;
	VPI_CHECK(vpiImageCreate(patchW,
	                         patchH * max_det,
	                         VPI_IMAGE_FORMAT_RGBA8,
	                         VPI_BACKEND_CUDA|VPI_BACKEND_CPU,
	                         &objPatches));


	// DEFINE IMG AND BBOXES HERE
	auto cvImg = cv::imread("/path/to/image");
	std::vector<std::vector<cv::Rect>> bboxes{{{200, 200, 200, 200}}};

	VPIImage vpiImg;
	VPI_CHECK(vpiImageCreateWrapperOpenCVMat(cvImg, VPI_BACKEND_CUDA, &vpiImg));
	VPIImage vpiImgRGBA8;
	VPI_CHECK(vpiImageCreate(cvImg.cols, cvImg.rows, VPI_IMAGE_FORMAT_RGBA8, VPI_BACKEND_CUDA, &vpiImgRGBA8));
	VPI_CHECK(vpiSubmitConvertImageFormat(vpiStream, VPI_BACKEND_CUDA, vpiImg, vpiImgRGBA8, nullptr));

	// Track
	int nTrackedObjects;
	VPI_CHECK(vpiArrayGetSize(trackedObjects, &nTrackedObjects));


	// update objects
	if (bboxes[0].size() > 0)
	{
		VPIArrayData trackedObjectsData;
		VPI_CHECK(vpiArrayLockData(trackedObjects, VPI_LOCK_READ_WRITE, VPI_ARRAY_BUFFER_HOST_AOS, &trackedObjectsData));

		// collect new objects
		std::vector<int> newObjects;
		for (int i = 0; i < bboxes[0].size(); ++i) {
			if (true)
				newObjects.push_back(i);
		}
		int nNewObjects = newObjects.size();

		// add new objects (not associated)
		VPI_CHECK(vpiArrayGetSize(trackedObjects, &nTrackedObjects));
		VPI_CHECK(vpiArraySetSize(trackedObjects, nTrackedObjects + nNewObjects));
		for (int i = 0; i < nNewObjects; ++i)
		{
			int idxBbox = newObjects[i];
			int idxObject = nTrackedObjects + i;
			VPIAxisAlignedBoundingBoxF32 newObjectBbox{(float)bboxes[0][idxBbox].x, (float)bboxes[0][idxBbox].y, (float)bboxes[0][idxBbox].width, (float)bboxes[0][idxBbox].height};
			VPIDCFTrackedBoundingBox newObject{newObjectBbox, VPI_TRACKING_STATE_NEW, 0, filterLR, filterChannelWeightsLR, new trackedObjectInfo{0}};
			((VPIDCFTrackedBoundingBox*)trackedObjectsData.buffer.aos.data)[idxObject] = newObject;
		}

		VPI_CHECK(vpiArrayUnlock(trackedObjects));
	}

	// update tracker
	VPI_CHECK(vpiArrayGetSize(trackedObjects, &nTrackedObjects));
	if (nTrackedObjects > 0)
	{
		VPIArrayData trackedObjectsData;

		// Extract from the current frame the image patches of each object given its refined bounding box.
		VPI_CHECK(vpiSubmitCropScalerBatch(vpiStream, VPI_BACKEND_CUDA, cropPayload,
		                                   &vpiImgRGBA8, 1, trackedObjects,
		                                   patchW, patchH, objPatches));

		// Update tracking information
		VPI_CHECK(vpiSubmitDCFTrackerUpdateBatch(vpiStream, VPI_BACKEND_CUDA, dcfPayload,
		                                         nullptr, 1,
		                                         nullptr, nullptr,
		                                         objPatches, trackedObjects, &dcfTrackerParams));

		VPI_CHECK(vpiStreamSync(vpiStream));

		VPI_CHECK(vpiArrayLockData(trackedObjects, VPI_LOCK_READ_WRITE, VPI_ARRAY_BUFFER_HOST_AOS, &trackedObjectsData));


		for (int i = 0; i < nTrackedObjects; ++i) {
			if (((VPIDCFTrackedBoundingBox *) trackedObjectsData.buffer.aos.data)[i].state != VPI_TRACKING_STATE_LOST)
				((VPIDCFTrackedBoundingBox *) trackedObjectsData.buffer.aos.data)[i].state = VPI_TRACKING_STATE_TRACKED;
		}

		VPI_CHECK(vpiArrayUnlock(trackedObjects));
	}

	vpiImageDestroy(vpiImg);

	return 0;
}

Cmake file contents:

add_executable(vpiError
        vpiError.cu)

find_package(vpi REQUIRED)
find_package(OpenCV 4 REQUIRED)

target_link_libraries(vpiError
        PUBLIC
        ${OpenCV_LIBS}
        vpi)

target_include_directories(vpiError
        PUBLIC
        ${OpenCV_INCLUDE_DIRS}
)
5

Hi,

Thanks for sharing.

We will check it and provide you with more information.

Thanks.

6

Hello!

One week has passed since my last message. Are there any updates?

Thank you in advance

7

Hi,

Our internal team is still checking this issue.
Will let you know once we get feedback.

Thanks.

9

Hello!

Just a reminder. Any updates?

10

Hi,

VPI_ERROR_INVALID_ARGUMENT might relate to the usage of our DCF feature.
Our internal team is working on a sample to demonstrate the functionality.

Will let you know once it is ready.
Thanks.

11

Hello!

Thank you! Looking forward for your sample.

12

Will this sample also be included in the upcoming VPI releases/documentation?

13

Hello!

One month has passed since I shared the minimal reproducible sample. Do you have any updates?

Thanks in advance

14

Hi,

Thanks a lot for the patience.

The sample is ready internally and is now under QA testing.
If things go well, it will be available in the upcoming JetPack release.

Thanks.

15

Hi,

The sample is released in VPI 3.2 / JetPack 6.1.
You can find the document in the below link accordingly.

https://docs.nvidia.com/vpi/sample_dcf_tracker.html

Thanks.

1 Like
17

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.