I am attempting to modify the feature_tracker demo to work with our camera.
Our implementation is different from the demo in that the LK Tracker is running in a background thread (created by subclassing the ‘Thread.cpp’ found in the tegra_multimedia_api/argus/samples/utils/Thread.cpp) while the main thread is capturing camera images using the ‘select’ function.
Within the constructor (not the thread initializer) of my image processor thread class I setup the lk_tracker and, similar to the main function in the lk_tracker demo, I obtain a current image and previous image from the ‘vx_delay’ element. I keep a reference to these iamges as class members so I do not need to keep calling the ‘vxGetReferenceFromDelay’ in the time sensitive Thread Execute function.
When I first obtain the reference to the images within the constructor I can map patches and umap them just fine but when I try and access those same images within the ‘Thread Execute’ function I receive a -12 from the vx_status, which is: VX_ERROR_INVALID_REFERENCE
I have called the function: ‘vxRetainReference((vx_reference) frame)’ with the idea that perhaps the ‘vx_delay’ had decided that it will destroy my current image references after the the execution left the scope of the constructor but it didn’t work.
Just to be clear I call the constructor from the main thread context and then call the ‘Thread Initialize’ and ‘Thread Execute’ functions within the child thread context.
As a reference here is my constructor:
VisionWorksController::VisionWorksController( uint32_t image_width, uint32_t image_height,
nvxio::ThreadSafeQueue<FRAME *> *ready_image_in_queue,
nvxio::ThreadSafeQueue<FRAME *> *empty_image_in_queue,
nvxio::ThreadSafeQueue<uint8_t *> *ready_image_out_queue,
nvxio::ThreadSafeQueue<uint8_t *> *empty_image_out_queue,
bool debug)
{
vx_status status;
this->debug = debug;
this->image_width = image_width;
this->image_height = image_height;
this->ready_image_in_queue = ready_image_in_queue;
this->empty_image_in_queue = empty_image_in_queue;
this->ready_image_out_queue = ready_image_out_queue;
this->empty_image_out_queue = empty_image_out_queue;
this->first = true;
ovxio::ContextGuard context;
vxDirective(context, VX_DIRECTIVE_ENABLE_PERFORMANCE);
//Setup the logs
vxRegisterLogCallback(context, &ovxio::stdoutLogCallback, vx_false_e);
printf("%s: Setting up LK Tracker for image with size: %d x %d...", __func__, this->image_width, this->image_height );
//Feature Tracker Specific
nvx::FeatureTracker::Params feature_tracker_params;
// parameters for optical flow node
feature_tracker_params.pyr_levels = TRACKER_PYR_LEVELS;
feature_tracker_params.lk_num_iters = TRACKER_LK_NUM_ITERS;
feature_tracker_params.lk_win_size = TRACKER_LK_WIN_SIZE;
// common parameters for corner detector node
feature_tracker_params.array_capacity = TRACKER_ARRAY_CAPACITY;
feature_tracker_params.detector_cell_size = TRACKER_DETECTOR_CELL_SIZE;
feature_tracker_params.use_harris_detector = TRACKER_USE_DETECTOR_HARRIS;
// parameters for harris_track node
feature_tracker_params.harris_k = TRACKER_HARRIS_K;
feature_tracker_params.harris_thresh = TRACKER_HARRIS_THRESH;
// parameters for fast_track node
feature_tracker_params.fast_type = TRACKER_FAST_TYPE;
feature_tracker_params.fast_thresh = TRACKER_FAST_THRESH;
//Frame Delay Object is used to hold previous and current frames from video sources in the frame_delay variable
vx_image image_ex = vxCreateImage(context, this->image_width, this->image_height, VX_DF_IMAGE_U8);
this->frame_delay = vxCreateDelay(context, (vx_reference)image_ex, 2);
if (this->frame_delay == NULL){
printf ("Failed to get Frame Delay\n");
assert(0);
}
vxReleaseImage(&image_ex);
this->prev_image = (vx_image) vxGetReferenceFromDelay(this->frame_delay, -1);
vxRetainReference((vx_reference) this->prev_image);
this->curr_image = (vx_image) vxGetReferenceFromDelay(this->frame_delay, 0);
status = vxRetainReference((vx_reference) this->curr_image);
this->tracker = nvx::FeatureTracker::create(context, feature_tracker_params);
this->sync_timer = nvxio::createSyncTimer();
printf("Success!\n");
//The following is just to test out the mapping and unmapping of patches from the current and previous frame
//Current Image Full Map
this->image_in_rect.start_x = 0u; //Start X
this->image_in_rect.start_y = 0u; //Start Y
this->image_in_rect.end_x = this->image_width; //End X
this->image_in_rect.end_y = this->image_height ; //End Y
vxMapImagePatch(this->curr_image, &this->image_in_rect, 0, &this->image_in_id, &this->image_in_addr, &this->image_in_ptr, VX_WRITE_ONLY, VX_MEMORY_TYPE_HOST, VX_NOGAP_X);
printf("Image In Patch Dimensions:\n");
printf(" Width x Height: %dx%d\n", this->image_in_addr.dim_x, this->image_in_addr.dim_y);
printf(" StepX x StepY: %d,%d\n", this->image_in_addr.step_x, this->image_in_addr.step_y);
printf(" StriX x StriY: %d,%d\n", this->image_in_addr.stride_x, this->image_in_addr.stride_y);
//Previous Image Full Map
this->image_out_rect.start_x = 0u; //Start X
this->image_out_rect.start_y = 0u; //Start Y
this->image_out_rect.end_x = this->image_width; //End X
this->image_out_rect.end_y = this->image_height; //End Y
vxMapImagePatch(this->prev_image, &this->image_out_rect, 0, &this->image_out_id, &this->image_out_addr, &this->image_out_ptr, VX_READ_ONLY, VX_MEMORY_TYPE_HOST, VX_NOGAP_X);
printf("Image Out Patch Dimensions:\n");
printf(" Width x Height: %dx%d\n", this->image_out_addr.dim_x, this->image_out_addr.dim_y);
printf(" StepX x StepY: %d,%d\n", this->image_out_addr.step_x, this->image_out_addr.step_y);
printf(" StriX x StriY: %d,%d\n", this->image_out_addr.stride_x, this->image_out_addr.stride_y);
vxUnmapImagePatch(this->curr_image, this->image_in_id);
vxUnmapImagePatch(this->prev_image, this->image_out_id);
}
Within the above constructor I call map and unmap patch to make sure I got a reference to the images and I have verified that the patch address information looks correct. Here is a reference to the ‘Thread Execute’ function that gets called over and over from Thread base class until the user requests a shutdown/join.
inline bool VisionWorksController::threadExecute()
{
vx_status status;
bool frame_ready = false;
FRAME *fpga_frame = NULL;
uint8_t *output_buffer = NULL;
double total_ms = 0.00;
//uint8_t temp_buffer[1024 * 768];
frame_ready = ready_image_in_queue->pop(fpga_frame, 1000);
if (!frame_ready){
printf("VW Controller: 1S Timed Out\n");
return true;
}
//XXX: Do I need to keep mapping and unmapping these patches or can I do it one time??
status = vxMapImagePatch(this->curr_image, &this->image_in_rect, 0, &this->image_in_id, &this->image_in_addr, &this->image_in_ptr, VX_WRITE_ONLY, VX_MEMORY_TYPE_HOST, 0);
if (status != VX_SUCCESS)
printf ("Failed to map image patch!: %d\n", status);
else
printf("Successfully mapped iamge patch!\n");
//XXX: Is this the best way to do this? Can I use the GPU to copy the data?
memcpy(this->image_in_ptr, fpga_frame->buffer, this->image_height * this->image_width * sizeof(uint8_t));
vxUnmapImagePatch(this->curr_image, this->image_in_id);
//Put the data back in the empty queue
empty_image_in_queue->push(fpga_frame);
if (this->first)
{
this->tracker->init(this->curr_image, NULL);
vxAgeDelay(this->frame_delay);
this->sync_timer->arm(1. / VID_FRAMERATE);
this->total_timer.tic();
proc_ms = 0;
this->first = false;
}
else
{
nvx::Timer proc_timer;
proc_timer.tic();
this->tracker->track(this->curr_image, NULL);
proc_ms = proc_timer.toc();
this->tracker->printPerfs();
this->sync_timer->synchronize();
total_ms = this->total_timer.toc();
vxAgeDelay(this->frame_delay);
printf("LK Tracking Total Time Between Frames: %f ms, Track Processing Time: %f\n", total_ms, proc_ms);
}
//Publish Data!
if (this->empty_image_out_queue->pop(output_buffer, 0)){
//XXX: Do I need to keep mapping and unmapping these patches or can I do it one time??
vxMapImagePatch(this->prev_image, &this->image_out_rect, 0, &this->image_out_id, &this->image_out_addr, &this->image_out_ptr, VX_READ_ONLY, VX_MEMORY_TYPE_HOST, 0);
//XXX: Is this the best way to do this? Can I use the GPU to copy the data?
memcpy(output_buffer, this->image_out_ptr, this->image_height * this->image_width * sizeof(uint8_t));
vxUnmapImagePatch(this->prev_image, this->image_out_id);
this->ready_image_out_queue->push(output_buffer);
}
else
printf("VX Controller: No Output buffer available\n");
return true;
}
Is there something I’m doing wrong?
Besides this issue I have a couple of other questions:
- Do I need to map and unmap a patch from an image every time I want to use it (like I do in the above example?) or can I map a patch one time at the beginning and unmap at the end?
- I found reference to 'nvx_cv::convertCVMatTypeToVXImageFormat' in the documentation but I couldn't find it in any samples or demos, is there any example that uses this? Where is the header file for this? I grepped around the /usr/share/visionworks folder and couldn't find a reference to -i 'CVMat'
Thanks in advance for any help.
Dave