Using nvbufsurface to run custom cuda kernel

Jetson orin nx 8Gb

deepstream 6.3

jetpack 5.1.3

tensorRt 8.5

Hello sir,
I am using NVIDIA jetson boards to apply computer vision algorithms on robots and have a serious problem that I have been dealing with this issue for 6 months.
In my previous architecture, i sink coming frames to my C++ code by appsink gstreamer plugin and after that i could process on frames, but this solution causes delay about 8ms. now i could write algorithms in cuda language and wanna run them on frames, my serious problem is that convert frames from CPU after appsink to GPU or cuda memory causes delay again. my frames are in cuda memory (SURFACE_ARRAY (memType = 4)) before sink to code and i could run cuda kernels directly on frames stored in cuda memory and remove sink to C++ code. i used gstdsexample code to do this job from deapstream 6.3 and change transform_ip function like below but i got illegal memory access.

// CUDA kernel launcher
extern "C" void launch_drawRedRectRGBA(void* devPtr,
                                       int width, int height,
                                       int pitch, int thickness);

// Helper macro for CUDA debugging
#define CUDA_CHECK(call) do { \
    cudaError_t _status = call; \
    if (_status != cudaSuccess) { \
        g_printerr("CUDA Error %d: %s at %s:%d\n", _status, cudaGetErrorString(_status), __FILE__, __LINE__); \
    } \
} while(0)

// ---------------------------------------------------------------------------
// Simplified gst_dsexample_transform_ip: runs a CUDA kernel on each frame.
// ---------------------------------------------------------------------------
static GstFlowReturn
gst_dsexample_transform_ip(GstBaseTransform *btrans, GstBuffer *inbuf)
{
    GstDsExample *dsexample = GST_DSEXAMPLE(btrans);
    GstMapInfo in_map_info;
    NvBufSurface *surface = NULL;
    NvDsBatchMeta *batch_meta = NULL;
    NvDsMetaList *l_frame = NULL;
    NvDsFrameMeta *frame_meta = NULL;
    GstFlowReturn flow_ret = GST_FLOW_ERROR;

    CUDA_CHECK(cudaSetDevice(dsexample->gpu_id));

    if (!gst_buffer_map(inbuf, &in_map_info, GST_MAP_READWRITE)) {
        g_printerr("Error: Failed to map input buffer\n");
        return GST_FLOW_ERROR;
    }

    surface = (NvBufSurface *)in_map_info.data;
    if (!surface) {
        g_printerr("Error: NvBufSurface not found\n");
        goto done;
    }

    if (CHECK_NVDS_MEMORY_AND_GPUID(dsexample, surface))
        goto done;

    batch_meta = gst_buffer_get_nvds_batch_meta(inbuf);
    if (!batch_meta) {
        g_printerr("Error: NvDsBatchMeta not found\n");
        goto done;
    }

    // Process each frame in batch
    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next)
    {
        frame_meta = (NvDsFrameMeta *)(l_frame->data);
        guint batch_id = frame_meta->batch_id;

        // ------------------------------------------------------------------
        // Ensure surface is mapped to CUDA
        // ------------------------------------------------------------------
        if (surface->memType == NVBUF_MEM_SURFACE_ARRAY) {
            if (NvBufSurfaceMap(surface, batch_id, 0, NVBUF_MAP_READ_WRITE) != 0) {
                g_printerr("Failed to map surface for CUDA\n");
                continue;
            }
            if (NvBufSurfaceSyncForDevice(surface, batch_id, 0) != 0) {
                g_printerr("NvBufSurfaceSyncForDevice failed\n");
                NvBufSurfaceUnMap(surface, batch_id, 0);
                continue;
            }
        }

        // ------------------------------------------------------------------
        // Get CUDA-accessible pointer and parameters
        // ------------------------------------------------------------------
        int width  = surface->surfaceList[batch_id].planeParams.width[0];
        int height = surface->surfaceList[batch_id].planeParams.height[0];
        int pitch  = surface->surfaceList[batch_id].planeParams.pitch[0];
        void *devPtr = surface->surfaceList[batch_id].dataPtr;

        if (!devPtr) {
            g_printerr("Null devPtr for frame %d\n", batch_id);
            continue;
        }

        // ------------------------------------------------------------------
        // Launch CUDA kernel
        // ------------------------------------------------------------------
        launch_drawRedRectRGBA(devPtr, width, height, pitch, 5);

        // Check for CUDA errors after kernel
        CUDA_CHECK(cudaPeekAtLastError());
        CUDA_CHECK(cudaDeviceSynchronize());

        // ------------------------------------------------------------------
        // Sync back if CPU might access it later
        // ------------------------------------------------------------------
        if (surface->memType == NVBUF_MEM_SURFACE_ARRAY) {
            if (NvBufSurfaceSyncForCpu(surface, batch_id, 0) != 0)
                g_printerr("NvBufSurfaceSyncForCpu failed\n");
            NvBufSurfaceUnMap(surface, batch_id, 0);
        }
    }

    flow_ret = GST_FLOW_OK;

done:
    gst_buffer_unmap(inbuf, &in_map_info);
    return flow_ret;
}

and my cuda kernel is so simple:

#include <cuda_runtime.h>

__global__ void drawRedRectRGBA(uchar4* img, int width, int height, int pitch_bytes, int thickness) {
    int x = blockIdx.x * blockDim.x + threadIdx.x;
    int y = blockIdx.y * blockDim.y + threadIdx.y;

    if (x >= width || y >= height) return;

    uchar4* row = (uchar4*)((char*)img + y * pitch_bytes);  // pitch in bytes

    if (x < thickness || x >= width - thickness || y < thickness || y >= height - thickness) {
        row[x] = make_uchar4(255, 0, 0, 255); // red
    }
}

extern "C" void launch_drawRedRectRGBA(void* devPtr,
                                       int width, int height, int pitch,
                                       int thickness)
{
    dim3 block(16, 16);
    dim3 grid((width  + block.x - 1) / block.x,
              (height + block.y - 1) / block.y);

    drawRedRectRGBA<<<grid, block>>>((uchar4*)devPtr, width, height, pitch, thickness);
    cudaDeviceSynchronize();
}

when i make so file of this plugin and run it with below gstreamer pipeline, i illegal memory access, at this stage frames show correctly but without effect of cuda kernel.

a@a:~/Desktop$ gst-launch-1.0 v4l2src device=/dev/video0 ! “video/x-raw, width=1920, height=1080, framerate=60/1” ! nvvideoconvert compute-hw=2 copy-hw=2 nvbuf-memory-type=4 ! “video/x-raw(memory:NVMM), format=RGBA” ! mux.sink_0 nvstreammux name=mux batch-size=1 width=1920 height=1080 nvbuf-memory-type=4 compute-hw=2 ! dsexample ! nvvideoconvert compute-hw=2 copy-hw=2 nvbuf-memory-type=4 ! fpsdisplaysink sync=false
Setting pipeline to PAUSED …
Pipeline is live and does not need PREROLL …
Setting pipeline to PLAYING …
New clock: GstSystemClock
CUDA Error 700: an illegal memory access was encountered at gstdsexample.cpp:785
CUDA Error 700: an illegal memory access was encountered at gstdsexample.cpp:786
CUDA Error 700: an illegal memory access was encountered at gstdsexample.cpp:785
CUDA Error 700: an illegal memory access was encountered at gstdsexample.cpp:786
CUDA Error 700: an illegal memory access was encountered at gstdsexample.cpp:786
CUDA Error 700: an illegal memory access was encountered at gstdsexample.cpp:785
CUDA Error 700: an illegal memory access was encountered at gstdsexample.cpp:786
^Chandling interrupt.
Interrupt: Stopping pipeline …
Execution ended after 0:00:03.536666880
Setting pipeline to NULL …
^C

In general, I want to run the cuda kernel before the downstream on the frames stored in the Cuda memory. wanna help to write this code or tell me how i must change this codes for applying effect of frames, even simple sample code that what i must do …
TNX alot

  1. Please upgrade the DeepStream version to the latest 7.1 GA.
  2. If you are referring the gstdsexample source code for how to get the CUDA memory from NvBufSurface. Please notice the following code inside the get_converted_mat() function:
...
  if(dsexample->is_integrated) {
#ifdef __aarch64__
    /* To use the converted buffer in CUDA, create an EGLImage and then use
    * CUDA-EGL interop APIs */
    if (USE_EGLIMAGE) {
      if (NvBufSurfaceMapEglImage (dsexample->inter_buf, 0) !=0 ) {
        goto error;
      }
      /* dsexample->inter_buf->surfaceList[0].mappedAddr.eglImage
      * Use interop APIs cuGraphicsEGLRegisterImage and
      * cuGraphicsResourceGetMappedEglFrame to access the buffer in CUDA */

      /* Destroy the EGLImage */
      NvBufSurfaceUnMapEglImage (dsexample->inter_buf, 0);
    }
#endif
  }
...

If you need the detailed usage of the “NvBufSurfaceMapEglImage”, “cuGraphicsEGLRegisterImage”, “cuGraphicsResourceGetMappedEglFrame” and “NvBufSurfaceUnMapEglImage” interfaces, please refer to the documents NVIDIA DeepStream SDK API Reference: NvBufSurface Types and Functions | NVIDIA Docs and CUDA Driver API :: CUDA Toolkit Documentation

There is also sample inside gst_nvinfer_allocator_alloc() function in /opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvinfer/gstnvinfer_allocator.cpp

Hello fiona.chen

Using get_converted_mat() and eglimage library not zero-copy method, i wrote a code with method and mapping egl buffer take about 6-7 ms that is so high for a camera 60 fps rate, it means that this way is not zero copy method …

I wanna a method to have or get frames in cuda with 0 delay or something near that. Is there any way to do this?

So that is why we normally allocated several buffers in pool and map them in advance, so that the following process can be done in the mapped cuda buffers in loop. It is not a necessary to do the mapping for every frame.

Hello Fiona,

I hope you are doing well.

I have a question regarding frame mapping in CUDA. At the moment, I assume that CUDA code cannot run continuously without mapping each frame individually. I have attached a portion of my code that handles this process, and I would appreciate hearing your thoughts on it.

My understanding is that mapping each frame is necessary, but if it is possible to map multiple frames at once (e.g., 10 frames), I would be grateful if you could advise me on how to implement this. Additionally, if there are ways to improve the performance of this section of the code, I would greatly appreciate your guidance.

Thank you very much for your time and support.

Best regards

static GstFlowReturn

process_frame_cuda_direct(GstDsExample *dsexample, NvBufSurface *surface, 

                          gint batch_id)

{

auto t_start = std::chrono::high_resolution_clock::now();

    NvBufSurfaceParams *surf_params = &surface->surfaceList[batch_id];

void *cuda_ptr = NULL;

    EGLResourceCache *cache = NULL;

    guint64 current_fd = (guint64)surf_params->bufferDesc;




    // OPTIMIZED: Quick lookup in pre-mapped buffer pool

for (int i = 0; i < dsexample->egl_cache_pool.num_cached; i++) {

if (dsexample->egl_cache_pool.buffers[i].dmabuf_fd == current_fd) {

            cache = &dsexample->egl_cache_pool.buffers[i];

break;

        }

    }




    // If not found, map it once and cache it

if (cache == NULL) {

if (dsexample->egl_cache_pool.num_cached < MAX_CACHED_BUFFERS) {

            cache = &dsexample->egl_cache_pool.buffers[dsexample->egl_cache_pool.num_cached];

dsexample->egl_cache_pool.num_cached++;

        } else {

            // Replace oldest buffer

            cache = &dsexample->egl_cache_pool.buffers[0];

if (cache->is_mapped) {

cuGraphicsUnmapResources(1, &cache->pResource, dsexample->cuda_stream);

cuGraphicsUnregisterResource(cache->pResource);

            }

        }

        // Map the buffer (this should only happen once per unique buffer)

NvBufSurfaceMapEglImage(surface, batch_id);

cuGraphicsEGLRegisterImage(&cache->pResource,

surf_params->mappedAddr.eglImage,

                                   CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);

cuGraphicsMapResources(1, &cache->pResource, dsexample->cuda_stream);

cuGraphicsResourceGetMappedEglFrame(&cache->eglFrame, 

cache->pResource, 0, 0);

cache->cuda_ptr = (void *)cache->eglFrame.frame.pPitch[0];

cache->dmabuf_fd = current_fd;

cache->is_mapped = TRUE;

GST_DEBUG("Mapped new buffer fd=%lu, cache_idx=%d", current_fd, 

dsexample->egl_cache_pool.num_cached - 1);

    }




auto t_map_done = std::chrono::high_resolution_clock::now();

double map_time_ms = std::chrono::duration<double, std::milli>(t_map_done - t_start).count();

    cuda_ptr = cache->cuda_ptr;

int width = surf_params->planeParams.width[0];

int height = surf_params->planeParams.height[0];

int pitch_bytes = surf_params->planeParams.pitch[0];




    // CRITICAL: All operations on the SAME stream for maximum throughput

if (dsexample->gray_mode) 

    {

        // Convert RGBA to Grayscale

rgbaToGray(cuda_ptr, dsexample->gray_dev, width, height,

                  pitch_bytes, dsexample->gray_pitch, dsexample->cuda_stream);




        // Image enhancement pipeline

if (dsexample->use_enhance) {

grayEnhance(dsexample->gray_dev, width, height, dsexample->gray_pitch,

dsexample->contrast, dsexample->brightness, dsexample->gamma,

dsexample->cuda_stream);

        }




if (dsexample->use_clahe && dsexample->clahe_d_lut) {

grayClahe(

dsexample->gray_dev,

                width, height,

                (int)dsexample->gray_pitch,

dsexample->clahe_clip_limit,

dsexample->clahe_tile_size,

dsexample->clahe_d_lut,

dsexample->cuda_stream);

        }




if (dsexample->use_histeq) {

HistEQ(dsexample->gray_dev, width, height, 

                  (int)dsexample->gray_pitch, dsexample->cuda_stream);

        }




if (dsexample->use_adaptive_histeq && dsexample->hist_state) {

adaptiveHistogramEqualization(

dsexample->gray_dev, dsexample->gray_dev,

                width, height,

                (int)dsexample->gray_pitch, (int)dsexample->gray_pitch,

dsexample->adaptive_percentile,

dsexample->adaptive_mapping_type,

dsexample->hist_state,

dsexample->cuda_stream);

        }




        // OPTIMIZED: Detect and track tiny objects

if (dsexample->d_objects) {

detectAndTrackTinyObjects(

dsexample->gray_dev,

                width, height,

                (int)dsexample->gray_pitch,

dsexample->d_objects,

dsexample->d_object_count,

dsexample->d_tracks,

dsexample->d_track_count,

dsexample->d_next_id,

dsexample->d_labels,

dsexample->d_binary,

                (int)dsexample->binary_pitch,

dsexample->cuda_stream

            );

        }




        // Convert back to RGBA

grayToRgba(dsexample->gray_dev, cuda_ptr, width, height,

dsexample->gray_pitch, pitch_bytes, dsexample->cuda_stream);




        // Draw reticle overlay

if (dsexample->reticle_enabled) {

draw_reticle(

                cuda_ptr,

                width, height,

                pitch_bytes,

dsexample->reticle_x0,

dsexample->reticle_y0,

dsexample->reticle_x1,

dsexample->reticle_y1,

dsexample->reticle_thickness,

dsexample->reticle_mode,

dsexample->reticle_search_area,

dsexample->cuda_stream);

        }

    }

else

    {

if (dsexample->use_enhance) {

rgbaEnhance(cuda_ptr, width, height, pitch_bytes, 

dsexample->contrast, dsexample->brightness, dsexample->gamma, 

dsexample->cuda_stream);

        }

    }




    // CRITICAL: Only one sync point at the end

cudaStreamSynchronize(dsexample->cuda_stream);




auto t_done = std::chrono::high_resolution_clock::now();

double total_time_ms = std::chrono::duration<double, std::milli>(t_done - t_start).count();

    // Statistics (only print occasionally to avoid overhead)

static int frame_count = 0;

static double total_proc_time = 0;

    frame_count++;

    total_proc_time += total_time_ms;

if (frame_count % 60 == 0) {

double avg_time = total_proc_time / 60.0;

double fps = 1000.0 / avg_time;

printf("=== PERF: Avg %.2f ms/frame (%.1f FPS), Mapping: %.3f ms ===\n", 

               avg_time, fps, map_time_ms);

        total_proc_time = 0;

    }




return GST_FLOW_OK;

}




static GstFlowReturn

gst_dsexample_transform_ip(GstBaseTransform *btrans, GstBuffer *inbuf)

{

    GstDsExample *dsexample = GST_DSEXAMPLE(btrans);

    GstMapInfo in_map_info;

    GstFlowReturn flow_ret = GST_FLOW_ERROR;

    NvBufSurface *surface = NULL;

    NvDsBatchMeta *batch_meta = NULL;

    NvDsFrameMeta *frame_meta = NULL;

    NvDsMetaList *l_frame = NULL;




dsexample->frame_num++;

cudaSetDevice(dsexample->gpu_id);




memset(&in_map_info, 0, sizeof(in_map_info));

gst_buffer_map(inbuf, &in_map_info, GST_MAP_READ);




nvds_set_input_system_timestamp(inbuf, GST_ELEMENT_NAME(dsexample));

    surface = (NvBufSurface *)in_map_info.data;




    batch_meta = gst_buffer_get_nvds_batch_meta(inbuf);




for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;

         l_frame = l_frame->next) {

        frame_meta = (NvDsFrameMeta *)(l_frame->data);

process_frame_cuda_direct(dsexample, surface, frame_meta->batch_id);

    }




    flow_ret = GST_FLOW_OK;




nvds_set_output_system_timestamp(inbuf, GST_ELEMENT_NAME(dsexample));

gst_buffer_unmap(inbuf, &in_map_info);

return flow_ret;

}

The frames in the batch are not guaranteed to be allocated in the continuous CUDA memory. There is no way to do what you say.

The nvv4l2decoder, nvstreammux, … plugins allocate limited number of NvBufSurface buffers for processing. The mapping is needed for the first time only.

Hi! Since we haven’t heard back from you for a while, we’re assuming everything is resolved and will close this topic. If you need any further help, don’t hesitate to open a new topic. Thanks!