gpu videoreader doesn’t work on Jetson for now.
But you can use nvivafilter plugin, as mentioned by @DaneLLL.
As sample, assuming you’re using opencv4tegra installed in /usr, you may try the code below.
[EDIT: For opencv3, you may use sample files from post #23 below]
Note that I’ve all included in one src file and one makefile, and changed @DaneLLL blur filter to a Sobel filter.
[Late EDIT: This was not a good idea. Using a Sobel filter with CV_8UC4 is wrong because the alpha channel can get wrong. I don’t know why EGL backend deals with it, but using nvvidconv for converting to I420 shows a black image (using Sobel Filter on splitted R,G,B and not alpha does what is expected). Note that filter may be created only once in init function.]
Pay attention to function cv_process(), this is where you will add your processing:
File gst-custom-opencv_cudaprocess.cu
/*
* Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <cuda.h>
#include <opencv2/opencv.hpp>
#include <opencv2/gpu/gpu.hpp>
#include "cudaEGL.h"
#if defined(__cplusplus)
extern "C" void Handle_EGLImage (EGLImageKHR image);
extern "C" {
#endif
typedef enum {
COLOR_FORMAT_Y8 = 0,
COLOR_FORMAT_U8_V8,
COLOR_FORMAT_RGBA,
COLOR_FORMAT_NONE
} ColorFormat;
typedef struct {
/**
* cuda-process API
*
* @param image : EGL Image to process
* @param userPtr : point to user alloc data, should be free by user
*/
void (*fGPUProcess) (EGLImageKHR image, void ** userPtr);
/**
* pre-process API
*
* @param sBaseAddr : Mapped Surfaces(YUV) pointers
* @param smemsize : surfaces size array
* @param swidth : surfaces width array
* @param sheight : surfaces height array
* @param spitch : surfaces pitch array
* @param sformat : surfaces format array
* @param nsurfcount : surfaces count
* @param userPtr : point to user alloc data, should be free by user
*/
void (*fPreProcess)(void **sBaseAddr,
unsigned int *smemsize,
unsigned int *swidth,
unsigned int *sheight,
unsigned int *spitch,
ColorFormat *sformat,
unsigned int nsurfcount,
void ** userPtr);
/**
* post-process API
*
* @param sBaseAddr : Mapped Surfaces(YUV) pointers
* @param smemsize : surfaces size array
* @param swidth : surfaces width array
* @param sheight : surfaces height array
* @param spitch : surfaces pitch array
* @param sformat : surfaces format array
* @param nsurfcount : surfaces count
* @param userPtr : point to user alloc data, should be free by user
*/
void (*fPostProcess)(void **sBaseAddr,
unsigned int *smemsize,
unsigned int *swidth,
unsigned int *sheight,
unsigned int *spitch,
ColorFormat *sformat,
unsigned int nsurfcount,
void ** userPtr);
} CustomerFunction;
void init (CustomerFunction * pFuncs);
#if defined(__cplusplus)
}
#endif
/**
* Dummy custom pre-process API implematation.
* It just access mapped surface userspace pointer &
* memset with specific pattern modifying pixel-data in-place.
*
* @param sBaseAddr : Mapped Surfaces pointers
* @param smemsize : surfaces size array
* @param swidth : surfaces width array
* @param sheight : surfaces height array
* @param spitch : surfaces pitch array
* @param nsurfcount : surfaces count
*/
static void
pre_process (void **sBaseAddr,
unsigned int *smemsize,
unsigned int *swidth,
unsigned int *sheight,
unsigned int *spitch,
ColorFormat *sformat,
unsigned int nsurfcount,
void ** usrptr)
{
/* add your custom pre-process here */
}
/**
* Dummy custom post-process API implematation.
* It just access mapped surface userspace pointer &
* memset with specific pattern modifying pixel-data in-place.
*
* @param sBaseAddr : Mapped Surfaces pointers
* @param smemsize : surfaces size array
* @param swidth : surfaces width array
* @param sheight : surfaces height array
* @param spitch : surfaces pitch array
* @param nsurfcount : surfaces count
*/
static void
post_process (void **sBaseAddr,
unsigned int *smemsize,
unsigned int *swidth,
unsigned int *sheight,
unsigned int *spitch,
ColorFormat *sformat,
unsigned int nsurfcount,
void ** usrptr)
{
/* add your custom post-process here */
}
static void cv_process(void *pdata, int32_t width, int32_t height)
{
/* Create a GpuMat with data pointer */
cv::gpu::GpuMat d_mat(height, width, CV_8UC4, pdata);
/* Apply Sobel filter */
cv::gpu::Sobel(d_mat, d_mat, CV_8UC4, 1, 1, 1, 1, 1, cv::BORDER_DEFAULT);
}
/**
* Performs CUDA Operations on egl image.
*
* @param image : EGL image
*/
static void
gpu_process (EGLImageKHR image, void ** usrptr)
{
CUresult status;
CUeglFrame eglFrame;
CUgraphicsResource pResource = NULL;
cudaFree(0);
status = cuGraphicsEGLRegisterImage(&pResource, image, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE);
if (status != CUDA_SUCCESS) {
printf("cuGraphicsEGLRegisterImage failed : %d \n", status);
return;
}
status = cuGraphicsResourceGetMappedEglFrame( &eglFrame, pResource, 0, 0);
if (status != CUDA_SUCCESS) {
printf ("cuGraphicsSubResourceGetMappedArray failed\n");
}
status = cuCtxSynchronize();
if (status != CUDA_SUCCESS) {
printf ("cuCtxSynchronize failed \n");
}
if (eglFrame.frameType == CU_EGL_FRAME_TYPE_PITCH) {
if (eglFrame.eglColorFormat == CU_EGL_COLOR_FORMAT_RGBA) {
/* Apply CV gpu processing */
cv_process(eglFrame.frame.pPitch[0], eglFrame.width, eglFrame.height);
} else {
printf ("Invalid eglcolorformat for opencv\n");
}
}
else {
printf ("Invalid frame type for opencv\n");
}
status = cuCtxSynchronize();
if (status != CUDA_SUCCESS) {
printf ("cuCtxSynchronize failed after memcpy \n");
}
status = cuGraphicsUnregisterResource(pResource);
if (status != CUDA_SUCCESS) {
printf("cuGraphicsEGLUnRegisterResource failed: %d \n", status);
}
}
extern "C" void
init (CustomerFunction * pFuncs)
{
pFuncs->fPreProcess = pre_process;
pFuncs->fGPUProcess = gpu_process;
pFuncs->fPostProcess = post_process;
}
Now, the makefile to be saved in the same directory:
###############################################################################
#
# Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of NVIDIA CORPORATION nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
###############################################################################
# Location of the CUDA Toolkit
CUDA_PATH ?= /usr/local/cuda
INCLUDE_DIR = /usr/include
LIB_DIR = /usr/lib/aarch64-linux-gnu
TEGRA_LIB_DIR = /usr/lib/aarch64-linux-gnu/tegra
# This is typical install path of opencv4tegra
OPENCV_DIR = /usr
# For hardfp
#LIB_DIR = /usr/lib/arm-linux-gnueabihf
#TEGRA_LIB_DIR = /usr/lib/arm-linux-gnueabihf/tegra
OSUPPER = $(shell uname -s 2>/dev/null | tr "[:lower:]" "[:upper:]")
OSLOWER = $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
OS_SIZE = $(shell uname -m | sed -e "s/i.86/32/" -e "s/x86_64/64/" -e "s/armv7l/32/")
OS_ARCH = $(shell uname -m | sed -e "s/i386/i686/")
GCC ?= g++
NVCC := $(CUDA_PATH)/bin/nvcc -ccbin $(GCC)
# internal flags
NVCCFLAGS := --shared
CCFLAGS := -fPIC
CVCCFLAGS:=-I$(OPENCV_DIR)/include
CVLDFLAGS:=-L$(OPENCV_DIR)/lib -lopencv_core -lopencv_gpu
LDFLAGS :=
# Extra user flags
EXTRA_NVCCFLAGS ?=
EXTRA_LDFLAGS ?=
EXTRA_CCFLAGS ?=
override abi := aarch64
LDFLAGS += --dynamic-linker=/lib/ld-linux-aarch64.so.1
# For hardfp
#override abi := gnueabihf
#LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
#CCFLAGS += -mfloat-abi=hard
ifeq ($(ARMv7),1)
NVCCFLAGS += -target-cpu-arch ARM
ifneq ($(TARGET_FS),)
CCFLAGS += --sysroot=$(TARGET_FS)
LDFLAGS += --sysroot=$(TARGET_FS)
LDFLAGS += -rpath-link=$(TARGET_FS)/lib
LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/$(abi)-linux-gnu
# For hardfp
#LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-$(abi)
endif
endif
# Debug build flags
dbg = 0
ifeq ($(dbg),1)
NVCCFLAGS += -g -G
TARGET := debug
else
TARGET := release
endif
ALL_CCFLAGS :=
ALL_CCFLAGS += $(NVCCFLAGS)
ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
ALL_LDFLAGS :=
ALL_LDFLAGS += $(ALL_CCFLAGS)
ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
# Common includes and paths for CUDA
INCLUDES := -I./
LIBRARIES := -L$(LIB_DIR) -lEGL -lGLESv2
LIBRARIES += -L$(TEGRA_LIB_DIR) -lcuda -lrt
################################################################################
# CUDA code generation flags
ifneq ($(OS_ARCH),armv7l)
GENCODE_SM10 := -gencode arch=compute_10,code=sm_10
endif
GENCODE_SM20 := -gencode arch=compute_20,code=sm_20
GENCODE_SM30 := -gencode arch=compute_30,code=sm_30
GENCODE_SM32 := -gencode arch=compute_32,code=sm_32
GENCODE_SM35 := -gencode arch=compute_35,code=sm_35
GENCODE_SM50 := -gencode arch=compute_50,code=sm_50
GENCODE_SMXX := -gencode arch=compute_50,code=compute_50
GENCODE_SM53 := -gencode arch=compute_53,code=compute_53 # for TX1
GENCODE_SM62 := -gencode arch=compute_62,code=compute_62 # for TX2
ifeq ($(OS_ARCH),armv7l)
GENCODE_FLAGS ?= $(GENCODE_SM32)
else
# This only support TX1(5.3) or TX2(6.2) -like architectures
GENCODE_FLAGS ?= $(GEGENCODE_SM53) $(GENCODE_SM62)
endif
# Target rules
all: build
build: lib-gst-custom-opencv_cudaprocess.so
gst-custom-opencv_cudaprocess.o : gst-custom-opencv_cudaprocess.cu
$(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(CVCCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
lib-gst-custom-opencv_cudaprocess.so : gst-custom-opencv_cudaprocess.o
$(NVCC) $(ALL_LDFLAGS) $(CVLDFLAGS) $(GENCODE_FLAGS) -o $@ $^ $(LIBRARIES)
clean:
rm lib-gst-custom-opencv_cudaprocess.so gst-custom-opencv_cudaprocess.o
clobber: clean
Then cd in that directory and just type :
make
for building. It should produce there a file lib-gst-custom-opencv_cudaprocess.so. This dynamic lib can be called from gst plugin nvivafilter.
In order to do that, from the same directory (or giving full path to your lib-gst-custom-opencv_cudaprocess.so), assuming using the TX1/TX2 onboard camera, just run it with gstreamer by:
gst-launch-1.0 nvcamerasrc ! 'video/x-raw(memory:NVMM), width=1280, height=720, format=I420, framerate=120/1' ! nvivafilter customer-lib-name=./lib-gst-custom-opencv_cudaprocess.so cuda-process=true ! 'video/x-raw(memory:NVMM), format=RGBA' ! nvegltransform ! nveglglessink
You should be able to see your onboard camera with sobel filter applied.