works fine now.
i can get a 30 fps video in color on my other machine. I use ffplay to look at the stream. VLC was kinda slow and ffplay was blazing fast. i have about 200ms of delay but that’s almost not noticeable in my use case. I am very happy with the result!
However it only works with downsampling activated. This feature halfs my 4k cam output (2x2 downsampling). If i want to stream the full 4k the code feezes in the line: (waited for 10 min)
gst_udpsink.write(cpuFrame);
None the less i am happy with the result and have marked your answer as the correct solution!
here is the code i ended up with for anyone interested:
#include <m3api/xiApi.h>
#include <iostream>
#include <opencv2/highgui.hpp>
#include <opencv2/cudaimgproc.hpp>
#include <opencv2/cudaarithm.hpp>
#include <cuda_runtime.h>
#include <chrono>
//example program runs with a limited amount of frames
#define NUMBER_OF_IMAGES 60000
// Define parameters for a static white balance
#define WB_BLUE 2
#define WB_GREEN 1
#define WB_RED 1.3
using namespace std;
int main(int argc, char *argv[]){
// Initialize XI_IMG structure
XI_IMG image;
memset(&image, 0, sizeof(XI_IMG));
image.size = sizeof(XI_IMG);
HANDLE xiH = NULL;
XI_RETURN stat = XI_OK;
//activate downsampling
bool downsampling = true;
//should the camera feed be rendered on the local screen?
bool render_x = true;
double frame_counter = 0;
try{
// Get device handle for the camera
stat = xiOpenDevice(0, &xiH);
if (stat != XI_OK)
throw "Opening device failed";
/* camera settings */
int OCVbayer = cv::COLOR_BayerBG2BGR; // Set demosaicing type
if(downsampling){xiSetParamInt(xiH, XI_PRM_DOWNSAMPLING , 2);} //Activate Downsampling if needed
xiSetParamInt(xiH, XI_PRM_IMAGE_DATA_FORMAT, XI_FRM_TRANSPORT_DATA); // Use transport data format (so we can debayer on gpu)
xiSetParamInt(xiH, XI_PRM_TRANSPORT_DATA_TARGET, XI_TRANSPORT_DATA_TARGET_ZEROCOPY); // Make data from the camera stream to zerocopy memory
xiSetParamInt(xiH, XI_PRM_OUTPUT_DATA_BIT_DEPTH, 8); // Using 8-bit images here
xiSetParamInt(xiH, XI_PRM_EXPOSURE, 30 *1000); // Exposure in ms
int width = -1;
xiGetParamInt(xiH, XI_PRM_WIDTH, &width); // Get width of image
int height = -1;
xiGetParamInt(xiH, XI_PRM_HEIGHT, &height); // Get height of image
// Start the image acquisition
xiStartAcquisition(xiH);
// Define pointer used for data on GPU
// Create GpuMat for the result images
void *imageGpu;
cv::cuda::GpuMat gpu_mat(height, width, CV_8UC3);
// Create a GUI window with OpenGL support
if (render_x){
cv::namedWindow("XIMEA camera", cv::WINDOW_OPENGL);
cv::resizeWindow("XIMEA camera", 1600, 900);
//cv::resizeWindow("XIMEA camera", width/3, height/3);
}
/* video writer to RTP/UDP sink multicast at 224.1.2.1 */
cv::VideoWriter gst_udpsink("appsrc ! video/x-raw, format=BGR, pixel-aspect-ratio=1/1 ! queue ! videoconvert ! video/x-raw, format=BGRx ! nvvidconv ! nvv4l2h264enc insert-vui=1 ! video/x-h264, stream-format=byte-stream, alignment=au ! h264parse ! video/x-h264, stream-format=byte-stream ! rtph264pay pt=96 config-interval=1 ! application/x-rtp, media=video, encoding-name=H264 ! udpsink host=224.1.2.1 port=5000 auto-multicast=true ", 0, 40, cv::Size (width, height));
if (!gst_udpsink.isOpened ()) {
std::cout << "Failed to open gst_udpsink writer." << std::endl;
return (-8);
}
/*
Run on another machine in the same network:
(make sure firewall isn't blocking multicast)
sdp file 'test.sdp':
m=video 5000 RTP/AVP 96
c=IN IP4 224.1.2.1
a=rtpmap:96 H264/90000
receive with ffplay:
$ffplay -fflags nobuffer -flags low_delay -framedrop test.sdp -protocol_whitelistile,udp,rtp
*/
// Acquire a number of images, process and render them
//start timer for fps calculation
auto begin = std::chrono::high_resolution_clock::now();
for (int i = 0; i < NUMBER_OF_IMAGES; i++){
// get the image from the camera
xiGetImage(xiH, 5000, &image); // Get host-pointer to image data
cudaHostGetDevicePointer(&imageGpu, image.bp, 0); // Convert to device pointer
cv::cuda::GpuMat gpu_mat_raw(height, width, CV_8UC1, imageGpu); // Create GpuMat from the device pointer
cv::cuda::demosaicing(gpu_mat_raw, gpu_mat, OCVbayer); // Demosaic raw bayer image to color image
cv::cuda::multiply(gpu_mat, cv::Scalar(WB_BLUE, WB_GREEN, WB_RED), gpu_mat); // Apply static white balance by multiplying the channels
// download to cpu mat
// very slow and i want to evade this
cv::Mat cpuFrame;
gpu_mat.download(cpuFrame);
//push to graphstream
gst_udpsink.write(cpuFrame);
//gst_udpsink.write(gpu_mat);
if (render_x){
// Render image to the screen (using OpenGL)
// i don't want to render to the screen, i want to stream
//cv::imshow("XIMEA camera", gpu_mat);
//cv::imshow("XIMEA camera", img);
cv::imshow("XIMEA camera", cpuFrame);
}
frame_counter++;
cv::waitKey(1);
}
//measure end time for fps calculation
auto finish_time = std::chrono::high_resolution_clock::now();
//calc delta time and fps
auto delta = std::chrono::duration_cast<std::chrono::milliseconds>(finish_time-begin);
double d = std::chrono::duration<double>(delta).count();
double fps = frame_counter/d;
if(downsampling){cout << "Downsampling 2X2 activated" << endl;}
cout<<"FPS: "<< fps << endl;
cout<<"Frame Count: "<< frame_counter << endl;
cout<<"Time needed: "<< delta.count()/1000.0 << endl;
gst_udpsink.release();
// Stop image acquisition and close device
xiStopAcquisition(xiH);
xiCloseDevice(xiH);
// Print errors
}catch(const char* message){
std::cerr << message << std::endl;
}
}
