Need access to raw image data

Hi,

I am working on a Jetson Nano with a 8MP IMX219 arducam. My final goal is to use Argus to create an SO file, which I can interface with Python to get access to the image data, similar to VideoCapture on OpenCV. However gstreamer on OpenCV with the nvargussrc gives me only 15 fps (20 would be preferred) and about 9 frames latency (which is not acceptable).
I modified the provided one shot example so that images are acquired in a loop and access to the image data is given:

#include #include #include #include #include "ArgusHelpers.h" #include "CommonOptions.h" #include "nvbuf_utils.h" #include #include

#define EXIT_IF_NULL(val,msg)
{if (!val) {printf("%s\n",msg); return EXIT_FAILURE;}}
#define EXIT_IF_NOT_OK(val,msg)
{if (val!=Argus::STATUS_OK) {printf("%s\n",msg); return EXIT_FAILURE;}}

int main(int argc, char** argv)
{
char c[2] = {46, 0};
//ArgusSamples::CommonOptions options(basename(&(c[0])),
ArgusSamples::CommonOptions options(&(c[0]),
ArgusSamples::CommonOptions::Option_D_CameraDevice |
ArgusSamples::CommonOptions::Option_M_SensorMode);
if (!options.parse(argc, argv))
return EXIT_FAILURE;
if (options.requestedExit())
return EXIT_SUCCESS;

const uint64_t FIVE_SECONDS_IN_NANOSECONDS = 5000000000;

/*
 * Set up Argus API Framework, identify available camera devices, and create
 * a capture session for the first available device
 */

Argus::UniqueObj<Argus::CameraProvider> cameraProvider(Argus::CameraProvider::create());

Argus::ICameraProvider *iCameraProvider =
    Argus::interface_cast<Argus::ICameraProvider>(cameraProvider);
EXIT_IF_NULL(iCameraProvider, "Cannot get core camera provider interface");
printf("Argus Version: %s\n", iCameraProvider->getVersion().c_str());

Argus::CameraDevice *device = ArgusSamples::ArgusHelpers::getCameraDevice(
        cameraProvider.get(), options.cameraDeviceIndex());
Argus::ICameraProperties *iCameraProperties =
    Argus::interface_cast<Argus::ICameraProperties>(device);
if (!iCameraProperties)
{
    REPORT_ERROR("Failed to get ICameraProperties interface");
    return EXIT_FAILURE;
}

Argus::SensorMode* sensorMode = ArgusSamples::ArgusHelpers::getSensorMode(
        device, options.sensorModeIndex());
Argus::ISensorMode *iSensorMode =
    Argus::interface_cast<Argus::ISensorMode>(sensorMode);
if (!iSensorMode)
{
    REPORT_ERROR("Failed to get sensor mode interface");
    return EXIT_FAILURE;
}

printf("Capturing from device %d using sensor mode %d (%dx%d)\n",
       options.cameraDeviceIndex(), options.sensorModeIndex(),
       iSensorMode->getResolution().width(), iSensorMode->getResolution().height());

Argus::Status status;
Argus::UniqueObj<Argus::CaptureSession> captureSession(
    iCameraProvider->createCaptureSession(device, &status));
EXIT_IF_NOT_OK(status, "Failed to create capture session");

Argus::ICaptureSession *iSession =
    Argus::interface_cast<Argus::ICaptureSession>(captureSession);
EXIT_IF_NULL(iSession, "Cannot get Capture Session Interface");

/*
 * Creates the stream between the Argus camera image capturing
 * sub-system (producer) and the image acquisition code (consumer).  A consumer object is
 * created from the stream to be used to request the image frame.  A successfully submitted
 * capture request activates the stream's functionality to eventually make a frame available
 * for acquisition.
 */

Argus::UniqueObj<Argus::OutputStreamSettings> streamSettings(
    iSession->createOutputStreamSettings(Argus::STREAM_TYPE_EGL));

Argus::IEGLOutputStreamSettings *iEGLStreamSettings =
    Argus::interface_cast<Argus::IEGLOutputStreamSettings>(streamSettings);
EXIT_IF_NULL(iEGLStreamSettings, "Cannot get IEGLOutputStreamSettings Interface");
iEGLStreamSettings->setPixelFormat(Argus::PIXEL_FMT_YCbCr_420_888);
iEGLStreamSettings->setResolution(iSensorMode->getResolution());
iEGLStreamSettings->setMetadataEnable(true);

Argus::UniqueObj<Argus::OutputStream> stream(
    iSession->createOutputStream(streamSettings.get()));
EXIT_IF_NULL(stream, "Failed to create EGLOutputStream");

Argus::UniqueObj<EGLStream::FrameConsumer> consumer(
    EGLStream::FrameConsumer::create(stream.get()));

EGLStream::IFrameConsumer *iFrameConsumer =
    Argus::interface_cast<EGLStream::IFrameConsumer>(consumer);
EXIT_IF_NULL(iFrameConsumer, "Failed to initialize Consumer");

Argus::UniqueObj<Argus::Request> request(
    iSession->createRequest(Argus::CAPTURE_INTENT_STILL_CAPTURE));

Argus::IRequest *iRequest = Argus::interface_cast<Argus::IRequest>(request);
EXIT_IF_NULL(iRequest, "Failed to get capture request interface");

status = iRequest->enableOutputStream(stream.get());
EXIT_IF_NOT_OK(status, "Failed to enable stream in capture request");

Argus::ISourceSettings *iSourceSettings =
    Argus::interface_cast<Argus::ISourceSettings>(request);
EXIT_IF_NULL(iSourceSettings, "Failed to get source settings request interface");
iSourceSettings->setSensorMode(sensorMode);

// uint32_t requestId = iSession->capture(request.get());
// EXIT_IF_NULL(requestId, "Failed to submit capture request");

// submit repeat request
status = iSession->repeat(request.get());
EXIT_IF_NOT_OK(status, "Failed to start repeat capture request");

int fd = -1;
void *pdata = NULL;


timespec spec;
double t0, t;

// aquire 10 frames
for (int k=0; k<10; k++)
{
    clock_gettime(CLOCK_REALTIME, &spec);
    t0 = (double)spec.tv_sec + (double)spec.tv_nsec*1.0e-9;

    Argus::UniqueObj<EGLStream::Frame> frame(iFrameConsumer->acquireFrame(FIVE_SECONDS_IN_NANOSECONDS, &status));

    EGLStream::IFrame *iFrame = Argus::interface_cast< EGLStream::IFrame>(frame);
    // EXIT_IF_NULL(iFrame, "Failed to get IFrame interface");
    if (!iFrame)
    {
        printf("Failed to get IFrame interface"); 
        return EXIT_FAILURE;
    }
    
    EGLStream::Image *image = iFrame->getImage();
    EXIT_IF_NULL(image, "Failed to get Image from iFrame->getImage()");
    
    EGLStream::NV::IImageNativeBuffer *iNativeBuffer = Argus::interface_cast<EGLStream::NV::IImageNativeBuffer>(image);
    //EXIT_IF_NULL(iNativeBuffer, "IImageNativeBuffer not supported by Image.");
    if (!iNativeBuffer)
    {
        printf("IImageNativeBuffer not supported by Image."); 
        return EXIT_FAILURE;
    }

    if (fd < 0)
    {
        // <----------------- NvBufferColorFormat_ABGR32
        fd = iNativeBuffer->createNvBuffer(iSensorMode->getResolution(),NvBufferColorFormat_ABGR32, NvBufferLayout_Pitch);
        printf("fd = %d\n", fd);
    }
    
    NvBufferMemMap(fd, 0, NvBufferMem_Read, &pdata);
    NvBufferMemSyncForCpu(fd, 0, &pdata);

    clock_gettime(CLOCK_REALTIME, &spec);
    t = (double)spec.tv_sec + (double)spec.tv_nsec*1.0e-9;
    if (pdata)
    {
        printf("%d: pdata got something! (%fsec)\n", k, t-t0);
    }
    else
    {
        printf("%d: pdata still NULL! (%fsec)\n", k, t-t0);
    }
    pdata = NULL;
}

/* Stop the repeating request and wait for idle. */
iSession->stopRepeat();
iSession->waitForIdle();

NvBufferDestroy(fd);

// Shut down Argus.
cameraProvider.reset();

return EXIT_SUCCESS;

}

This code seems to work and from the time measurement it will give me about 20fps. But whenever I try to pull stuff out of the main function (ultimately I need to get rid of the main function for an SO file), I get different errors, depending on how I do the split up. It is always one of the EXIT_IF_NOT_OK or EXIT_IF_NULL macros that indicate, that a resource could not be allocated or a call to create an object crashes.
Here is one example, of how I tried to make the above code ready for an SO file:

#include "libArgusCam.h" #include #include #include #include #include "ArgusHelpers.h" #include "CommonOptions.h" #include "nvbuf_utils.h" #include #include "Event.h" #include #include #include

#define EXIT_IF_NULL(val,msg)
{if (!val) {printf("%s\n",msg); return retval;}}
#define EXIT_IF_NOT_OK(val,msg)
{if (val!=Argus::STATUS_OK) {printf("%s\n",msg); return retval;}}

bool bKeepThreadAlive = true;
int nXRes = -1;
int nYRes = -1;
int nSize = 0;
void* pData = NULL;
CEvent cEvent(false);
pthread_t nTID;

void* worker(void* V)
{
char c[2] = {46, 0};

void* retval = NULL;

ArgusSamples::CommonOptions options(basename(&(c[0])),
                                    ArgusSamples::CommonOptions::Option_D_CameraDevice |
                                    ArgusSamples::CommonOptions::Option_M_SensorMode);

const uint64_t FIVE_SECONDS_IN_NANOSECONDS = 5000000000;

/*
 * Set up Argus API Framework, identify available camera devices, and create
 * a capture session for the first available device
 */

Argus::UniqueObj<Argus::CameraProvider> cameraProvider(Argus::CameraProvider::create());
printf("CameraProvider\n");
Argus::ICameraProvider *iCameraProvider =
    Argus::interface_cast<Argus::ICameraProvider>(cameraProvider);
printf("iCameraProvider\n");
EXIT_IF_NULL(iCameraProvider, "Cannot get core camera provider interface");
//printf("Argus Version: %s\n", iCameraProvider->getVersion().c_str());


Argus::CameraDevice *device = ArgusSamples::ArgusHelpers::getCameraDevice(
        cameraProvider.get(), options.cameraDeviceIndex());
printf("CameraDvice\n");
Argus::ICameraProperties *iCameraProperties =
    Argus::interface_cast<Argus::ICameraProperties>(device);
    printf("iCameraProperties\n");
EXIT_IF_NULL(iCameraProperties, "Failed to get ICameraProperties interface")

Argus::SensorMode* sensorMode = ArgusSamples::ArgusHelpers::getSensorMode(
        device, options.sensorModeIndex());
printf("SensorMode\n");
Argus::ISensorMode *iSensorMode =
    Argus::interface_cast<Argus::ISensorMode>(sensorMode);
printf("iSensorMode\n");
EXIT_IF_NULL(sensorMode, "Failed to get sensor mode interface")

/*
printf("Capturing from device %d using sensor mode %d (%dx%d)\n",
       options.cameraDeviceIndex(), options.sensorModeIndex(),
       iSensorMode->getResolution().width(), iSensorMode->getResolution().height());
*/
nXRes = iSensorMode->getResolution().width();
nYRes = iSensorMode->getResolution().height();
nSize = nXRes * nYRes * 3;
printf("(%d,%d)\n", nXRes, nYRes);

Argus::Status status;
Argus::UniqueObj<Argus::CaptureSession> captureSession(iCameraProvider->createCaptureSession(device, &status));
printf("CaptureSession\n");
EXIT_IF_NOT_OK(status, "Failed to create capture session");

Argus::ICaptureSession *iSession =
    Argus::interface_cast<Argus::ICaptureSession>(captureSession);
printf("iCapturSession\n");
EXIT_IF_NULL(iSession, "Cannot get Capture Session Interface");

/*
 * Creates the stream between the Argus camera image capturing
 * sub-system (producer) and the image acquisition code (consumer).  A consumer object is
 * created from the stream to be used to request the image frame.  A successfully submitted
 * capture request activates the stream's functionality to eventually make a frame available
 * for acquisition.
 */

Argus::UniqueObj<Argus::OutputStreamSettings> streamSettings(
    iSession->createOutputStreamSettings(Argus::STREAM_TYPE_EGL));
printf("streamSettings\n");

Argus::IEGLOutputStreamSettings *iEGLStreamSettings =
    Argus::interface_cast<Argus::IEGLOutputStreamSettings>(streamSettings);
printf("iEGLStreamSettings\n");
EXIT_IF_NULL(iEGLStreamSettings, "Cannot get IEGLOutputStreamSettings Interface");
iEGLStreamSettings->setPixelFormat(Argus::PIXEL_FMT_YCbCr_420_888);
iEGLStreamSettings->setResolution(iSensorMode->getResolution());
iEGLStreamSettings->setMetadataEnable(true);

Argus::UniqueObj<Argus::OutputStream> stream(
    iSession->createOutputStream(streamSettings.get()));
printf("stream\n");
EXIT_IF_NULL(stream, "Failed to create EGLOutputStream");

Argus::UniqueObj<EGLStream::FrameConsumer> consumer(
    EGLStream::FrameConsumer::create(stream.get()));
printf("consumer\n");
EGLStream::IFrameConsumer *iFrameConsumer =
    Argus::interface_cast<EGLStream::IFrameConsumer>(consumer);
printf("iFrameConsumer\n");
EXIT_IF_NULL(iFrameConsumer, "Failed to initialize Consumer");

Argus::UniqueObj<Argus::Request> request(
    iSession->createRequest(Argus::CAPTURE_INTENT_STILL_CAPTURE));
printf("request\n");

Argus::IRequest *iRequest = Argus::interface_cast<Argus::IRequest>(request);
printf("iRequest\n");
EXIT_IF_NULL(iRequest, "Failed to get capture request interface");

status = iRequest->enableOutputStream(stream.get());
EXIT_IF_NOT_OK(status, "Failed to enable stream in capture request");

Argus::ISourceSettings *iSourceSettings =
    Argus::interface_cast<Argus::ISourceSettings>(request);
printf("iSourceSettings\n");
EXIT_IF_NULL(iSourceSettings, "Failed to get source settings request interface");
iSourceSettings->setSensorMode(sensorMode);

// submit repeat request
status = iSession->repeat(request.get());
EXIT_IF_NOT_OK(status, "Failed to start repeat capture request");

int fd = -1;
int count = 0;
pData = NULL;

printf("start loop\n");
while (bKeepThreadAlive)
{
    printf("%d            \r", ++count);
    Argus::UniqueObj<EGLStream::Frame> frame(iFrameConsumer->acquireFrame(FIVE_SECONDS_IN_NANOSECONDS, &status));

    EGLStream::IFrame *iFrame = Argus::interface_cast< EGLStream::IFrame>(frame);
    EXIT_IF_NULL(iFrame, "Failed to get IFrame interface");
    
    EGLStream::Image *image = iFrame->getImage();
    EXIT_IF_NULL(image, "Failed to get Image from iFrame->getImage()");
    
    EGLStream::NV::IImageNativeBuffer *iNativeBuffer = Argus::interface_cast<EGLStream::NV::IImageNativeBuffer>(image);
    EXIT_IF_NULL(iNativeBuffer, "IImageNativeBuffer not supported by Image.");
    
    if (fd < 0)
    {
        // <----------------- NvBufferColorFormat_ABGR32
        fd = iNativeBuffer->createNvBuffer(iSensorMode->getResolution(),NvBufferColorFormat_ABGR32, NvBufferLayout_Pitch);
    }
    
    // Maybe Mutex is necessary here to seafeguard access to pdata
    NvBufferMemMap(fd, 0, NvBufferMem_Read, &pData);
    NvBufferMemSyncForCpu(fd, 0, &pData);
    cEvent.Set();
}

/* Stop the repeating request and wait for idle. */
iSession->stopRepeat();
iSession->waitForIdle();

NvBufferDestroy(fd);

// Shut down Argus.
cameraProvider.reset();

return retval;

}

extern “C”
{
void getFrame(unsigned char* p)
{
cEvent.Wait(); // make sure new frame has arrived
// Maybe Mutex is necessary here to seafeguard access to pdata
memcpy(p, pData, nSize*sizeof(unsigned char));
cEvent.Reset();
}

void getResolution(int* x, int* y)
{
    while (nYRes < 0)
    {
        usleep(10000);
    }
    
   *x = nXRes;
   *y = nYRes;
}

void startCam()
{
    pthread_create(&nTID, NULL, worker, NULL);
}

void stopCam()
{
    bKeepThreadAlive = false;
    pthread_join(nTID, NULL);
    printf("Thread stopped!\n");
}

}

The above code displays the camera resolution correctly (line 78) and then crashes in line 81:

Argus::UniqueObj captureSession(iCameraProvider->createCaptureSession(device, &status));

Note: to keep the contents of this post limited I posted only relevant code. If somebody wants the rest of the code necessary to run it, like the header file for the above code or the code for the class CEvent, please let me know.

Any kind of help is greatly appreciated.

I can well understand that nobody wants to answer to this post. If I came across this post and had to answer, I would probably recommend the book “C++ for Beginners”. But let me tell you, I am a professional software engineer with decades worth of experience. I have literally done stuff like that a million times and never had a problem. I am really desperate for help here.

Thanks,
Markus