ARGUS API for 6 Cameras TX1

I am using argus api for multicamera capture which is eventually consumed by cuda buffers for processing.
Currently i have modified the denoise application which works fine but shows a very high increase in latency(observed visually) (of about 0.5 s ) when i increase the number of capture sessions. I would like to know if the current implementation as below is optimal. Also the frames from individual cameras are not in sync.

Thank You. Below is the code.

/*

  • Copyright © 2016, NVIDIA CORPORATION. All rights reserved.
  • Redistribution and use in source and binary forms, with or without
  • modification, are permitted provided that the following conditions
  • are met:
    • Redistributions of source code must retain the above copyright
  • notice, this list of conditions and the following disclaimer.
    • Redistributions in binary form must reproduce the above copyright
  • notice, this list of conditions and the following disclaimer in the
  • documentation and/or other materials provided with the distribution.
    • Neither the name of NVIDIA CORPORATION nor the names of its
  • contributors may be used to endorse or promote products derived
  • from this software without specific prior written permission.
  • THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS’’ AND ANY
  • EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  • IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  • PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  • CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  • EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  • PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  • PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  • OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  • (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  • OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    */

#include “Error.h”
#include “EGLGlobal.h”
#include “GLContext.h”
#include “Window.h”
#include “Thread.h”
#include “PreviewConsumer.h”

#include <Argus/Argus.h>
#include <EGLStream/EGLStream.h>

#include <unistd.h>
#include <stdlib.h>

using namespace Argus;

/*

  • This sample outputs capture requests to two streams, one of which has denoise algorithms
  • enabled while the other does not, and then renders them to a split-screen window.
    */

namespace ArgusSamples
{

#define NUM_SESSIONS 6

// Constants.
static const uint32_t CAPTURE_TIME = 10; // In seconds.
static const Size STREAM_SIZE (1920, 1080);
static const NormalizedRect SOURCE_CLIP_RECT (0.4f, 0.4f, 0.6f, 0.6f);

// Globals.
UniqueObj g_cameraProvider;
EGLDisplayHolder g_display;

// Debug print macros.
#define PRODUCER_PRINT(…) printf("PRODUCER: " VA_ARGS)

static bool execute()
{
// Initialize the window and EGL display.
Window &window = Window::getInstance();
window.setWindowRect(0, 0, STREAM_SIZE.width, STREAM_SIZE.height);
PROPAGATE_ERROR(g_display.initialize(window.getEGLNativeDisplay()));

// Initialize the Argus camera provider.
UniqueObj cameraProvider(CameraProvider::create());
ICameraProvider *iCameraProvider = interface_cast(cameraProvider);
if (!iCameraProvider)
ORIGINATE_ERROR(“Failed to get ICameraProvider interface”);

// Create a capture session using the first available device.
std::vector<CameraDevice*> cameraDevices;
if (iCameraProvider->getCameraDevices(&cameraDevices) != STATUS_OK)
ORIGINATE_ERROR(“Failed to get CameraDevices”);
if (cameraDevices.size() < NUM_SESSIONS)
ORIGINATE_ERROR(“Not enough CameraDevices available”);

UniqueObj captureSession[NUM_SESSIONS];
ICaptureSession *iCaptureSession[NUM_SESSIONS];

for(int i = 0 ; i < NUM_SESSIONS; i++)
{
captureSession[i] = UniqueObj(iCameraProvider->createCaptureSession(cameraDevices[i]));
iCaptureSession[i] = interface_cast(captureSession[i]);
if (!iCaptureSession[i])
ORIGINATE_ERROR("Failed to create CaptureSession ");
}

// Create six output streams for 4 previews .
PRODUCER_PRINT(“Creating 4 output streams\n”);

UniqueObj streamSettings[NUM_SESSIONS];
IOutputStreamSettings *iStreamSettings[NUM_SESSIONS];

for(int i = 0 ; i < NUM_SESSIONS ; i++)
{
streamSettings[i] = UniqueObj(iCaptureSession[i]->createOutputStreamSettings());
iStreamSettings[i] = interface_cast(streamSettings[i]);
if (!iStreamSettings[i])
ORIGINATE_ERROR(“Failed to create OutputStreamSettings”);

iStreamSettings[i]->setPixelFormat(PIXEL_FMT_YCbCr_420_888);
iStreamSettings[i]->setResolution(STREAM_SIZE);
iStreamSettings[i]->setEGLDisplay(g_display.get());
}

UniqueObj previewStream[NUM_SESSIONS];
IStream *iPreviewStream[NUM_SESSIONS];
for(int i = 0 ; i < NUM_SESSIONS ; i++)
{

previewStream[i] = UniqueObj(iCaptureSession[i]->createOutputStream(streamSettings[i].get()));
iPreviewStream[i] = interface_cast(previewStream[i]);
if (!iPreviewStream[i])
ORIGINATE_ERROR(“Failed to create preview stream”);

}

// Connect a PreviewConsumer to the streams to render a split-screen, side-by-side rendering.

PRODUCER_PRINT(“Launching consumer thread\n”);
std::vector eglStreams;

for(int i = 0 ; i < NUM_SESSIONS ; i++)
{
eglStreams.push_back(iPreviewStream[i]->getEGLStream());
}

PreviewConsumerThread consumerThread(g_display.get(), eglStreams,
PreviewConsumerThread::LAYOUT_SPLIT_VERTICAL,
false /* Sync stream frames */);

PROPAGATE_ERROR(consumerThread.initialize());

//consumerThread.setLineWidth(1);
//consumerThread.setLineColor(1.0f, 0.0f, 0.0f);

// Wait until the consumer is connected to the streams.
PROPAGATE_ERROR(consumerThread.waitRunning());

UniqueObj request[NUM_SESSIONS];

IRequest *iRequest[NUM_SESSIONS];
// Create capture request and enable output streams.

for(int i = 0 ; i < NUM_SESSIONS ; i++)
{
request[i] = UniqueObj(iCaptureSession[i]->createRequest());
iRequest[i] = interface_cast(request[i]);
if (!iRequest[i])
ORIGINATE_ERROR(“Failed to create Request”);

iRequest[i]->enableOutputStream(previewStream[i].get());

}

// Submit capture requests.
PRODUCER_PRINT(“Starting repeat capture requests.\n”);

for(int i = 0; i < NUM_SESSIONS ; i++)
{
if (iCaptureSession[i]->repeat(request[i].get()) != STATUS_OK)
ORIGINATE_ERROR(“Failed to start repeat capture request”);
}

// Wait for CAPTURE_TIME seconds.
PROPAGATE_ERROR(window.pollingSleep(CAPTURE_TIME));

// Stop the repeating request and wait for idle.
for(int i = 0 ; i < NUM_SESSIONS ; i++)
{
iCaptureSession[i]->stopRepeat();
iCaptureSession[i]->waitForIdle();

// Destroy the output streams and wait for the consumer thread to complete.
previewStream[i].reset();

}

PROPAGATE_ERROR(consumerThread.shutdown());

// Shut down Argus.
g_cameraProvider.reset();

// Shut down the window (destroys window’s EGLSurface).
window.shutdown();

// Cleanup the EGL display
PROPAGATE_ERROR(g_display.cleanup());

PRODUCER_PRINT(“Done – exiting.\n”);
return true;

}

}; // namespace ArgusSamples

int main(int argc, const char *argv)
{
if (!ArgusSamples::execute())
return EXIT_FAILURE;

return EXIT_SUCCESS;
}

hello tejas95,

just for confirmation.
before you adding denoise feature, did you see high latency from the preview images?
thanks

Hi, thanks for the reply.

I did not add the denoise feature. I modified the code and observed the before mentioned results. As i increase the number of capture sessions i observe an increase in latency. I would like to know how to preview multiple cameras in a single capture session.

Thanks

Also i tried capturing multiple cameras in a single session. It works well for two cameras but hangs for more than two cameras. Below is the code.

/*
     * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
     *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions
     * are met:
     *  * Redistributions of source code must retain the above copyright
     *    notice, this list of conditions and the following disclaimer.
     *  * Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in the
     *    documentation and/or other materials provided with the distribution.
     *  * Neither the name of NVIDIA CORPORATION nor the names of its
     *    contributors may be used to endorse or promote products derived
     *    from this software without specific prior written permission.
     *
     * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
     * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
     * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
     * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
     * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
     * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     */

    #include "Error.h"
    #include "EGLGlobal.h"
    #include "GLContext.h"
    #include "Window.h"
    #include "Thread.h"
    #include "PreviewConsumer.h"

    #include <Argus/Argus.h>
    #include <EGLStream/EGLStream.h>

    #include <unistd.h>
    #include <stdlib.h>

    using namespace Argus;

    /*
     * This sample outputs capture requests to two streams, one of which has denoise algorithms
     * enabled while the other does not, and then renders them to a split-screen window.
     */

    namespace ArgusSamples
    {
		
	
	#define NUM_SESSIONS 2
	
    // Constants.
    static const uint32_t       CAPTURE_TIME    = 10; // In seconds.
    static const Size           STREAM_SIZE      (640, 480);
    static const NormalizedRect SOURCE_CLIP_RECT (0.4f, 0.4f, 0.6f, 0.6f);

    // Globals.
    UniqueObj<CameraProvider> g_cameraProvider;
    EGLDisplayHolder g_display;

    // Debug print macros.
    #define PRODUCER_PRINT(...) printf("PRODUCER: " __VA_ARGS__)

    static bool execute()
    {
        // Initialize the window and EGL display.
        Window &window = Window::getInstance();
        window.setWindowRect(0, 0, STREAM_SIZE.width, STREAM_SIZE.height);
        PROPAGATE_ERROR(g_display.initialize(window.getEGLNativeDisplay()));

        // Initialize the Argus camera provider.
        UniqueObj<CameraProvider> cameraProvider(CameraProvider::create());
        ICameraProvider *iCameraProvider = interface_cast<ICameraProvider>(cameraProvider);
        if (!iCameraProvider)
            ORIGINATE_ERROR("Failed to get ICameraProvider interface");

        // Create a capture session using the first available device.
        std::vector<CameraDevice*> cameraDevices;
        if (iCameraProvider->getCameraDevices(&cameraDevices) != STATUS_OK)
            ORIGINATE_ERROR("Failed to get CameraDevices");
        if (cameraDevices.size() < NUM_SESSIONS)
            ORIGINATE_ERROR("Not enough CameraDevices available");

		
		
		UniqueObj<CaptureSession> captureSession;
		ICaptureSession *iCaptureSession;
		
		std::vector<CameraDevice*>cameras;
		for(int i = 0 ; i < NUM_SESSIONS ; i++)
			cameras.push_back(cameraDevices[i]);
			
		
        captureSession = UniqueObj<CaptureSession>(iCameraProvider->createCaptureSession(cameras));
		iCaptureSession = interface_cast<ICaptureSession>(captureSession);
			if (!iCaptureSession)
				ORIGINATE_ERROR("Failed to create CaptureSession ");
		


       
   

  
        // Create six output streams for 4 previews .
        PRODUCER_PRINT("Creating 4 output streams\n");
        
        UniqueObj<OutputStreamSettings> streamSettings;
        IOutputStreamSettings *iStreamSettings;
        
      
		streamSettings = UniqueObj<OutputStreamSettings>(iCaptureSession->createOutputStreamSettings());
		iStreamSettings = interface_cast<IOutputStreamSettings>(streamSettings);
		if (!iStreamSettings)
			ORIGINATE_ERROR("Failed to create OutputStreamSettings");
   
		iStreamSettings->setPixelFormat(PIXEL_FMT_YCbCr_420_888);
		iStreamSettings->setResolution(STREAM_SIZE);
		iStreamSettings->setEGLDisplay(g_display.get());
		
          
     
        
      
        
        UniqueObj<OutputStream> previewStream[NUM_SESSIONS];
        IStream *iPreviewStream[NUM_SESSIONS];
        for(int i = 0 ; i < NUM_SESSIONS ; i++)
        {
			iStreamSettings->setCameraDevice(cameras[i]);
			previewStream[i] = UniqueObj<OutputStream>(iCaptureSession->createOutputStream(streamSettings.get()));
            iPreviewStream[i] = interface_cast<IStream>(previewStream[i]);
			if (!iPreviewStream[i])
				ORIGINATE_ERROR("Failed to create preview stream");

		}
			
       
        



  
        // Connect a PreviewConsumer to the streams to render a split-screen, side-by-side rendering.
        
    
                                             
 	


     
		UniqueObj<Request> request;
		
		IRequest *iRequest;
        // Create capture request and enable output streams.
        
        
		request = UniqueObj<Request>(iCaptureSession->createRequest());
		iRequest = interface_cast<IRequest>(request);
		if (!iRequest)
			ORIGINATE_ERROR("Failed to create Request");
			
		for(int i = 0;  i < NUM_SESSIONS ; i++)
			iRequest->enableOutputStream(previewStream[i].get());
			
	
        
    
		    PRODUCER_PRINT("Launching consumer thread\n");
        std::vector<EGLStreamKHR> eglStreams;
        
        for(int i = 0 ; i < NUM_SESSIONS ; i++)
        {
			eglStreams.push_back(iPreviewStream[i]->getEGLStream());
		}
     
    
        
        PreviewConsumerThread consumerThread(g_display.get(), eglStreams);
      
		PROPAGATE_ERROR(consumerThread.initialize());
    
        //consumerThread.setLineWidth(1);
        //consumerThread.setLineColor(1.0f, 0.0f, 0.0f);

        // Wait until the consumer is connected to the streams.
        PROPAGATE_ERROR(consumerThread.waitRunning());

	



        

  
       

        // Submit capture requests.
        PRODUCER_PRINT("Starting repeat capture requests.\n");
        
       
        if (iCaptureSession->repeat(request.get()) != STATUS_OK)
            ORIGINATE_ERROR("Failed to start repeat capture request");
		
            
            


        // Wait for CAPTURE_TIME seconds.
        PROPAGATE_ERROR(window.pollingSleep(CAPTURE_TIME));

        // Stop the repeating request and wait for idle.
     
			iCaptureSession->stopRepeat();
			iCaptureSession->waitForIdle();
      
         for(int i = 0 ; i < NUM_SESSIONS ; i++)
        {

        // Destroy the output streams and wait for the consumer thread to complete.
			previewStream[i].reset();
			
			
		}
  
        
     
        PROPAGATE_ERROR(consumerThread.shutdown());

        // Shut down Argus.
        g_cameraProvider.reset();

        // Shut down the window (destroys window's EGLSurface).
        window.shutdown();

        // Cleanup the EGL display
        PROPAGATE_ERROR(g_display.cleanup());

        PRODUCER_PRINT("Done -- exiting.\n");

        return true;

    }

    }; // namespace ArgusSamples

    int main(int argc, const char *argv[])
    {
        if (!ArgusSamples::execute())
            return EXIT_FAILURE;

        return EXIT_SUCCESS;
    }

I get the following error when i change NUM_SESSIONS from 2 to 3

PRODUCER: Starting repeat capture requests.
SCF: Error BadParameter: Buffer index is too big (in src/components/CaptureContainerImpl.cpp, function assignTempBuffer(), line 101)
SCF: Error BadParameter: (propagating from src/components/stages/TempBufferAcquireStage.cpp, function doExecute(), line 64)
SCF: Error BadParameter: Buffer index is too big (in src/components/CaptureContainerImpl.cpp, function assignTempBuffer(), line 101)
SCF: Error BadParameter: (propagating from src/components/stages/TempBufferAcquireStage.cpp, function doExecute(), line 64)
SCF: Error BadParameter: Buffer index is too big (in src/components/CaptureContainerImpl.cpp, function assignTempBuffer(), line 101)
SCF: Error BadParameter: (propagating from src/components/stages/TempBufferAcquireStage.cpp, function doExecute(), line 64)
SCF: Error BadParameter: Buffer index is too big (in src/components/CaptureContainerImpl.cpp, function assignTempBuffer(), line 101)
SCF: Error BadParameter: (propagating from src/components/stages/TempBufferAcquireStage.cpp, function doExecute(), line 64)
SCF: Error BadParameter: Buffer index is too big (in src/components/CaptureContainerImpl.cpp, function assignTempBuffer(), line 101)
SCF: Error BadParameter: (propagating from src/components/stages/TempBufferAcquireStage.cpp, function doExecute(), line 64)

This issue has been solved. I had to change the sensor mode to 1920x1080.

hello tejas95,

just for double confirmation.
you can make 6 camera works well on TX1 after setting them to 1920x1080, right?

yes .