Argus: Timeout with three cameras

Hi,
in our application, we need to capture synchronized streams of three cameras. Therefore I added the three devices to a single capture session. This was working well when using L4T 24.2. Now I updated to L4T 28.1 and get some timeouts when running the three cameras. It seems that this only happens when I run them in the same capture session, because it works with three instances of argus_camera. I also did some tests with the syncSensor example in the ArgusSDK. The example was working with two cameras, so I extended the sample to work with three cameras as seen below:

/*
 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *  * Neither the name of NVIDIA CORPORATION nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <Argus/Argus.h>
#include <cuda.h>
#include <cudaEGL.h>

#include "CUDAHelper.h"
#include "EGLGlobal.h"
#include "Error.h"
#include "KLDistance.h"
#include "Thread.h"

// Histogram generation methods borrowed from cudaHistogram sample.
#include "../cudaHistogram/histogram.h"


using namespace Argus;

namespace ArgusSamples
{

/*
 * This sample opens a session with two sensors, it then using CUDA computes the histogram
 * of the sensors and computes a KL distance between the two histograms. A small value near
 * 0 indicates that the two images are alike. The processing of the images happens in the worker
 * thread of StereoDisparityConsumerThread. While the main app thread is used to drive the captures.
 */

// Constants.
static const uint32_t         CAPTURE_TIME  = 5; // In seconds.
static const Size2D<uint32_t> STREAM_SIZE(640, 480);

// Globals and derived constants.
UniqueObj<CameraProvider> g_cameraProvider;
EGLDisplayHolder g_display;

// Debug print macros.
#define PRODUCER_PRINT(...) printf("PRODUCER: " __VA_ARGS__)
#define CONSUMER_PRINT(...) printf("CONSUMER: " __VA_ARGS__)

/*******************************************************************************
 * Argus disparity class
 *   This class will analyze frames from 2 synchronized sensors and compute the
 *   KL distance between the two images. Large values of KL indicate a large disparity
 *   while a value of 0.0 indicates that the images are alike.
 ******************************************************************************/
class StereoDisparityConsumerThread : public Thread
{
public:
    explicit StereoDisparityConsumerThread(IStream *leftStream,
                                           IStream *rightStream,
                                           IStream *topStream)
                                         : m_leftStream(leftStream)
                                         , m_rightStream(rightStream)
                                         , m_topStream(topStream)
                                         , m_cudaContext(0)
                                         , m_cuStreamLeft(NULL)
                                         , m_cuStreamRight(NULL)
                                         , m_cuStreamTop(NULL)
    {
    }
    ~StereoDisparityConsumerThread()
    {
    }

private:
    /** @name Thread methods */
    /**@{*/
    virtual bool threadInitialize();
    virtual bool threadExecute();
    virtual bool threadShutdown();
    /**@}*/

    IStream      *m_leftStream;
    IStream      *m_rightStream;
    IStream      *m_topStream;
    CUcontext     m_cudaContext;
    CUeglStreamConnection m_cuStreamLeft;
    CUeglStreamConnection m_cuStreamRight;
    CUeglStreamConnection m_cuStreamTop;
};

/**
 * Utility class to acquire and process an EGLStream frame from a CUDA
 * consumer as long as the object is in scope.
 */
class ScopedCudaEGLStreamFrameAcquire
{
public:
    /**
     * Constructor blocks until a frame is acquired or an error occurs (ie. stream is
     * disconnected). Caller should check which condition was satisfied using hasValidFrame().
     */
    ScopedCudaEGLStreamFrameAcquire(CUeglStreamConnection& connection);

    /**
     * Destructor releases frame back to EGLStream.
     */
    ~ScopedCudaEGLStreamFrameAcquire();

    /**
     * Returns true if a frame was acquired (and is compatible with this consumer).
     */
    bool hasValidFrame() const;

    /**
     * Use CUDA to generate a histogram from the acquired frame.
     * @param[out] histogramData Output array for the histogram.
     * @param[out] time Time to generate histogram, in milliseconds.
     */
    bool generateHistogram(unsigned int histogramData[HISTOGRAM_BINS], float *time);

    /**
     * Returns the size (resolution) of the frame.
     */
    Size2D<uint32_t> getSize() const;

private:

    /**
     * Returns whether or not the frame format is supported.
     */
    bool checkFormat() const;

    CUeglStreamConnection& m_connection;
    CUstream m_stream;
    CUgraphicsResource m_resource;
    CUeglFrame m_frame;
};

bool StereoDisparityConsumerThread::threadInitialize()
{
    // Create CUDA and connect egl streams.
    PROPAGATE_ERROR(initCUDA(&m_cudaContext));

    CONSUMER_PRINT("Connecting CUDA consumer to left stream\n");
    CUresult cuResult = cuEGLStreamConsumerConnect(&m_cuStreamLeft, m_leftStream->getEGLStream());
    if (cuResult != CUDA_SUCCESS)
    {
        ORIGINATE_ERROR("Unable to connect CUDA to EGLStream as a consumer (CUresult %s)",
            getCudaErrorString(cuResult));
    }

    CONSUMER_PRINT("Connecting CUDA consumer to right stream\n");
    cuResult = cuEGLStreamConsumerConnect(&m_cuStreamRight, m_rightStream->getEGLStream());
    if (cuResult != CUDA_SUCCESS)
    {
        ORIGINATE_ERROR("Unable to connect CUDA to EGLStream as a consumer (CUresult %s)",
            getCudaErrorString(cuResult));
    }

    CONSUMER_PRINT("Connecting CUDA consumer to top stream\n");
    cuResult = cuEGLStreamConsumerConnect(&m_cuStreamTop, m_topStream->getEGLStream());
    if (cuResult != CUDA_SUCCESS)
    {
        ORIGINATE_ERROR("Unable to connect CUDA to EGLStream as a consumer (CUresult %s)",
            getCudaErrorString(cuResult));
    }
    return true;
}

bool StereoDisparityConsumerThread::threadExecute()
{
    CONSUMER_PRINT("Waiting for Argus producer to connect to left stream.\n");
    m_leftStream->waitUntilConnected();

    CONSUMER_PRINT("Waiting for Argus producer to connect to right stream.\n");
    m_rightStream->waitUntilConnected();

    CONSUMER_PRINT("Waiting for Argus producer to connect to top stream.\n");
    m_topStream->waitUntilConnected();

    CONSUMER_PRINT("Streams connected, processing frames.\n");
    unsigned int histogramLeft[HISTOGRAM_BINS];
    unsigned int histogramRight[HISTOGRAM_BINS];
    unsigned int histogramTop[HISTOGRAM_BINS];
    while (true)
    {
        ScopedCudaEGLStreamFrameAcquire left(m_cuStreamLeft);
        ScopedCudaEGLStreamFrameAcquire right(m_cuStreamRight);
        ScopedCudaEGLStreamFrameAcquire top(m_cuStreamTop);

        if (!left.hasValidFrame() || !right.hasValidFrame() || !top.hasValidFrame())
            break;

        // Calculate histograms.
        float time = 0.0f;
        if (left.generateHistogram(histogramLeft, &time) &&
            right.generateHistogram(histogramRight, &time) &&
            top.generateHistogram(histogramTop, &time))
        {
            // Calculate KL distance.
            float distance = 0.0f;
            Size2D<uint32_t> size = right.getSize();
            float dTime = computeKLDistance(histogramRight,
                                            histogramLeft,
                                            HISTOGRAM_BINS,
                                            size.width() * size.height(),
                                            &distance);
            CONSUMER_PRINT("KL distance of %6.3f with %5.2f ms computing histograms and "
                           "%5.2f ms spent computing distance\n",
                           distance, time, dTime);

            dTime = computeKLDistance(histogramRight,
                                      histogramTop,
                                      HISTOGRAM_BINS,
                                      size.width() * size.height(),
                                      &distance);
            CONSUMER_PRINT("KL distance of %6.3f with %5.2f ms computing histograms and "
                           "%5.2f ms spent computing distance\n",
                           distance, time, dTime);
        }
    }
    CONSUMER_PRINT("No more frames. Cleaning up.\n");

    PROPAGATE_ERROR(requestShutdown());

    return true;
}

bool StereoDisparityConsumerThread::threadShutdown()
{
    // Disconnect from the streams.
    cuEGLStreamConsumerDisconnect(&m_cuStreamLeft);
    cuEGLStreamConsumerDisconnect(&m_cuStreamRight);
    cuEGLStreamConsumerDisconnect(&m_cuStreamTop);

    PROPAGATE_ERROR(cleanupCUDA(&m_cudaContext));

    CONSUMER_PRINT("Done.\n");
    return true;
}

ScopedCudaEGLStreamFrameAcquire::ScopedCudaEGLStreamFrameAcquire(CUeglStreamConnection& connection)
    : m_connection(connection)
    , m_stream(NULL)
    , m_resource(0)
{
    CUresult r = cuEGLStreamConsumerAcquireFrame(&m_connection, &m_resource, &m_stream, -1);
    if (r == CUDA_SUCCESS)
    {
        cuGraphicsResourceGetMappedEglFrame(&m_frame, m_resource, 0, 0);
    }
}

ScopedCudaEGLStreamFrameAcquire::~ScopedCudaEGLStreamFrameAcquire()
{
    if (m_resource)
    {
        cuEGLStreamConsumerReleaseFrame(&m_connection, m_resource, &m_stream);
    }
}

bool ScopedCudaEGLStreamFrameAcquire::hasValidFrame() const
{
    return m_resource && checkFormat();
}

bool ScopedCudaEGLStreamFrameAcquire::generateHistogram(unsigned int histogramData[HISTOGRAM_BINS],
                                                        float *time)
{
    if (!hasValidFrame() || !histogramData || !time)
        ORIGINATE_ERROR("Invalid state or output parameters");

    // Create surface from luminance channel.
    CUDA_RESOURCE_DESC cudaResourceDesc;
    memset(&cudaResourceDesc, 0, sizeof(cudaResourceDesc));
    cudaResourceDesc.resType = CU_RESOURCE_TYPE_ARRAY;
    cudaResourceDesc.res.array.hArray = m_frame.frame.pArray[0];
    CUsurfObject cudaSurfObj = 0;
    CUresult cuResult = cuSurfObjectCreate(&cudaSurfObj, &cudaResourceDesc);
    if (cuResult != CUDA_SUCCESS)
    {
        ORIGINATE_ERROR("Unable to create the surface object (CUresult %s)",
                         getCudaErrorString(cuResult));
    }

    // Generated the histogram.
    *time += histogram(cudaSurfObj, m_frame.width, m_frame.height, histogramData);

    // Destroy surface.
    cuSurfObjectDestroy(cudaSurfObj);

    return true;
}

Size2D<uint32_t> ScopedCudaEGLStreamFrameAcquire::getSize() const
{
    if (hasValidFrame())
        return Size2D<uint32_t>(m_frame.width, m_frame.height);
    return Size2D<uint32_t>(0, 0);
}

bool ScopedCudaEGLStreamFrameAcquire::checkFormat() const
{
    if ((m_frame.eglColorFormat != CU_EGL_COLOR_FORMAT_YUV420_PLANAR) &&
        (m_frame.eglColorFormat != CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR) &&
        (m_frame.eglColorFormat != CU_EGL_COLOR_FORMAT_YUV422_PLANAR) &&
        (m_frame.eglColorFormat != CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR))
    {
        ORIGINATE_ERROR("Only YUV color formats are supported");
    }
    if (m_frame.cuFormat != CU_AD_FORMAT_UNSIGNED_INT8)
    {
        ORIGINATE_ERROR("Only 8-bit unsigned int formats are supported");
    }
    return true;
}

static bool execute()
{
    // Initialize EGL.
    PROPAGATE_ERROR(g_display.initialize());

    // Initialize the Argus camera provider.
    UniqueObj<CameraProvider> cameraProvider(CameraProvider::create());

    // Get the ICameraProvider interface from the global CameraProvider.
    ICameraProvider *iCameraProvider = interface_cast<ICameraProvider>(cameraProvider);
    if (!iCameraProvider)
        ORIGINATE_ERROR("Failed to get ICameraProvider interface");
    printf("Argus Version: %s\n", iCameraProvider->getVersion().c_str());

    // Get the camera devices.
    std::vector<CameraDevice*> cameraDevices;
    iCameraProvider->getCameraDevices(&cameraDevices);
    if (cameraDevices.size() < 3)
        ORIGINATE_ERROR("Must have at least 3 sensors available");

    std::vector <CameraDevice*> lrCameras;
    lrCameras.push_back(cameraDevices[0]); // Left Camera (the 1st camera will be used for AC)
    lrCameras.push_back(cameraDevices[1]); // Right Camera
    lrCameras.push_back(cameraDevices[2]); // Top Camera

    // Create the capture session, AutoControl will be based on what the 1st device sees.
    UniqueObj<CaptureSession> captureSession(iCameraProvider->createCaptureSession(lrCameras));
    ICaptureSession *iCaptureSession = interface_cast<ICaptureSession>(captureSession);
    if (!iCaptureSession)
        ORIGINATE_ERROR("Failed to get capture session interface");

    // Create stream settings object and set settings common to both streams.
    UniqueObj<OutputStreamSettings> streamSettings(iCaptureSession->createOutputStreamSettings());
    IOutputStreamSettings *iStreamSettings = interface_cast<IOutputStreamSettings>(streamSettings);
    if (!iStreamSettings)
        ORIGINATE_ERROR("Failed to create OutputStreamSettings");
    iStreamSettings->setPixelFormat(PIXEL_FMT_YCbCr_420_888);
    iStreamSettings->setResolution(STREAM_SIZE);
    iStreamSettings->setEGLDisplay(g_display.get());

    // Create egl streams
    PRODUCER_PRINT("Creating left stream.\n");
    iStreamSettings->setCameraDevice(lrCameras[0]);
    UniqueObj<OutputStream> streamLeft(iCaptureSession->createOutputStream(streamSettings.get()));
    IStream *iStreamLeft = interface_cast<IStream>(streamLeft);
    if (!iStreamLeft)
        ORIGINATE_ERROR("Failed to create left stream");

    PRODUCER_PRINT("Creating right stream.\n");
    iStreamSettings->setCameraDevice(lrCameras[1]);
    UniqueObj<OutputStream> streamRight(iCaptureSession->createOutputStream(streamSettings.get()));
    IStream *iStreamRight = interface_cast<IStream>(streamRight);
    if (!iStreamRight)
        ORIGINATE_ERROR("Failed to create right stream");

    PRODUCER_PRINT("Creating top stream.\n");
    iStreamSettings->setCameraDevice(lrCameras[2]);
    UniqueObj<OutputStream> streamTop(iCaptureSession->createOutputStream(streamSettings.get()));
    IStream *iStreamTop = interface_cast<IStream>(streamTop);
    if (!iStreamTop)
        ORIGINATE_ERROR("Failed to create right stream");

    PRODUCER_PRINT("Launching disparity checking consumer\n");
    StereoDisparityConsumerThread disparityConsumer(iStreamLeft, iStreamRight, iStreamTop);
    PROPAGATE_ERROR(disparityConsumer.initialize());
    PROPAGATE_ERROR(disparityConsumer.waitRunning());

    // Create a request
    UniqueObj<Request> request(iCaptureSession->createRequest());
    IRequest *iRequest = interface_cast<IRequest>(request);
    if (!iRequest)
        ORIGINATE_ERROR("Failed to create Request");

    // Enable both streams in the request.
    iRequest->enableOutputStream(streamLeft.get());
    iRequest->enableOutputStream(streamRight.get());
    iRequest->enableOutputStream(streamTop.get());

    // Submit capture for the specified time.
    PRODUCER_PRINT("Starting repeat capture requests.\n");
    if (iCaptureSession->repeat(request.get()) != STATUS_OK)
        ORIGINATE_ERROR("Failed to start repeat capture request for preview");
    sleep(CAPTURE_TIME);

    // Stop the capture requests and wait until they are complete.
    iCaptureSession->stopRepeat();
    iCaptureSession->waitForIdle();

    // Disconnect Argus producer from the EGLStreams (and unblock consumer acquire).
    PRODUCER_PRINT("Captures complete, disconnecting producer.\n");
    iStreamLeft->disconnect();
    iStreamRight->disconnect();
    iStreamTop->disconnect();

    // Wait for the consumer thread to complete.
    PROPAGATE_ERROR(disparityConsumer.shutdown());

    // Shut down Argus.
    cameraProvider.reset();

    // Cleanup the EGL display
    PROPAGATE_ERROR(g_display.cleanup());

    PRODUCER_PRINT("Done -- exiting.\n");
    return true;
}

}; // namespace ArgusSamples

int main(int argc, const char *argv[])
{
    printf("Executing Argus Sample: %s\n", basename(argv[0]));

    if (!ArgusSamples::execute())
        return EXIT_FAILURE;

    return EXIT_SUCCESS;
}

When running this sample, I also get some timeouts and the application hangs. This is the output of argus_daemon:

=== Listening for connections... ===
=== Connection 7F7BA261E0 established ===
NvPclHwGetModuleList: No module data found
NvPclHwGetModuleList: No module data found
PCLHW_DTParser
LoadOverridesFile: looking for override file [/Calib/camera_override.isp] 1/16LoadOverridesFile: looking for override file [/data/nvcam/settings/camera_overrides.isp] 2/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/camera_overrides.isp] 3/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/camera_overrides.isp] 4/16LoadOverridesFile: looking for override file [/data/nvcam/camera_overrides.isp] 5/16LoadOverridesFile: looking for override file [/data/nvcam/settings/imx185_center_liimx185.isp] 6/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/imx185_center_liimx185.isp] 7/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/imx185_center_liimx185.isp] 8/16---- imager: No override file found. ----
LoadOverridesFile: looking for override file [/Calib/camera_override.isp] 1/16LoadOverridesFile: looking for override file [/data/nvcam/settings/camera_overrides.isp] 2/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/camera_overrides.isp] 3/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/camera_overrides.isp] 4/16LoadOverridesFile: looking for override file [/data/nvcam/camera_overrides.isp] 5/16LoadOverridesFile: looking for override file [/data/nvcam/settings/imx185_front_liimx185.isp] 6/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/imx185_front_liimx185.isp] 7/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/imx185_front_liimx185.isp] 8/16---- imager: No override file found. ----
LoadOverridesFile: looking for override file [/Calib/camera_override.isp] 1/16LoadOverridesFile: looking for override file [/data/nvcam/settings/camera_overrides.isp] 2/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/camera_overrides.isp] 3/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/camera_overrides.isp] 4/16LoadOverridesFile: looking for override file [/data/nvcam/camera_overrides.isp] 5/16LoadOverridesFile: looking for override file [/data/nvcam/settings/imx185_bottom_liimx185.isp] 6/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/imx185_bottom_liimx185.isp] 7/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/imx185_bottom_liimx185.isp] 8/16---- imager: No override file found. ----
CameraProvider result: provider=0x7f75257ff0, shim=0x7f75248320, status=0, rpc status=1, size=9
Growing thread pool to 1 threads
PowerServiceCore:handleRequests: timePassed = 1113
PowerServiceCore:handleRequests: timePassed = 1102
SCF: Error Timeout:  (propagating from src/services/capture/CaptureServiceEvent.cpp, function wait(), line 59)
Error: Camera HwEvents wait, this may indicate a hardware timeout occured,abort current/incoming cc
launchCC abort cc 107 session 2
SCF: Error Timeout:  (propagating from src/api/Session.cpp, function capture(), line 830)
(Argus) Error Timeout: Failed to submit first capture request (propagating from src/api/CaptureSessionImpl.cpp, function submitCaptureRequests(), line 311)
(Argus) Error Timeout:  (propagating from src/api/CaptureSessionImpl.cpp, function threadFunction(), line 777)
PowerServiceCore:handleRequests: timePassed = 1012

So my question is: Is this a bug or do I something wrong? As noted before, the same code was running without issues in L4T 24.2.

@hesmar
Could you attached your binary file here for reproduce the problem.

Please find the binary here: https://box.tu-chemnitz.de/index.php/s/OhUjreGokduJVfG

@hesmar
There’s no problem on my board to run three camera. You may need have more investigated.

Executing Argus Sample: argus_syncsensor
Argus Version: 0.96.2 (multi-process)
PRODUCER: Creating left stream.
PRODUCER: Creating right stream.
PRODUCER: Creating top stream.
PRODUCER: Launching disparity checking consumer
Initializing CUDA
CONSUMER: Connecting CUDA consumer to left stream
CONSUMER: Connecting CUDA consumer to right stream
CONSUMER: Connecting CUDA consumer to top stream
CONSUMER: Waiting for Argus producer to connect to left stream.
PRODUCER: Starting repeat capture requests.
CONSUMER: Waiting for Argus producer to connect to right stream.
CONSUMER: Waiting for Argus producer to connect to top stream.
CONSUMER: Streams connected, processing frames.
CONSUMER: KL distance of -0.007 with  4.64 ms computing histograms and  1.71 ms spent computing distance
CONSUMER: KL distance of -0.005 with  4.64 ms computing histograms and  2.24 ms spent computing distance
CONSUMER: KL distance of -0.007 with  3.67 ms computing histograms and  1.46 ms spent computing distance
CONSUMER: KL distance of -0.005 with  3.67 ms computing histograms and  1.08 ms spent computing distance
CONSUMER: KL distance of -0.007 with  8.40 ms computing histograms and  1.75 ms spent computing distance
CONSUMER: KL distance of -0.005 with  8.40 ms computing histograms and  1.50 ms spent computing distance
CONSUMER: KL distance of  0.031 with  1.74 ms computing histograms and  1.21 ms spent computing distance
CONSUMER: KL distance of -0.087 with  1.74 ms computing histograms and  1.29 ms spent computing distance
CONSUMER: KL distance of  0.027 with  1.75 ms computing histograms and  1.62 ms spent computing distance
CONSUMER: KL distance of -0.083 with  1.75 ms computing histograms and  1.27 ms spent computing distance
CONSUMER: KL distance of  0.025 with 10.62 ms computing histograms and  5.46 ms spent computing distance
CONSUMER: KL distance of -0.063 with 10.62 ms computing histograms and  1.39 ms spent computing distance
CONSUMER: KL distance of  0.027 with  1.90 ms computing histograms and  1.18 ms spent computing distance
CONSUMER: KL distance of -0.068 with  1.90 ms computing histograms and  1.14 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.08 ms computing histograms and  1.55 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.08 ms computing histograms and  1.46 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.23 ms computing histograms and  1.32 ms spent computing distance
CONSUMER: KL distance of -0.074 with  1.23 ms computing histograms and  1.89 ms spent computing distance
CONSUMER: KL distance of  0.015 with  1.74 ms computing histograms and  1.24 ms spent computing distance
CONSUMER: KL distance of -0.038 with  1.74 ms computing histograms and  1.26 ms spent computing distance
CONSUMER: KL distance of  0.022 with  2.02 ms computing histograms and  3.12 ms spent computing distance
CONSUMER: KL distance of -0.058 with  2.02 ms computing histograms and  1.25 ms spent computing distance
CONSUMER: KL distance of  0.024 with  2.66 ms computing histograms and  0.90 ms spent computing distance
CONSUMER: KL distance of -0.061 with  2.66 ms computing histograms and  1.04 ms spent computing distance
CONSUMER: KL distance of  0.027 with  1.81 ms computing histograms and  1.32 ms spent computing distance
CONSUMER: KL distance of -0.064 with  1.81 ms computing histograms and  8.70 ms spent computing distance
CONSUMER: KL distance of  0.026 with  1.77 ms computing histograms and  1.47 ms spent computing distance
CONSUMER: KL distance of -0.067 with  1.77 ms computing histograms and  1.56 ms spent computing distance
CONSUMER: KL distance of  0.027 with  9.54 ms computing histograms and  7.03 ms spent computing distance
CONSUMER: KL distance of -0.067 with  9.54 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of  0.026 with  1.80 ms computing histograms and  0.95 ms spent computing distance
CONSUMER: KL distance of -0.069 with  1.80 ms computing histograms and  1.00 ms spent computing distance
CONSUMER: KL distance of  0.027 with  0.72 ms computing histograms and  1.24 ms spent computing distance
CONSUMER: KL distance of -0.068 with  0.72 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of  0.027 with  2.83 ms computing histograms and  2.67 ms spent computing distance
CONSUMER: KL distance of -0.069 with  2.83 ms computing histograms and  1.33 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.15 ms computing histograms and  1.27 ms spent computing distance
CONSUMER: KL distance of -0.070 with  1.15 ms computing histograms and  1.30 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.78 ms computing histograms and  1.56 ms spent computing distance
CONSUMER: KL distance of -0.069 with  1.78 ms computing histograms and  1.22 ms spent computing distance
CONSUMER: KL distance of  0.028 with  6.93 ms computing histograms and  1.04 ms spent computing distance
CONSUMER: KL distance of -0.070 with  6.93 ms computing histograms and  1.11 ms spent computing distance
CONSUMER: KL distance of  0.027 with  7.84 ms computing histograms and  1.99 ms spent computing distance
CONSUMER: KL distance of -0.071 with  7.84 ms computing histograms and  1.35 ms spent computing distance
CONSUMER: KL distance of  0.027 with  0.70 ms computing histograms and  1.17 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.70 ms computing histograms and  1.31 ms spent computing distance
CONSUMER: KL distance of  0.027 with  1.26 ms computing histograms and  1.25 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.26 ms computing histograms and  1.08 ms spent computing distance
CONSUMER: KL distance of  0.027 with  7.30 ms computing histograms and  2.30 ms spent computing distance
CONSUMER: KL distance of -0.071 with  7.30 ms computing histograms and  1.13 ms spent computing distance
CONSUMER: KL distance of  0.028 with  0.62 ms computing histograms and  0.97 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.62 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.18 ms computing histograms and  1.62 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.18 ms computing histograms and  1.36 ms spent computing distance
CONSUMER: KL distance of  0.027 with  3.60 ms computing histograms and  1.10 ms spent computing distance
CONSUMER: KL distance of -0.072 with  3.60 ms computing histograms and  1.99 ms spent computing distance
CONSUMER: KL distance of  0.027 with  1.73 ms computing histograms and  1.32 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.73 ms computing histograms and  1.11 ms spent computing distance
CONSUMER: KL distance of  0.029 with  4.00 ms computing histograms and  1.11 ms spent computing distance
CONSUMER: KL distance of -0.071 with  4.00 ms computing histograms and  1.28 ms spent computing distance
CONSUMER: KL distance of  0.029 with  3.09 ms computing histograms and  1.35 ms spent computing distance
CONSUMER: KL distance of -0.072 with  3.09 ms computing histograms and  1.31 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.74 ms computing histograms and  1.34 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.74 ms computing histograms and  1.22 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.72 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.72 ms computing histograms and  1.44 ms spent computing distance
CONSUMER: KL distance of  0.028 with  9.62 ms computing histograms and 12.91 ms spent computing distance
CONSUMER: KL distance of -0.072 with  9.62 ms computing histograms and  3.00 ms spent computing distance
CONSUMER: KL distance of  0.028 with  0.84 ms computing histograms and  3.51 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.84 ms computing histograms and  2.12 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.30 ms computing histograms and  2.18 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.30 ms computing histograms and  1.29 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.28 ms computing histograms and  1.83 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.28 ms computing histograms and  2.75 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.34 ms computing histograms and  1.48 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.34 ms computing histograms and  1.32 ms spent computing distance
CONSUMER: KL distance of  0.029 with  3.64 ms computing histograms and  1.17 ms spent computing distance
CONSUMER: KL distance of -0.072 with  3.64 ms computing histograms and  1.53 ms spent computing distance
CONSUMER: KL distance of  0.029 with  5.45 ms computing histograms and  8.47 ms spent computing distance
CONSUMER: KL distance of -0.073 with  5.45 ms computing histograms and 10.23 ms spent computing distance
CONSUMER: KL distance of  0.028 with 10.16 ms computing histograms and  1.43 ms spent computing distance
CONSUMER: KL distance of -0.072 with 10.16 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of  0.028 with  0.77 ms computing histograms and  1.14 ms spent computing distance
CONSUMER: KL distance of -0.073 with  0.77 ms computing histograms and  1.82 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.74 ms computing histograms and  1.21 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.74 ms computing histograms and  1.24 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.75 ms computing histograms and  1.14 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.75 ms computing histograms and  1.50 ms spent computing distance
CONSUMER: KL distance of  0.028 with  3.40 ms computing histograms and  9.98 ms spent computing distance
CONSUMER: KL distance of -0.073 with  3.40 ms computing histograms and  1.44 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.76 ms computing histograms and  1.31 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.76 ms computing histograms and  1.13 ms spent computing distance
CONSUMER: KL distance of  0.028 with  3.93 ms computing histograms and  9.24 ms spent computing distance
CONSUMER: KL distance of -0.073 with  3.93 ms computing histograms and  2.56 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.77 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.77 ms computing histograms and  1.31 ms spent computing distance
CONSUMER: KL distance of  0.029 with  6.26 ms computing histograms and  1.77 ms spent computing distance
CONSUMER: KL distance of -0.073 with  6.26 ms computing histograms and  1.47 ms spent computing distance
CONSUMER: KL distance of  0.030 with  3.32 ms computing histograms and  1.20 ms spent computing distance
CONSUMER: KL distance of -0.072 with  3.32 ms computing histograms and  1.34 ms spent computing distance
CONSUMER: KL distance of  0.030 with  3.76 ms computing histograms and  1.18 ms spent computing distance
CONSUMER: KL distance of -0.072 with  3.76 ms computing histograms and  1.53 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.77 ms computing histograms and  1.45 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.77 ms computing histograms and  1.24 ms spent computing distance
CONSUMER: KL distance of  0.030 with  5.68 ms computing histograms and  1.58 ms spent computing distance
CONSUMER: KL distance of -0.073 with  5.68 ms computing histograms and  1.27 ms spent computing distance
CONSUMER: KL distance of  0.028 with  5.69 ms computing histograms and  4.86 ms spent computing distance
CONSUMER: KL distance of -0.073 with  5.69 ms computing histograms and  1.18 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.24 ms computing histograms and  1.41 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.24 ms computing histograms and  1.97 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.91 ms computing histograms and  1.47 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.91 ms computing histograms and  1.40 ms spent computing distance
CONSUMER: KL distance of  0.029 with  5.42 ms computing histograms and  1.45 ms spent computing distance
CONSUMER: KL distance of -0.073 with  5.42 ms computing histograms and  2.48 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.63 ms computing histograms and  0.99 ms spent computing distance
CONSUMER: KL distance of -0.073 with  0.63 ms computing histograms and  1.40 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.17 ms computing histograms and  1.09 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.17 ms computing histograms and  1.41 ms spent computing distance
CONSUMER: KL distance of  0.028 with  5.45 ms computing histograms and  1.50 ms spent computing distance
CONSUMER: KL distance of -0.073 with  5.45 ms computing histograms and  1.19 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.63 ms computing histograms and  0.93 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.63 ms computing histograms and  1.20 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.66 ms computing histograms and  1.37 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.66 ms computing histograms and  1.30 ms spent computing distance
CONSUMER: KL distance of  0.029 with  3.50 ms computing histograms and  1.14 ms spent computing distance
CONSUMER: KL distance of -0.073 with  3.50 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.72 ms computing histograms and  1.29 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.72 ms computing histograms and  8.67 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.74 ms computing histograms and  1.65 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.74 ms computing histograms and  1.14 ms spent computing distance
CONSUMER: KL distance of  0.030 with 10.29 ms computing histograms and  1.94 ms spent computing distance
CONSUMER: KL distance of -0.073 with 10.29 ms computing histograms and  3.04 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.72 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.72 ms computing histograms and  1.70 ms spent computing distance
CONSUMER: KL distance of  0.031 with  4.05 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of -0.071 with  4.05 ms computing histograms and  1.10 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.95 ms computing histograms and  1.19 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.95 ms computing histograms and  1.33 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.72 ms computing histograms and  1.38 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.72 ms computing histograms and  1.11 ms spent computing distance
CONSUMER: KL distance of  0.030 with  2.01 ms computing histograms and  1.25 ms spent computing distance
CONSUMER: KL distance of -0.072 with  2.01 ms computing histograms and  1.46 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.95 ms computing histograms and  3.52 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.95 ms computing histograms and  5.57 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.87 ms computing histograms and  1.51 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.87 ms computing histograms and  1.66 ms spent computing distance
CONSUMER: KL distance of  0.030 with  3.74 ms computing histograms and  1.26 ms spent computing distance
CONSUMER: KL distance of -0.072 with  3.74 ms computing histograms and  1.45 ms spent computing distance
CONSUMER: KL distance of  0.028 with  1.22 ms computing histograms and  1.01 ms spent computing distance
CONSUMER: KL distance of -0.073 with  1.22 ms computing histograms and  1.34 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.54 ms computing histograms and  1.23 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.54 ms computing histograms and  1.88 ms spent computing distance
CONSUMER: KL distance of  0.030 with  2.44 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of -0.072 with  2.44 ms computing histograms and  1.10 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.63 ms computing histograms and  1.09 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.63 ms computing histograms and  1.23 ms spent computing distance
CONSUMER: KL distance of  0.031 with  0.67 ms computing histograms and  1.09 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.67 ms computing histograms and  1.30 ms spent computing distance
CONSUMER: KL distance of  0.029 with  2.41 ms computing histograms and  2.05 ms spent computing distance
CONSUMER: KL distance of -0.072 with  2.41 ms computing histograms and  1.42 ms spent computing distance
CONSUMER: KL distance of  0.030 with  2.64 ms computing histograms and  2.78 ms spent computing distance
CONSUMER: KL distance of -0.071 with  2.64 ms computing histograms and  1.43 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.94 ms computing histograms and  0.95 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.94 ms computing histograms and  1.60 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.20 ms computing histograms and  1.66 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.20 ms computing histograms and  1.39 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.81 ms computing histograms and  1.49 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.81 ms computing histograms and  1.37 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.75 ms computing histograms and  9.02 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.75 ms computing histograms and  8.48 ms spent computing distance
CONSUMER: KL distance of  0.029 with  9.20 ms computing histograms and  1.33 ms spent computing distance
CONSUMER: KL distance of -0.071 with  9.20 ms computing histograms and  1.51 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.74 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.74 ms computing histograms and  1.17 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.74 ms computing histograms and  1.19 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.74 ms computing histograms and  1.19 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.54 ms computing histograms and  1.59 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.54 ms computing histograms and  1.92 ms spent computing distance
CONSUMER: KL distance of  0.031 with  1.93 ms computing histograms and  9.00 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.93 ms computing histograms and  7.73 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.80 ms computing histograms and  1.38 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.80 ms computing histograms and  1.60 ms spent computing distance
CONSUMER: KL distance of  0.030 with  9.55 ms computing histograms and 10.40 ms spent computing distance
CONSUMER: KL distance of -0.071 with  9.55 ms computing histograms and  1.56 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.82 ms computing histograms and  1.07 ms spent computing distance
CONSUMER: KL distance of -0.070 with  0.82 ms computing histograms and  3.77 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.76 ms computing histograms and  1.38 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.76 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.96 ms computing histograms and  1.28 ms spent computing distance
CONSUMER: KL distance of -0.072 with  0.96 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of  0.029 with  1.23 ms computing histograms and  1.32 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.23 ms computing histograms and  1.30 ms spent computing distance
CONSUMER: KL distance of  0.030 with  5.24 ms computing histograms and  1.22 ms spent computing distance
CONSUMER: KL distance of -0.071 with  5.24 ms computing histograms and  1.47 ms spent computing distance
CONSUMER: KL distance of  0.031 with  1.77 ms computing histograms and  1.21 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.77 ms computing histograms and  1.54 ms spent computing distance
CONSUMER: KL distance of  0.029 with 11.77 ms computing histograms and  1.84 ms spent computing distance
CONSUMER: KL distance of -0.071 with 11.77 ms computing histograms and  1.29 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.75 ms computing histograms and  1.39 ms spent computing distance
CONSUMER: KL distance of -0.070 with  0.75 ms computing histograms and  1.20 ms spent computing distance
CONSUMER: KL distance of  0.031 with  0.78 ms computing histograms and  1.37 ms spent computing distance
CONSUMER: KL distance of -0.070 with  0.78 ms computing histograms and  1.50 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.75 ms computing histograms and  1.17 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.75 ms computing histograms and  1.32 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.80 ms computing histograms and  7.69 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.80 ms computing histograms and  5.78 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.98 ms computing histograms and  1.44 ms spent computing distance
CONSUMER: KL distance of -0.072 with  1.98 ms computing histograms and  1.90 ms spent computing distance
CONSUMER: KL distance of  0.031 with 10.27 ms computing histograms and 11.37 ms spent computing distance
CONSUMER: KL distance of -0.071 with 10.27 ms computing histograms and  1.28 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.71 ms computing histograms and  1.71 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.71 ms computing histograms and  1.21 ms spent computing distance
CONSUMER: KL distance of  0.029 with  2.57 ms computing histograms and  2.24 ms spent computing distance
CONSUMER: KL distance of -0.071 with  2.57 ms computing histograms and  2.07 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.20 ms computing histograms and  1.09 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.20 ms computing histograms and  6.88 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.24 ms computing histograms and  1.23 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.24 ms computing histograms and  1.41 ms spent computing distance
CONSUMER: KL distance of  0.030 with  3.87 ms computing histograms and  1.47 ms spent computing distance
CONSUMER: KL distance of -0.072 with  3.87 ms computing histograms and  1.51 ms spent computing distance
CONSUMER: KL distance of  0.030 with  3.30 ms computing histograms and  6.97 ms spent computing distance
CONSUMER: KL distance of -0.071 with  3.30 ms computing histograms and  7.40 ms spent computing distance
CONSUMER: KL distance of  0.030 with  7.30 ms computing histograms and  3.29 ms spent computing distance
CONSUMER: KL distance of -0.071 with  7.30 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of  0.031 with  0.70 ms computing histograms and  1.10 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.70 ms computing histograms and  1.41 ms spent computing distance
CONSUMER: KL distance of  0.029 with  0.69 ms computing histograms and  1.10 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.69 ms computing histograms and  1.48 ms spent computing distance
CONSUMER: KL distance of  0.030 with  3.51 ms computing histograms and  1.40 ms spent computing distance
CONSUMER: KL distance of -0.071 with  3.51 ms computing histograms and  1.42 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.64 ms computing histograms and  5.35 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.64 ms computing histograms and  7.83 ms spent computing distance
CONSUMER: KL distance of  0.031 with  2.09 ms computing histograms and  1.52 ms spent computing distance
CONSUMER: KL distance of -0.070 with  2.09 ms computing histograms and  1.55 ms spent computing distance
CONSUMER: KL distance of  0.031 with  9.49 ms computing histograms and 10.89 ms spent computing distance
CONSUMER: KL distance of -0.071 with  9.49 ms computing histograms and  1.54 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.95 ms computing histograms and  1.09 ms spent computing distance
CONSUMER: KL distance of -0.071 with  0.95 ms computing histograms and  1.10 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.13 ms computing histograms and  3.01 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.13 ms computing histograms and  4.15 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.20 ms computing histograms and  1.51 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.20 ms computing histograms and  1.25 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.22 ms computing histograms and  1.65 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.22 ms computing histograms and  1.31 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.75 ms computing histograms and  1.18 ms spent computing distance
CONSUMER: KL distance of -0.070 with  1.75 ms computing histograms and  1.20 ms spent computing distance
CONSUMER: KL distance of  0.031 with  2.18 ms computing histograms and  9.80 ms spent computing distance
CONSUMER: KL distance of -0.071 with  2.18 ms computing histograms and  8.51 ms spent computing distance
CONSUMER: KL distance of  0.031 with  8.28 ms computing histograms and  1.21 ms spent computing distance
CONSUMER: KL distance of -0.070 with  8.28 ms computing histograms and  1.37 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.73 ms computing histograms and  1.63 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.73 ms computing histograms and  1.43 ms spent computing distance
CONSUMER: KL distance of  0.030 with  1.73 ms computing histograms and  1.27 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.73 ms computing histograms and  1.52 ms spent computing distance
CONSUMER: KL distance of  0.031 with 10.26 ms computing histograms and  1.22 ms spent computing distance
CONSUMER: KL distance of -0.070 with 10.26 ms computing histograms and  1.40 ms spent computing distance
CONSUMER: KL distance of  0.032 with  0.76 ms computing histograms and  1.09 ms spent computing distance
CONSUMER: KL distance of -0.070 with  0.76 ms computing histograms and  1.15 ms spent computing distance
CONSUMER: KL distance of  0.030 with  0.75 ms computing histograms and  1.80 ms spent computing distance
CONSUMER: KL distance of -0.070 with  0.75 ms computing histograms and  1.53 ms spent computing distance
CONSUMER: KL distance of  0.031 with  0.94 ms computing histograms and  1.17 ms spent computing distance
CONSUMER: KL distance of -0.070 with  0.94 ms computing histograms and  1.17 ms spent computing distance
CONSUMER: KL distance of  0.031 with  1.78 ms computing histograms and  1.63 ms spent computing distance
CONSUMER: KL distance of -0.071 with  1.78 ms computing histograms and  1.67 ms spent computing distance
CONSUMER: KL distance of  0.031 with  1.57 ms computing histograms and  1.45 ms spent computing distance
CONSUMER: KL distance of -0.070 with  1.57 ms computing histograms and  1.45 ms spent computing distance
PRODUCER: Captures complete, disconnecting producer.
CONSUMER: KL distance of  0.031 with  0.70 ms computing histograms and  2.93 ms spent computing distance
CONSUMER: KL distance of -0.070 with  0.70 ms computing histograms and  1.92 ms spent computing distance
CONSUMER: No more frames. Cleaning up.
CONSUMER: Done.
PRODUCER: Done -- exiting.

@ShaneCCC:
Which cameras do you use? We use the imx185 camera modules from Leopard Imaging. I tried it with the Nvidia Devkit as well as with the LI-TX1-CB carrier board from Leopard. On both platforms I get these timeouts. When I run the example application from #3 on a TX2 everything works well. So it seems to be an issue with the TX1 only.
Do you have any idea what I could do for further debugging?

@hesmar
Could you try to boost the system as performance by jetson_clocks.sh first.

Running jetson_clocks.sh before did not help.

Regardless of running jetson_clocks.sh or not, it seems that the software receives a small number of frames before getting the timeout. Please see the application output:

Executing Argus Sample: argus_syncsensor
Argus Version: 0.96.2 (multi-process)
PRODUCER: Creating left stream.
PRODUCER: Creating right stream.
PRODUCER: Creating top stream.
PRODUCER: Launching disparity checking consumer
Initializing CUDA
CONSUMER: Connecting CUDA consumer to left stream
CONSUMER: Connecting CUDA consumer to right stream
CONSUMER: Connecting CUDA consumer to top stream
CONSUMER: Waiting for Argus producer to connect to left stream.
PRODUCER: Starting repeat capture requests.
CONSUMER: Waiting for Argus producer to connect to right stream.
CONSUMER: Waiting for Argus producer to connect to top stream.
CONSUMER: Streams connected, processing frames.
CONSUMER: KL distance of -0.003 with  3.13 ms computing histograms and  0.97 ms spent computing distance
CONSUMER: KL distance of -0.001 with  3.13 ms computing histograms and  1.02 ms spent computing distance
CONSUMER: KL distance of -0.004 with  6.08 ms computing histograms and  2.39 ms spent computing distance
CONSUMER: KL distance of -0.001 with  6.08 ms computing histograms and  1.30 ms spent computing distance
CONSUMER: KL distance of -0.004 with  1.91 ms computing histograms and  1.12 ms spent computing distance
CONSUMER: KL distance of -0.001 with  1.91 ms computing histograms and  1.13 ms spent computing distance
CONSUMER: KL distance of -0.004 with  1.75 ms computing histograms and  1.16 ms spent computing distance
CONSUMER: KL distance of -0.001 with  1.75 ms computing histograms and  1.26 ms spent computing distance
CONSUMER: KL distance of -0.004 with  3.23 ms computing histograms and  1.13 ms spent computing distance
CONSUMER: KL distance of -0.001 with  3.23 ms computing histograms and  1.15 ms spent computing distance
(Argus) Error Timeout:  (propagating from src/rpc/socket/client/ClientSocketManager.cpp, function send(), line 132)
(Argus) Error Timeout:  (propagating from src/rpc/socket/client/SocketClientDispatch.cpp, function dispatch(), line 101)
Segmentation fault

Could you clear build the APP by the “cmake -DDISABLE_MULTIPROCESS=ON …” and before launch the APP please kill argus_daemon first and do not start argus_daemon to run the syncsensor APP.

Building and running the app with multiprocess disabled also did not help.

@hesmar
Does any timeout message from the kernel message?
What’s the resolution for this sensor. Could you try a small sensor mode.

This is the output from dmesg when running the app:

[  192.320450] tegra_mipi_cal 700e3000.mipical: Mipi cal timeout,val:5bb1, lanes:300000
[  192.328248] tegra_mipi_cal 700e3000.mipical: MIPI_CAL_CTRL                  0x00 0x2a000000
[  192.336596] tegra_mipi_cal 700e3000.mipical: CIL_MIPI_CAL_STATUS            0x08 0x00005bb1
[  192.345048] tegra_mipi_cal 700e3000.mipical: CIL_MIPI_CAL_STATUS_2          0x0c 0x00000000
[  192.353396] tegra_mipi_cal 700e3000.mipical: CILA_MIPI_CAL_CONFIG           0x14 0x00200000
[  192.361785] tegra_mipi_cal 700e3000.mipical: CILB_MIPI_CAL_CONFIG           0x18 0x00200000
[  192.370134] tegra_mipi_cal 700e3000.mipical: CILC_MIPI_CAL_CONFIG           0x1c 0x00000000
[  192.378570] tegra_mipi_cal 700e3000.mipical: CILD_MIPI_CAL_CONFIG           0x20 0x00000000
[  192.386918] tegra_mipi_cal 700e3000.mipical: CILE_MIPI_CAL_CONFIG           0x24 0x00000000
[  192.395309] tegra_mipi_cal 700e3000.mipical: CILF_MIPI_CAL_CONFIG           0x28 0x00000000
[  192.403653] tegra_mipi_cal 700e3000.mipical: DSIA_MIPI_CAL_CONFIG           0x38 0x00000000
[  192.412064] tegra_mipi_cal 700e3000.mipical: DSIB_MIPI_CAL_CONFIG           0x3c 0x00000000
[  192.420410] tegra_mipi_cal 700e3000.mipical: DSIC_MIPI_CAL_CONFIG           0x40 0x00000000
[  192.428791] tegra_mipi_cal 700e3000.mipical: DSID_MIPI_CAL_CONFIG           0x44 0x00000000
[  192.437156] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG0             0x58 0x00000000
[  192.445570] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG1             0x5c 0x00000000
[  192.454184] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG2             0x60 0x00000000
[  192.462780] tegra_mipi_cal 700e3000.mipical: DSIA_MIPI_CAL_CONFIG_2         0x64 0x00000000
[  192.471367] tegra_mipi_cal 700e3000.mipical: DSIB_MIPI_CAL_CONFIG_2         0x68 0x00000000
[  192.479770] tegra_mipi_cal 700e3000.mipical: DSIC_MIPI_CAL_CONFIG_2         0x70 0x00000000
[  192.488186] tegra_mipi_cal 700e3000.mipical: DSID_MIPI_CAL_CONFIG_2         0x74 0x00000000
[  193.392525] tegra_mipi_cal 700e3000.mipical: Mipi cal timeout,val:5bb1, lanes:c00000
[  193.400312] tegra_mipi_cal 700e3000.mipical: MIPI_CAL_CTRL                  0x00 0x2a000000
[  193.408657] tegra_mipi_cal 700e3000.mipical: CIL_MIPI_CAL_STATUS            0x08 0x00005bb1
[  193.417005] tegra_mipi_cal 700e3000.mipical: CIL_MIPI_CAL_STATUS_2          0x0c 0x00000000
[  193.425351] tegra_mipi_cal 700e3000.mipical: CILA_MIPI_CAL_CONFIG           0x14 0x00000000
[  193.433730] tegra_mipi_cal 700e3000.mipical: CILB_MIPI_CAL_CONFIG           0x18 0x00000000
[  193.442077] tegra_mipi_cal 700e3000.mipical: CILC_MIPI_CAL_CONFIG           0x1c 0x00200000
[  193.450421] tegra_mipi_cal 700e3000.mipical: CILD_MIPI_CAL_CONFIG           0x20 0x00200000
[  193.458786] tegra_mipi_cal 700e3000.mipical: CILE_MIPI_CAL_CONFIG           0x24 0x00000000
[  193.467164] tegra_mipi_cal 700e3000.mipical: CILF_MIPI_CAL_CONFIG           0x28 0x00000000
[  193.475509] tegra_mipi_cal 700e3000.mipical: DSIA_MIPI_CAL_CONFIG           0x38 0x00000000
[  193.483850] tegra_mipi_cal 700e3000.mipical: DSIB_MIPI_CAL_CONFIG           0x3c 0x00000000
[  193.492194] tegra_mipi_cal 700e3000.mipical: DSIC_MIPI_CAL_CONFIG           0x40 0x00000000
[  193.500562] tegra_mipi_cal 700e3000.mipical: DSID_MIPI_CAL_CONFIG           0x44 0x00000000
[  193.508906] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG0             0x58 0x00000000
[  193.517265] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG1             0x5c 0x00000000
[  193.525609] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG2             0x60 0x00000000
[  193.533989] tegra_mipi_cal 700e3000.mipical: DSIA_MIPI_CAL_CONFIG_2         0x64 0x00000000
[  193.542331] tegra_mipi_cal 700e3000.mipical: DSIB_MIPI_CAL_CONFIG_2         0x68 0x00000000
[  193.550671] tegra_mipi_cal 700e3000.mipical: DSIC_MIPI_CAL_CONFIG_2         0x70 0x00000000
[  193.559040] tegra_mipi_cal 700e3000.mipical: DSID_MIPI_CAL_CONFIG_2         0x74 0x00000000
[  194.460521] tegra_mipi_cal 700e3000.mipical: Mipi cal timeout,val:6bb1, lanes:3000000
[  194.468354] tegra_mipi_cal 700e3000.mipical: MIPI_CAL_CTRL                  0x00 0x2a000000
[  194.476741] tegra_mipi_cal 700e3000.mipical: CIL_MIPI_CAL_STATUS            0x08 0x00006bb1
[  194.485100] tegra_mipi_cal 700e3000.mipical: CIL_MIPI_CAL_STATUS_2          0x0c 0x00000000
[  194.493501] tegra_mipi_cal 700e3000.mipical: CILA_MIPI_CAL_CONFIG           0x14 0x00000000
[  194.501891] tegra_mipi_cal 700e3000.mipical: CILB_MIPI_CAL_CONFIG           0x18 0x00000000
[  194.510275] tegra_mipi_cal 700e3000.mipical: CILC_MIPI_CAL_CONFIG           0x1c 0x00000000
[  194.518703] tegra_mipi_cal 700e3000.mipical: CILD_MIPI_CAL_CONFIG           0x20 0x00000000
[  194.527108] tegra_mipi_cal 700e3000.mipical: CILE_MIPI_CAL_CONFIG           0x24 0x00200000
[  194.535481] tegra_mipi_cal 700e3000.mipical: CILF_MIPI_CAL_CONFIG           0x28 0x00200000
[  194.543849] tegra_mipi_cal 700e3000.mipical: DSIA_MIPI_CAL_CONFIG           0x38 0x00000000
[  194.552239] tegra_mipi_cal 700e3000.mipical: DSIB_MIPI_CAL_CONFIG           0x3c 0x00000000
[  194.560583] tegra_mipi_cal 700e3000.mipical: DSIC_MIPI_CAL_CONFIG           0x40 0x00000000
[  194.568927] tegra_mipi_cal 700e3000.mipical: DSID_MIPI_CAL_CONFIG           0x44 0x00000000
[  194.577270] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG0             0x58 0x00000000
[  194.585637] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG1             0x5c 0x00000000
[  194.593999] tegra_mipi_cal 700e3000.mipical: MIPI_BIAS_PAD_CFG2             0x60 0x00000000
[  194.602340] tegra_mipi_cal 700e3000.mipical: DSIA_MIPI_CAL_CONFIG_2         0x64 0x00000000
[  194.610681] tegra_mipi_cal 700e3000.mipical: DSIB_MIPI_CAL_CONFIG_2         0x68 0x00000000
[  194.619050] tegra_mipi_cal 700e3000.mipical: DSIC_MIPI_CAL_CONFIG_2         0x70 0x00000000
[  194.627391] tegra_mipi_cal 700e3000.mipical: DSID_MIPI_CAL_CONFIG_2         0x74 0x00000000

The resolution is 1920x1080. Currently this is the only supported resolution in the driver. I could try to adapt the driver but it would take some time. Therefore I would prefer to try an other approach to resolve the issue if you have one.

I encountered a similar issue on the TX2 now. There, the timeouts are random. Sometimes it starts without any issue, but sometimes argus gives me timeout after starting the application.Also setting Nvpmodel to MAX-N and using jetson_clocks does not help. I am not sure, if reason of the timeouts is the same like on the TX1. The output of dmesg looks a little bit different:

[  653.878679] argus_syncsenso[1711]: unhandled level 2 translation fault (11) at 0x00000004, esr 0x92000006
[  653.888293] pgd = ffffffc1c35e4000
[  653.891703] [00000004] *pgd=00000002182fc003, *pud=00000002182fc003, *pmd=0000000000000000

[  653.901507] CPU: 0 PID: 1711 Comm: argus_syncsenso Not tainted 4.4.38-tegra #3
[  653.908727] Hardware name: quill (DT)
[  653.912392] task: ffffffc1df688c80 ti: ffffffc1ca138000 task.ti: ffffffc1ca138000
[  653.919869] PC is at 0x7f8f2493f4
[  653.923186] LR is at 0x7f8f2493e4
[  653.926501] pc : [<0000007f8f2493f4>] lr : [<0000007f8f2493e4>] pstate: 60000000
[  653.933889] sp : 0000007fd1b0edf0
[  653.937205] x29: 0000007fd1b0ee70 x28: 0000000000000000 
[  653.942536] x27: 0000000000000000 x26: 0000000000000000 
[  653.947868] x25: 0000000000000000 x24: 0000000000000000 
[  653.953220] x23: 0000000000000000 x22: 0000000000000000 
[  653.958552] x21: 0000000000000000 x20: 00000000005105e0 
[  653.963885] x19: 0000007fd1b0ee48 x18: 000000000000000f 
[  653.969220] x17: 0000007f8dacdd30 x16: 0000007f8d9a94a0 
[  653.974550] x15: 0000007f8f3a1cc0 x14: 2f746e65696c632f 
[  653.979882] x13: 74656b636f732f63 x12: 70722f637273206d 
[  653.985217] x11: 6f726620676e6974 x10: 616761706f727028 
[  653.990546] x9 : 20203a74756f656d x8 : 0000000000000040 
[  653.995877] x7 : 0000000000000000 x6 : 0000007fd1b0bf6c 
[  654.001212] x5 : 0000000000000000 x4 : 0000000000000000 
[  654.006543] x3 : 0000000000000000 x2 : 0000000000000001 
[  654.011873] x1 : 0000000000000000 x0 : 0000007fd1b0ee48 

[  654.018699] Library at 0x7f8f2493f4: 0x7f8f1f0000 /usr/lib/aarch64-linux-gnu/tegra/libargus_socketclient.so
[  654.028431] Library at 0x7f8f2493e4: 0x7f8f1f0000 /usr/lib/aarch64-linux-gnu/tegra/libargus_socketclient.so
[  654.038162] vdso base = 0x7f8f3a0000

Unfortunately I did not have the time to implement different sensor modes. It would also not be an option for us to reduce the resolution so I would appreciate any advice for what I can do to debug the issue.

Update:
This is the output of journalctl:

Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvPclHwGetModuleList: No module data found
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvPclHwGetModuleList: No module data found
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: PCLHW_DTParser
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: LoadOverridesFile: looking for override file [/Calib/camera_override.isp] 1/16LoadOverridesFile: looking for override file [/data/nvcam/settings/camera_overrides.isp] 2/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/camera_overrides.isp] 3/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/camera_overrides.isp] 4/16LoadOverridesFile: looking for override file [/data/nvcam/camera_overrides.isp] 5/16LoadOverridesFile: looking for override file [/data/nvcam/settings/imx185_center_liimx185.isp] 6/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/imx185_center_liimx185.isp] 7/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/imx185_center_liimx185.isp] 8/16---- imager: No override file found. ----
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 594: Error: Invalid isp config attribute: "//Thisistheextradelayintroducedwhileprogrammingthegroupholdsettingapartfromdefault//totaldelayof3framesi.e.2framesingroupholdsetttingstoeffectand1frametocapture.//Settingdefaultvalueaszeroasitisdisabledbydefault.ae.MaxGroupHoldDelayFrameCount=0"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 3177: Error: Invalid isp config attribute element: "fd.Available=TRUE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 3178: Error: Invalid isp config attribute element: "fd.ForceEnable=FALSE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "isp4.cfg" Line 333: Error: Invalid isp config attribute: "ds.enable=TRUE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: LoadOverridesFile: looking for override file [/Calib/camera_override.isp] 1/16LoadOverridesFile: looking for override file [/data/nvcam/settings/camera_overrides.isp] 2/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/camera_overrides.isp] 3/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/camera_overrides.isp] 4/16LoadOverridesFile: looking for override file [/data/nvcam/camera_overrides.isp] 5/16LoadOverridesFile: looking for override file [/data/nvcam/settings/imx185_front_liimx185.isp] 6/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/imx185_front_liimx185.isp] 7/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/imx185_front_liimx185.isp] 8/16---- imager: No override file found. ----
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 594: Error: Invalid isp config attribute: "//Thisistheextradelayintroducedwhileprogrammingthegroupholdsettingapartfromdefault//totaldelayof3framesi.e.2framesingroupholdsetttingstoeffectand1frametocapture.//Settingdefaultvalueaszeroasitisdisabledbydefault.ae.MaxGroupHoldDelayFrameCount=0"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 3177: Error: Invalid isp config attribute element: "fd.Available=TRUE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 3178: Error: Invalid isp config attribute element: "fd.ForceEnable=FALSE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "isp4.cfg" Line 333: Error: Invalid isp config attribute: "ds.enable=TRUE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: LoadOverridesFile: looking for override file [/Calib/camera_override.isp] 1/16LoadOverridesFile: looking for override file [/data/nvcam/settings/camera_overrides.isp] 2/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/camera_overrides.isp] 3/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/camera_overrides.isp] 4/16LoadOverridesFile: looking for override file [/data/nvcam/camera_overrides.isp] 5/16LoadOverridesFile: looking for override file [/data/nvcam/settings/imx185_bottom_liimx185.isp] 6/16LoadOverridesFile: looking for override file [/opt/nvidia/nvcam/settings/imx185_bottom_liimx185.isp] 7/16LoadOverridesFile: looking for override file [/var/nvidia/nvcam/settings/imx185_bottom_liimx185.isp] 8/16---- imager: No override file found. ----
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 594: Error: Invalid isp config attribute: "//Thisistheextradelayintroducedwhileprogrammingthegroupholdsettingapartfromdefault//totaldelayof3framesi.e.2framesingroupholdsetttingstoeffectand1frametocapture.//Settingdefaultvalueaszeroasitisdisabledbydefault.ae.MaxGroupHoldDelayFrameCount=0"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 3177: Error: Invalid isp config attribute element: "fd.Available=TRUE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "common.cfg" Line 3178: Error: Invalid isp config attribute element: "fd.ForceEnable=FALSE"
Dec 11 10:06:17 tegra-ubuntu argus_daemon[1102]: NvCameraIspConfigFileLoad: Config file "isp4.cfg" Line 333: Error: Invalid isp config attribute: "ds.enable=TRUE"
Dec 11 10:06:18 tegra-ubuntu argus_daemon[1102]: captureErrorCallback Stream 2.0 capture 1 failed: ts 133987039840 frame 192 error 2 data 0x00400062
Dec 11 10:06:18 tegra-ubuntu argus_daemon[1102]: SCF: Error Timeout:  (propagating from src/services/capture/CaptureServiceDevice.cpp, function issueCaptures(), line 1066)
Dec 11 10:06:18 tegra-ubuntu argus_daemon[1102]: SCF: Error Timeout:  (propagating from src/common/Utils.cpp, function workerThread(), line 114)
Dec 11 10:06:18 tegra-ubuntu argus_daemon[1102]: SCF: Error Timeout: Worker thread CaptureScheduler frameStart failed (in src/common/Utils.cpp, function workerThread(), line 131)
Dec 11 10:06:19 tegra-ubuntu argus_daemon[1102]: SCF: Error Timeout:  (propagating from src/services/capture/CaptureServiceEvent.cpp, function wait(), line 59)
Dec 11 10:06:19 tegra-ubuntu argus_daemon[1102]: Error: Camera HwEvents wait, this may indicate a hardware timeout occured,abort current/incoming cc
Dec 11 10:06:19 tegra-ubuntu argus_daemon[1102]: launchCC abort cc 107 session 2
Dec 11 10:06:19 tegra-ubuntu argus_daemon[1102]: SCF: Error Timeout:  (propagating from src/api/Session.cpp, function capture(), line 830)
Dec 11 10:06:19 tegra-ubuntu argus_daemon[1102]: (Argus) Error Timeout: Failed to submit first capture request (propagating from src/api/CaptureSessionImpl.cpp, function submitCaptureRequests(), line 311)
Dec 11 10:06:19 tegra-ubuntu argus_daemon[1102]: (Argus) Error Timeout:  (propagating from src/api/CaptureSessionImpl.cpp, function threadFunction(), line 777)