Record traces of DLA

Robotics & Edge Computing Jetson Systems Jetson Nano
1

On running the command: /usr/src/tensorrt/bin/trtexec --onnx=model_gn.onnx --shapes=input:32x3x32x32 --saveEngine=model_gn.engine --exportProfile=model_gn.json --int8 --useDLACore=0 --allowGPUFallback --useSpinWait --separateProfileRun --verbose, I do get the details of layers running on DLA and GPU, but I only get traces of GPU:

[04/28/2025-11:06:45] [I] === Trace details ===
[04/28/2025-11:06:45] [I] Trace averages of 10 runs:
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.50102 ms - Host latency: 1.5608 ms (enqueue 0.778024 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.49293 ms - Host latency: 1.55209 ms (enqueue 0.777457 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.50078 ms - Host latency: 1.56141 ms (enqueue 0.775542 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.49539 ms - Host latency: 1.55526 ms (enqueue 0.777785 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.50013 ms - Host latency: 1.55927 ms (enqueue 0.785571 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.49252 ms - Host latency: 1.5517 ms (enqueue 0.777322 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.49584 ms - Host latency: 1.55502 ms (enqueue 0.780673 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.49623 ms - Host latency: 1.55559 ms (enqueue 0.775174 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.50175 ms - Host latency: 1.56211 ms (enqueue 0.779517 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.48832 ms - Host latency: 1.54827 ms (enqueue 0.782928 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.50067 ms - Host latency: 1.55959 ms (enqueue 0.780905 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.48969 ms - Host latency: 1.54971 ms (enqueue 0.781357 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.48878 ms - Host latency: 1.54963 ms (enqueue 0.783652 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.48514 ms - Host latency: 1.54452 ms (enqueue 0.782791 ms)
[04/28/2025-11:06:45] [I] Average on 10 runs - GPU latency: 1.50404 ms - Host latency: 1.56283 ms (enqueue 0.773969 ms)
...

I want to know is there any way to record the latency and other metrics for DLA?

3

Hi,

Please check the below section from the same GitHub:

Please use --trace=cuda,nvtx,cublas,cudla,cusparse,cudnn,nvmedia to collect the trace for onboard accelerators.

Thanks.

4

Is there a way to measure the execution time of DLA and GPU separately via code instead of nsys?

5

Hi,

Please check if tegrastats can meet your requirements:

Thanks.

6

I tried measuring the execution time of DLA like the code attached below, kindly validate:

import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit  # DO NOT REMOVE
import time
import os
import csv
import warnings
import argparse

warnings.filterwarnings('ignore')

os.environ['TENSORRT_LOGGER_SEVERITY'] = 'VERBOSE'

def build_engine(onnx_file_path, batch_size, dla_core=0, dtype='fp16'):
    logger = trt.Logger(trt.Logger.VERBOSE)
    builder = trt.Builder(logger)
    network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
    parser = trt.OnnxParser(network, logger)
    
    with open(onnx_file_path, 'rb') as model:
        if not parser.parse(model.read()):
            print('ERROR: Failed to parse the ONNX file.')
            for error in range(parser.num_errors):
                print(parser.get_error(error))
            return None

    # define the optimization configuration
    profile = builder.create_optimization_profile()
    profile.set_shape(
        'input',
        (batch_size, 3, 224, 224),   # min shape
        (batch_size, 3, 224, 224),   # optimal shape
        (batch_size, 3, 224, 224)    # max shape
    )   # Will have to set the same batch size for it to run on DLA

    trt_dtype = {'fp16': trt.BuilderFlag.FP16, 'int8': trt.BuilderFlag.INT8}

    config = builder.create_builder_config()
    config.add_optimization_profile(profile)
    config.set_flag(trt.BuilderFlag.GPU_FALLBACK)
    config.default_device_type = trt.DeviceType.DLA
    config.set_flag(trt_dtype[dtype])
    config.DLA_core = dla_core

    serialized_engine = builder.build_serialized_network(network, config)
    if serialized_engine is None:
        print('Failed to build serialized engine')
        return None
    
    runtime = trt.Runtime(logger)
    engine = runtime.deserialize_cuda_engine(serialized_engine)
    return engine

def allocate_buffers(engine):
    inputs, outputs = [], []
    stream = cuda.Stream()
    
    for binding in engine:
        size = trt.volume(engine.get_tensor_shape(binding))
        dtype = trt.nptype(engine.get_tensor_dtype(binding))
        host_mem = cuda.pagelocked_empty(size, dtype)
        device_mem = cuda.mem_alloc(host_mem.nbytes)
        tensor_info = {'name': binding, 'host': host_mem, 'device': device_mem}
        if engine.get_tensor_mode(binding) == trt.TensorIOMode.INPUT:
            inputs.append(tensor_info)
        elif engine.get_tensor_mode(binding) == trt.TensorIOMode.OUTPUT:
            outputs.append(tensor_info)
    
    return inputs, outputs, stream

def infer(context, inputs, outputs, stream):
    for inp in inputs:
        cuda.memcpy_htod_async(inp['device'], inp['host'], stream)
        context.set_tensor_address(inp['name'], inp['device'])
    for out in outputs:
        context.set_tensor_address(out['name'], out['device'])
    
    context.execute_async_v3(stream.handle)

    for out in outputs:
        cuda.memcpy_dtoh_async(out['host'], out['device'], stream)
    
    stream.synchronize()


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--onnx', required=True, help='Path to .onnx file')
    parser.add_argument('--log', required=True, help='name of the log file')
    parser.add_argument('--dla_core', required=False, default=0, type=int, choices=[0, 1])
    parser.add_argument('--iterations', required=False, default=100, type=int)
    parser.add_argument('--dtype', required=False, default='fp16', choices=['fp16', 'int8'])

    args = parser.parse_args()

    onnx_file_path = args.onnx
    log_file_name = os.path.join('/media/ssd/kunal/logs', args.log)
    dla_core = args.dla_core
    num_iterations = args.iterations
    dtype = args.dtype

    batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256]
    
    file = open(log_file_name, 'w', newline='')
    writer = csv.writer(file)
    writer.writerow(['Batch Size', 'Average Throughput (images/s)', 'Average Latency (ms)'])

    for batch_size in batch_sizes:
        engine = build_engine(onnx_file_path, batch_size, dla_core, dtype)
        if engine is None:
            return

        inputs, outputs, stream = allocate_buffers(engine)
        context = engine.create_execution_context()

        infer(context, inputs, outputs, stream)  # Warm-up, to be ignored

        # TODO: Fix the time recording logic use cuda event sync
        total_time = 0
        for _ in range(num_iterations):
            start_time = time.time()
            infer(context, inputs, outputs, stream)
            end_time = time.time()
            latency = (end_time - start_time)
            total_time += latency

            writer.writerow([batch_size, batch_size / latency, latency * 100])


        avg_time = total_time / num_iterations
        throughput = batch_size / avg_time

        print('Average latency:', avg_time)
        print('Average throughput (images/s):', throughput)

    file.close()

                    
if __name__ == '__main__':
    main()
7

Hi,

It’s more recommended to capture the status of the node to measure the usage of DLA.
Capturing the duration on the user space might include some non-dla tasks like data transfer and gpu fallback.

Thanks.