I am trying to run a ResNet-50 model concurrently on both the DLA cores but I get strange results:
The total throughput across both the DLA core should ideally be double than that when run on a single DLA core.
Can anyone explain the reason?
Hi @kunal.sahoo2003 ,
Can you please try using Nsight Systems to see if both DLAs are executing concurrently or not?
Ensure that inference is explicitly bound to DLA_0 and DLA_1.
import torch
import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit # DO NOT REMOVE
import time
import os
import csv
import warnings
import argparse
import sys
warnings.filterwarnings('ignore')
class DualLogger:
def __init__(self, log_path):
self.terminal = sys.stdout
self.log_file = open(log_path, "a")
def write(self, message):
if message.strip(): # Avoid writing empty lines
self.terminal.write(message)
self.log_file.write(f"[KUNAL] {message}")
self.log_file.flush()
def flush(self):
self.terminal.flush()
self.log_file.flush()
def build_engine(onnx_file_path, batch_size, dla_core=0, dtype='fp16', logger=None):
builder = trt.Builder(logger)
network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
parser = trt.OnnxParser(network, logger)
with open(onnx_file_path, 'rb') as model:
if not parser.parse(model.read()):
print('ERROR: Failed to parse the ONNX file.')
for error in range(parser.num_errors):
print(parser.get_error(error))
return None
profile = builder.create_optimization_profile()
input_name = network.get_input(0).name
profile.set_shape(
input_name,
(batch_size, 3, 300, 300), # min
(batch_size, 3, 300, 300), # opt
(batch_size, 3, 300, 300) # max
)
trt_dtype = {'fp16': trt.BuilderFlag.FP16, 'int8': trt.BuilderFlag.INT8}
config = builder.create_builder_config()
config.add_optimization_profile(profile)
config.set_flag(trt.BuilderFlag.GPU_FALLBACK)
config.default_device_type = trt.DeviceType.DLA
config.set_flag(trt_dtype[dtype])
config.DLA_core = dla_core
serialized_engine = builder.build_serialized_network(network, config)
if serialized_engine is None:
print('Failed to build serialized engine')
return None
runtime = trt.Runtime(logger)
engine = runtime.deserialize_cuda_engine(serialized_engine)
return engine
def allocate_buffers(engine):
inputs, outputs, stream = [], [], cuda.Stream()
for binding in engine:
size = trt.volume(engine.get_tensor_shape(binding))
dtype = trt.nptype(engine.get_tensor_dtype(binding))
host_mem = cuda.pagelocked_empty(size, dtype)
device_mem = cuda.mem_alloc(host_mem.nbytes)
tensor_info = {'name': binding, 'host': host_mem, 'device': device_mem}
if engine.get_tensor_mode(binding) == trt.TensorIOMode.INPUT:
inputs.append(tensor_info)
elif engine.get_tensor_mode(binding) == trt.TensorIOMode.OUTPUT:
outputs.append(tensor_info)
return inputs, outputs, stream
def infer(context, inputs, outputs, stream):
for inp in inputs:
cuda.memcpy_htod_async(inp['device'], inp['host'], stream)
context.set_tensor_address(inp['name'], inp['device'])
for out in outputs:
context.set_tensor_address(out['name'], out['device'])
context.execute_async_v3(stream.handle)
for out in outputs:
cuda.memcpy_dtoh_async(out['host'], out['device'], stream)
stream.synchronize()
def barrier_sync(sync_dir, batch_size):
os.makedirs(sync_dir, exist_ok=True)
sync_file = os.path.join(sync_dir, f'sync_{batch_size}_{os.getpid()}')
with open(sync_file, 'w') as f:
f.write('ready')
while True:
ready_files = [f for f in os.listdir(sync_dir) if f.startswith(f'sync_{batch_size}_')]
if len(ready_files) >= 2:
break
time.sleep(0.1)
time.sleep(0.05)
try:
os.remove(sync_file)
except FileNotFoundError:
pass
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--onnx', required=True, help='Path to .onnx file')
parser.add_argument('--log', required=True, help='CSV output log file')
parser.add_argument('--dla_core', required=False, default=0, type=int, choices=[0, 1])
parser.add_argument('--iterations', required=False, default=100, type=int)
parser.add_argument('--dtype', required=False, default='fp16', choices=['fp16', 'int8'])
parser.add_argument('--sync_dir', required=True, help='Directory for sync files')
parser.add_argument('--trt_log_file', required=True, help='Path to save the TensorRT verbose log')
args = parser.parse_args()
# Configure DualLogger for both stdout and file
sys.stdout = DualLogger(args.trt_log_file)
# Set up TensorRT logger with VERBOSE mode
trt_logger = trt.Logger(trt.Logger.VERBOSE)
batch_sizes = [128, 192, 256, 384, 512]
file = open(args.log, 'w', newline='')
writer = csv.writer(file)
writer.writerow(['Batch Size', 'Average Throughput (images/s)', 'Average Latency (ms)'])
for batch_size in batch_sizes:
print(f"[DLA Core {args.dla_core}] Preparing engine for batch size {batch_size}...")
engine = build_engine(args.onnx, batch_size, args.dla_core, args.dtype, logger=trt_logger)
if engine is None:
return
inputs, outputs, stream = allocate_buffers(engine)
context = engine.create_execution_context()
infer(context, inputs, outputs, stream) # Warm-up
barrier_sync(args.sync_dir, batch_size)
total_time = 0
for _ in range(args.iterations):
start = time.time()
infer(context, inputs, outputs, stream)
end = time.time()
latency = (end - start)
total_time += latency
writer.writerow([batch_size, batch_size / latency, latency * 1000])
avg_time = total_time / args.iterations
throughput = batch_size / avg_time
print(f'[DLA Core {args.dla_core}] Batch {batch_size}: Avg Latency = {avg_time*1000:.2f} ms | Throughput = {throughput:.2f} images/s')
file.close()
if __name__ == '__main__':
main()
Could you share a command for that?
Whenever I am trying to use NSYS on my custom script, then I do not SEE any DLA traces being logged. Whereas when I run the trtexec command mentioned in the GitHub repository of NVIDIA, that traces.
Can you help me what went wrong.
Command I am running:
sudo nsys profile --trace=cuda,nvtx,cublas,cudla,cusparse,cudnn,nvmedia --output=ssd_mobnet2.nsys-rep /media/ssd/kunal/.venv/bin/python DLA_throughput_beta.py --onnx onnx/ssd_mobilenet_v1_coco_2018_01_28_prepared.onnx --log_prefix ssd_mob --iterations 10 --dtype fp16