How can I define the input data format into NHWC?

AI & Data Science Deep Learning (Training & Inference) TensorRT
1

import tensorrt as trt
import trt_common as common
import numpy as np
TRT_LOGGER = trt.Logger(trt.Logger.WARNING)

def layer_define():
    with trt.Builder(TRT_LOGGER) as builder, builder.create_network() as network:
        builder.max_workspace_size = common.GiB(1)
        builder.fp16_mode = True
        #builder.strict_type_constraints = True
        #builder.int8_mode = True
        #builder.int8_calibrator = calib

        input_tensor = network.add_input(name='input', dtype=trt.float32, shape=(1, 64, 56, 56))

        conv1_weight = np.ones((64, 64, 3, 3), dtype=np.float32)
        conv1 = network.add_convolution(input_tensor, 64, (3, 3), conv1_weight, trt.Weights())

        conv1.get_output(0).name = 'output'
        network.mark_output(conv1.get_output(0))
        engine = builder.build_cuda_engine(network)
    return engine

def run_engine():
    with layer_define() as engine:
        inputs, outputs, bindings, stream = common.allocate_buffers(engine)
        inputs[0].host = np.ones((1, 64, 56, 56), dtype=np.float32)
        with engine.create_execution_context() as context:
            [output] = common.do_inference(context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream, run_iter=1)
            [output] = common.do_inference(context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream, run_iter=1000)

def main():
#    engine = layer_define()
    run_engine()

if __name__ == "__main__":
    main()

From above code, when I use nvprof I can find as follows:

                9.98%  16.609ms      1001  16.592us  14.881us  17.889us  void cuInt8::nchwTonhwc<float, int=32, int=32, int=2>(float const *, __half*, int, int, int, int, int, int, int, int)
                7.52%  12.526ms      1001  12.513us  11.968us  13.601us  void cuInt8::nhwcTonchw<float, int=32, int=32, int=2>(__half const *, float*, int, int, int, int, int, int)

So I want to know how I define the input data format with NHWC?

2

Hi @Chad.Ding
In C++, you can use ITensor::setAllowedFormats
https://docs.nvidia.com/deeplearning/tensorrt/api/c_api/classnvinfer1_1_1_i_tensor.html#abb6a8ccb84e2e4547371e10574a5bfb4
In Python,
you can use iTensor.allowed_formats
https://docs.nvidia.com/deeplearning/tensorrt/api/python_api/infer/Graph/LayerBase.html#tensorrt.ITensor

Thanks!

3

I have a similar issue in tensor rt 7 c++, i have tried using setAllowedformat to set NHWC for 32bit inference but it gives error stating kHWC format doesnt exist. can you please tell how to set input layout to NHWC? or is it even possible?
Currently I am doing like this :

network->getInput(0)->setAllowedFormats(static_castnvinfer1::TensorFormats(1 << static_cast(nvinfer1::TensorFormat::kHWC)));
config->setFlag(nvinfer1::BuilderFlag::kSTRICT_TYPES);

but its giving following error

error: ‘kHWC’ is not a member of ‘nvinfer1::TensorFormat’
network->getInput(0)->setAllowedFormats(static_castnvinfer1::TensorFormats(1 << static_cast(nvinfer1::TensorFormat::kHWC)));

However here it is shown as a member:
https://docs.nvidia.com/deeplearning/tensorrt/api/c_api/namespacenvinfer1.html#ac3e115b1a2b1e578e8221ef99d27cd45