How to support Dynamic Shapes for int32 input?

Description

I have a model from tensorflow whose format is ONNX. My model has 2 inputs, their dims are [1, 1] and [1, 20]. So how can i make it support Dynamic Shapes ?

The model can support batch if i set shape [2000, 1], [2000, 20] (2000 for example) when i transport it to ONNX from tensorflow, but it can only work at 2000 batchs…

Accroding to doc of nvidia, i can use resize layer, but resize layer does not support INT32 data type(why?).

Here is my code, it read a serialized tensorflow model from file, and parse model to ONNX.
THANKS!

bool TdmEngine::ParseOnnxModel(const std::string& model_file) {
  auto builder = nvinfer1::createInferBuilder(gLogger);
  if (!builder) {
    std::cout << "Create inference builder failed." << std::endl;
    return false;
  }
  // 支持动态输入
  const auto explicitBatch =
      1U << static_cast<uint32_t>(
          nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
  auto network = builder->createNetworkV2(explicitBatch);
  if (!network) {
    std::cout << "Create network failed." << std::endl;
    return false;
  }
  // 解析器
  auto parser = samplesCommon::infer_object(
      nvonnxparser::createParser(*network, gLogger));
  if (!parser) {
    std::cout << "Failed to create Parse." << std::endl;
    return false;
  }
  // 读取本地文件
  bool parsingSuccess = parser->parseFromFile(model_file.c_str(), 3);
  if (!parsingSuccess) {
    std::cout << "Failed to parse model." << std::endl;
    return false;
  }
  // 配置
  auto config = builder->createBuilderConfig();
  if (!config) {
    std::cout << "Create builder config failed." << std::endl;
    return false;
  }
  config->setMaxWorkspaceSize(16_GB);

  auto pf_min = nvinfer1::OptProfileSelector::kMIN;
  auto pf_opt = nvinfer1::OptProfileSelector::kOPT;
  auto pf_max = nvinfer1::OptProfileSelector::kMAX;
  // 设置profile
  auto profile = builder->createOptimizationProfile();
  bool set_dim1 =
      profile->setDimensions("user_input", pf_min, nvinfer1::Dims2{1600, 20});
  bool set_dim2 =
      profile->setDimensions("user_input", pf_opt, nvinfer1::Dims2{1600, 20});
  bool set_dim3 =
      profile->setDimensions("user_input", pf_max, nvinfer1::Dims2{8000, 20});

  bool set_dim4 =
      profile->setDimensions("node_input", pf_min, nvinfer1::Dims2{1600, 1});
  bool set_dim5 =
      profile->setDimensions("node_input", pf_opt, nvinfer1::Dims2{1600, 1});
  bool set_dim6 =
      profile->setDimensions("node_input", pf_max, nvinfer1::Dims2{8000, 1});
  if (!set_dim1 || !set_dim2 || !set_dim3 || !set_dim4 || !set_dim5 ||
      !set_dim6) {
    std::cout << "set dim failed." << std::endl;
    return false;
  }
  auto profile_index = config->addOptimizationProfile(profile);
  if (profile_index != 0) {
    std::cout << "add profile failed." << std::endl;
    return false;
  }
  // 生成engine
  auto eng = builder->buildEngineWithConfig(*network, *config);
  if (!eng) {
    std::cout << "Create engine failed." << std::endl;
    return false;
  }
  engine_ =
      std::move(std::shared_ptr<nvinfer1::ICudaEngine>(eng, InferDeleter()));
  return true;
}

Here is the code to build a resize engine, but resize layer does not support INT32

bool TdmEngine::BuildResizeEngine() {
  auto builder = nvinfer1::createInferBuilder(gLogger);
  if (!builder) {
    std::cout << "Create inference builder failed." << std::endl;
    return false;
  }
  // 支持动态输入
  const auto explicitBatch =
      1U << static_cast<uint32_t>(
          nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
  auto network = builder->createNetworkV2(explicitBatch);
  if (!network) {
    std::cout << "Create network failed." << std::endl;
    return false;
  }
  auto input =
      network->addInput("input", nvinfer1::DataType::kINT32, Dims2{-1, -1});
  auto resizeLayer = network->addResize(*input);
  resizeLayer->setOutputDimensions(cuda_engine_->getBindingDimensions(0));
  network->markOutput(*resizeLayer->getOutput(0));
  // 配置
  auto config = builder->createBuilderConfig();
  if (!config) {
    std::cout << "Create builder config failed." << std::endl;
    return false;
  }
  // profile
  auto profile = builder->createOptimizationProfile();
  profile->setDimensions(input->getName(), OptProfileSelector::kMIN,
                         Dims2{1600, 1});
  profile->setDimensions(input->getName(), OptProfileSelector::kOPT,
                         Dims2{1600, 20});
  profile->setDimensions(input->getName(), OptProfileSelector::kMAX,
                         Dims2{8000, 20});
  config->addOptimizationProfile(profile);

  // 生成engine
  auto eng = builder->buildEngineWithConfig(*network, *config);
  if (!eng) {
    std::cout << "Create engine failed." << std::endl;
    return false;
  }
  resize_engine_ =
      std::move(std::shared_ptr<nvinfer1::ICudaEngine>(eng, InferDeleter()));
  return true;
}

Environment

TensorRT Version: TensorRT-7.0.0.11
GPU Type: P40
Nvidia Driver Version: 418.40.04
CUDA Version: 10.1
CUDNN Version: 7
Operating System + Version: centos 7.2
Python Version (if applicable):
TensorFlow Version (if applicable):
PyTorch Version (if applicable):
Baremetal or Container (if container which image + tag):

Relevant Files

Please attach or include links to any models, data, files, or scripts necessary to reproduce your issue. (Github repo, Google Drive, Dropbox, etc.)

Steps To Reproduce

Please include:

  • Exact steps/commands to build your repro
  • Exact steps/commands to run your repro
  • Full traceback of errors encountered

Hi @695561506,
Request you to check the below link to understand dynamic shapes.


Also, you can refer to the below sample.

Thanks!

THANKS!
I read the guide and i still can not solve my problem:
Sample ONNX MNIST, create another network preprocessorNetwork to resize input for MNIST model, then when we run inference, we should run preprocessorNetwork firstly, and run MNIST model.

Of course i almost copy the way in my code, but i find that this sample can not work. Because my model input’s data type is INT32, resize layer do not support INT32.

// This sample use nvinfer1::DataType::kFLOAT, but i use kINT32, so it will goes  wrong while build engine
auto input = preprocessorNetwork->addInput("input", nvinfer1::DataType::kFLOAT, Dims4{-1, 1, -1, -1});

THANKS a lot