Description
I have a model from tensorflow whose format is ONNX. My model has 2 inputs, their dims are [1, 1] and [1, 20]. So how can i make it support Dynamic Shapes ?
The model can support batch if i set shape [2000, 1], [2000, 20] (2000 for example) when i transport it to ONNX from tensorflow, but it can only work at 2000 batchs…
Accroding to doc of nvidia, i can use resize layer, but resize layer does not support INT32 data type(why?).
Here is my code, it read a serialized tensorflow model from file, and parse model to ONNX.
THANKS!
bool TdmEngine::ParseOnnxModel(const std::string& model_file) {
auto builder = nvinfer1::createInferBuilder(gLogger);
if (!builder) {
std::cout << "Create inference builder failed." << std::endl;
return false;
}
// 支持动态输入
const auto explicitBatch =
1U << static_cast<uint32_t>(
nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
auto network = builder->createNetworkV2(explicitBatch);
if (!network) {
std::cout << "Create network failed." << std::endl;
return false;
}
// 解析器
auto parser = samplesCommon::infer_object(
nvonnxparser::createParser(*network, gLogger));
if (!parser) {
std::cout << "Failed to create Parse." << std::endl;
return false;
}
// 读取本地文件
bool parsingSuccess = parser->parseFromFile(model_file.c_str(), 3);
if (!parsingSuccess) {
std::cout << "Failed to parse model." << std::endl;
return false;
}
// 配置
auto config = builder->createBuilderConfig();
if (!config) {
std::cout << "Create builder config failed." << std::endl;
return false;
}
config->setMaxWorkspaceSize(16_GB);
auto pf_min = nvinfer1::OptProfileSelector::kMIN;
auto pf_opt = nvinfer1::OptProfileSelector::kOPT;
auto pf_max = nvinfer1::OptProfileSelector::kMAX;
// 设置profile
auto profile = builder->createOptimizationProfile();
bool set_dim1 =
profile->setDimensions("user_input", pf_min, nvinfer1::Dims2{1600, 20});
bool set_dim2 =
profile->setDimensions("user_input", pf_opt, nvinfer1::Dims2{1600, 20});
bool set_dim3 =
profile->setDimensions("user_input", pf_max, nvinfer1::Dims2{8000, 20});
bool set_dim4 =
profile->setDimensions("node_input", pf_min, nvinfer1::Dims2{1600, 1});
bool set_dim5 =
profile->setDimensions("node_input", pf_opt, nvinfer1::Dims2{1600, 1});
bool set_dim6 =
profile->setDimensions("node_input", pf_max, nvinfer1::Dims2{8000, 1});
if (!set_dim1 || !set_dim2 || !set_dim3 || !set_dim4 || !set_dim5 ||
!set_dim6) {
std::cout << "set dim failed." << std::endl;
return false;
}
auto profile_index = config->addOptimizationProfile(profile);
if (profile_index != 0) {
std::cout << "add profile failed." << std::endl;
return false;
}
// 生成engine
auto eng = builder->buildEngineWithConfig(*network, *config);
if (!eng) {
std::cout << "Create engine failed." << std::endl;
return false;
}
engine_ =
std::move(std::shared_ptr<nvinfer1::ICudaEngine>(eng, InferDeleter()));
return true;
}
Here is the code to build a resize engine, but resize layer does not support INT32
bool TdmEngine::BuildResizeEngine() {
auto builder = nvinfer1::createInferBuilder(gLogger);
if (!builder) {
std::cout << "Create inference builder failed." << std::endl;
return false;
}
// 支持动态输入
const auto explicitBatch =
1U << static_cast<uint32_t>(
nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
auto network = builder->createNetworkV2(explicitBatch);
if (!network) {
std::cout << "Create network failed." << std::endl;
return false;
}
auto input =
network->addInput("input", nvinfer1::DataType::kINT32, Dims2{-1, -1});
auto resizeLayer = network->addResize(*input);
resizeLayer->setOutputDimensions(cuda_engine_->getBindingDimensions(0));
network->markOutput(*resizeLayer->getOutput(0));
// 配置
auto config = builder->createBuilderConfig();
if (!config) {
std::cout << "Create builder config failed." << std::endl;
return false;
}
// profile
auto profile = builder->createOptimizationProfile();
profile->setDimensions(input->getName(), OptProfileSelector::kMIN,
Dims2{1600, 1});
profile->setDimensions(input->getName(), OptProfileSelector::kOPT,
Dims2{1600, 20});
profile->setDimensions(input->getName(), OptProfileSelector::kMAX,
Dims2{8000, 20});
config->addOptimizationProfile(profile);
// 生成engine
auto eng = builder->buildEngineWithConfig(*network, *config);
if (!eng) {
std::cout << "Create engine failed." << std::endl;
return false;
}
resize_engine_ =
std::move(std::shared_ptr<nvinfer1::ICudaEngine>(eng, InferDeleter()));
return true;
}
Environment
TensorRT Version: TensorRT-7.0.0.11
GPU Type: P40
Nvidia Driver Version: 418.40.04
CUDA Version: 10.1
CUDNN Version: 7
Operating System + Version: centos 7.2
Python Version (if applicable):
TensorFlow Version (if applicable):
PyTorch Version (if applicable):
Baremetal or Container (if container which image + tag):
Relevant Files
Please attach or include links to any models, data, files, or scripts necessary to reproduce your issue. (Github repo, Google Drive, Dropbox, etc.)
Steps To Reproduce
Please include:
- Exact steps/commands to build your repro
- Exact steps/commands to run your repro
- Full traceback of errors encountered