@SunilJB So I tried to follow the example provided by TensorRT, but I seem to have run into a problem. The model I am trying to work with is: models/vision/super_resolution/sub_pixel_cnn_2016 at main · onnx/models · GitHub
The code I have implemented:
samplesCommon::OnnxSampleParams initializeSampleParams()
{
samplesCommon::OnnxSampleParams params;
params.dataDirs.push_back("data/mnist/");
params.dataDirs.push_back("data/samples/mnist/");
/*params.onnxFileName = "mnist.onnx";
params.inputTensorNames.push_back("Input3");
params.outputTensorNames.push_back("Plus214_Output_0");*/
params.onnxFileName = "super_resolution.onnx";
params.inputTensorNames.push_back("input");
params.outputTensorNames.push_back("output");
params.batchSize = 1;
params.int8 = false;
params.fp16 = false;
return params;
}
void SampleDynamicReshape::build()
{
auto builder = makeUnique(nvinfer1::createInferBuilder(gLogger.getTRTLogger()));
// This function will also set mPredictionInputDims and mPredictionOutputDims,
// so it needs to be called before building the preprocessor.
buildPredictionEngine(builder);
buildPreprocessorEngine(builder);
}
void SampleDynamicReshape::buildPreprocessorEngine(const SampleUniquePtr<nvinfer1::IBuilder>&
builder)
{
// Create the preprocessor engine using a network that supports full dimensions (createNetworkV2).
auto preprocessorNetwork = makeUnique(
builder->createNetworkV2(1U << static_cast<uint32_t>(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)));
// Reshape a dynamically shaped input to the size expected by the model, (1, 1, 28, 28).
//auto input = preprocessorNetwork->addInput("input", nvinfer1::DataType::kFLOAT, Dims4{ 1, 1, -1, -1 });
auto input = preprocessorNetwork->addInput("input", nvinfer1::DataType::kFLOAT, Dims4{ -1, 1, 1, 1 });
auto resizeLayer = preprocessorNetwork->addResize(*input);
resizeLayer->setOutputDimensions(mPredictionInputDims);
preprocessorNetwork->markOutput(*resizeLayer->getOutput(0));
// Finally, configure and build the preprocessor engine.
auto preprocessorConfig = makeUnique(builder->createBuilderConfig());
// Create an optimization profile so that we can specify a range of input dimensions.
auto profile = builder->createOptimizationProfile();
// This profile will be valid for all images whose size falls in the range of [(1, 1, 1, 1), (1, 1, 224, 224)]
// but TensorRT will optimize for (1, 1, 28, 28)
/*profile->setDimensions(input->getName(), OptProfileSelector::kMIN, Dims4{ 1, 1, 1, 1});
profile->setDimensions(input->getName(), OptProfileSelector::kOPT, Dims4{ 1, 1, 28, 28 });
profile->setDimensions(input->getName(), OptProfileSelector::kMAX, Dims4{ 1, 1, 56, 56 });*/
profile->setDimensions(input->getName(), OptProfileSelector::kMIN, Dims4{ 1, 1, 1, 1 });
profile->setDimensions(input->getName(), OptProfileSelector::kOPT, Dims4{ 1, 1, 224, 224 });
profile->setDimensions(input->getName(), OptProfileSelector::kMAX, Dims4{ 1, 1, 224, 224});
preprocessorConfig->addOptimizationProfile(profile);
mPreprocessorEngine = makeUnique(builder->buildEngineWithConfig(*preprocessorNetwork, *preprocessorConfig));
gLogInfo << "Profile dimensions in preprocessor engine:\n";
gLogInfo << " Minimum = " << mPreprocessorEngine->getProfileDimensions(0, 0, OptProfileSelector::kMIN) << '\n';
gLogInfo << " Optimum = " << mPreprocessorEngine->getProfileDimensions(0, 0, OptProfileSelector::kOPT) << '\n';
gLogInfo << " Maximum = " << mPreprocessorEngine->getProfileDimensions(0, 0, OptProfileSelector::kMAX)
<< std::endl;
}
void SampleDynamicReshape::buildPredictionEngine(const SampleUniquePtr<nvinfer1::IBuilder>&
builder)
{
// Create a network using the parser.
const auto explicitBatch = 1U << static_cast<uint32_t>(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH);
auto network = makeUnique(builder->createNetworkV2(explicitBatch));
auto parser = nvonnxparser::createParser(*network, gLogger.getTRTLogger());
bool parsingSuccess = parser->parseFromFile(
locateFile(mParams.onnxFileName, mParams.dataDirs).c_str(), static_cast<int>(gLogger.getReportableSeverity()));
if (!parsingSuccess)
{
throw std::runtime_error{ "Failed to parse model" };
}
/*// Attach a softmax layer to the end of the network.
auto softmax = network->addSoftMax(*network->getOutput(0));
// Set softmax axis to 1 since network output has shape [1, 10] in full dims mode
softmax->setAxes(1 << 1);
network->unmarkOutput(*network->getOutput(0));
network->markOutput(*softmax->getOutput(0));*/
// Get information about the inputs/outputs directly from the model.
mPredictionInputDims = network->getInput(0)->getDimensions();
mPredictionOutputDims = network->getOutput(0)->getDimensions();
// Create a builder config
auto config = makeUnique(builder->createBuilderConfig());
config->setMaxWorkspaceSize(16_MiB);
if (mParams.fp16)
{
config->setFlag(BuilderFlag::kFP16);
}
if (mParams.int8)
{
config->setFlag(BuilderFlag::kINT8);
samplesCommon::setAllTensorScales(network.get(), 127.0f, 127.0f);
}
// Build the prediciton engine.
mPredictionEngine = makeUnique(builder->buildEngineWithConfig(*network, *config));
}
So the model seems to load and input/output is read, but I get an error on the following line:
mPredictionEngine = makeUnique(builder->buildEngineWithConfig(*network, *config));
[E] [TRT] Network has dynamic or shape inputs, but no optimization profile has been defined.
[E] [TRT] Network validation failed.
Seems that the right input shape is detected:
