Description
I’m using the nvcr.io/nvidia/pytorch:25.11-py3 docker to test nvfp4 on RTX-6000 Blackwell using Model-Optimizer.
I wrote a simple two-layer GeMM and quantize it to nvfp4 using the `torch_quant_to_onnx.py`. I found that by default the input of Q/DQ layers are forced to cast to fp32 while my GeMM layers are using fp16 (see the onnx below)
With these casts, the speedup with nvfp4 is only ~1.5x. After I checked the doc and I changed the following config
cfg["quant_cfg"]["*input_quantizer"]["trt_high_precision_dtype"] = "Half"
cfg["quant_cfg"]["*weight_quantizer"]["trt_high_precision_dtype"] = "Half"
The fp32 casts before Q/DQ layers are gone. My questions are
- How does model-optimizer decide to use fp32 / fp16 for middle representation Q/DQ? I didn’t see fp32 when I’m running the torch_ptq example for ViT, is it coming from calibration set?
- For my real-world model, how shall I tune this precisition dtype to avoid extra casting layers? Is there a guide for it?
Environment
TensorRT Version: 10.2
GPU Type: RTX-6000 Blackwell
Nvidia Driver Version: 570
CUDA Version: 13.0
CUDNN Version:
Operating System + Version: Ubuntu 22.04
Python Version (if applicable):
TensorFlow Version (if applicable):
PyTorch Version (if applicable):
Baremetal or Container (if container which image + tag):
Relevant Files
Please attach or include links to any models, data, files, or scripts necessary to reproduce your issue. (Github repo, Google Drive, Dropbox, etc.)
Steps To Reproduce
Simply run the following codes in the above docker with model-optimizer installed. The `export_to_onnx` function is copied from Model-Optimizer example.
import argparse
import torch
import torch.nn as nn
import os
import copy
import modelopt.torch.quantization as mtq
from modelopt.torch._deploy.utils import OnnxBytes, get_onnx_bytes_and_metadata
class TwoGemm(nn.Module):
# MLP-style: (B,T,H) -> (B,T,4H) -> (B,T,H)
def __init__(self, H: int):
super().__init__()
self.fc1 = nn.Linear(H, 4 * H, bias=False)
self.fc2 = nn.Linear(4 * H, H, bias=True)
def forward(self, x):
b, t, h = x.shape
x = x.reshape(b * t, h).contiguous() # (B*T, H)
x = self.fc1(x) # GEMM1
x = self.fc2(x) # GEMM2
x = x.reshape(b, t, h).contiguous() # (B, T, H)
return x
# From Model-Optimizer example
def export_to_onnx(model, input_shape, onnx_save_path, device, weights_dtype="fp32"):
"""Export the torch model to ONNX format."""
# Create input tensor with same precision as model's first parameter
input_dtype = model.parameters().__next__().dtype
print("Export input dtype: ", input_dtype)
input_tensor = torch.randn(input_shape, dtype=input_dtype).to(device)
model_name = os.path.basename(onnx_save_path).replace(".onnx", "")
onnx_bytes, _ = get_onnx_bytes_and_metadata(
model=model,
dummy_input=(input_tensor,),
weights_dtype=weights_dtype,
model_name=model_name,
onnx_opset=21
)
onnx_bytes_obj = OnnxBytes.from_bytes(onnx_bytes)
# Write the onnx model to the specified directory without cleaning the directory
onnx_bytes_obj.write_to_disk(os.path.dirname(onnx_save_path), clean_dir=False)
# Model-Optimizer requires a "calibration" data set
def make_synth_calib_loader(num_batches: int, B: int, T: int, H: int, device, dtype):
for _ in range(num_batches):
yield torch.zeros((B, T, H), device=device, dtype=dtype)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--onnx_out", type=str, required=True)
ap.add_argument("--B", type=int, default=128)
ap.add_argument("--T", type=int, default=196)
ap.add_argument("--H", type=int, default=1024)
ap.add_argument("--calib_batches", type=int, default=128)
ap.add_argument("--opset", type=int, default=21)
args = ap.parse_args()
torch.manual_seed(0)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
dtype = torch.float16
model = TwoGemm(args.H).to(device=device, dtype=dtype).eval()
# model = TwoGemmGeluSoftmax(args.H).to(device=device, dtype=dtype).eval()
# ---- NVFP4 quantization with a calibration forward loop ----
calib_iter = make_synth_calib_loader(
num_batches=args.calib_batches,
B=args.B, T=args.T, H=args.H,
device=device, dtype=dtype,
)
def forward_loop(m):
# m.eval()
# with torch.no_grad():
for x in calib_iter:
m(x)
cfg = copy.deepcopy(mtq.NVFP4_DEFAULT_CFG)
# cfg["quant_cfg"]["*input_quantizer"]["trt_high_precision_dtype"] = "Half"
# cfg["quant_cfg"]["*weight_quantizer"]["trt_high_precision_dtype"] = "Half"
qmodel = mtq.quantize(model, cfg, forward_loop=forward_loop)
print("\nQuantization Summary:")
mtq.print_quant_summary(qmodel)
# ---- Export quantized model to ONNX (static shape) ----
x = torch.randn((args.B, args.T, args.H), device=device, dtype=dtype)
export_to_onnx(qmodel, x.shape, args.onnx_out, device, weights_dtype="fp16")
print(f"\n[OK] Saved NVFP4-quantized ONNX to: {args.onnx_out}")
print(f" shape=({args.B},{args.T},{args.H}), calib_batches={args.calib_batches}, opset={args.opset}")
if __name__ == "__main__":
main()
