Hello,
I use mlnx_qos tool set connectx-7 rdma nic rate limit meet error as below:
$ sudo mlnx_qos -i ens25np0 -r 0,0,0,0,0,350,0,0
Rate limit is not supported on your system!
DCBX mode: OS controlled
Priority trust state: pcp
Receive buffer size (bytes): 0,1046016,0,0,0,0,0,0,max_buffer_size=4148640
Cable len: 7
PFC configuration:
priority 0 1 2 3 4 5 6 7
enabled 0 0 0 0 0 0 0 0
buffer 1 1 1 1 1 1 1 1
tc: 0 ratelimit: unlimited, tsa: vendor
priority: 1
tc: 1 ratelimit: unlimited, tsa: vendor
priority: 0
tc: 2 ratelimit: unlimited, tsa: vendor
priority: 2
tc: 3 ratelimit: unlimited, tsa: vendor
priority: 3
tc: 4 ratelimit: unlimited, tsa: vendor
priority: 4
tc: 5 ratelimit: 50.0 Gbps, tsa: vendor
priority: 5
tc: 6 ratelimit: unlimited, tsa: vendor
priority: 6
tc: 7 ratelimit: unlimited, tsa: vendor
priority: 7
error msg :Rate limit is not supported on your system!
dmesg error msg: mlx5_core 0000:53:00.0 ens25np0: tc_5 maxrate 300000000 Kbps exceeds limit 255000000
more detail log: ConnectX-7-log.txt (15.9 KB)
hardware info：
Device type: ConnectX7
Name: MCX75310AAS-NEA_Ax
Description: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;
Device: 13:00.0
From kernel source code ， I found that qetcr_qshr_max_bw_val_msb capability bit not enable.
static ssize_t mlx5e_store_maxrate(struct device *device,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct mlx5e_priv *priv = netdev_priv(to_net_dev(device));
struct mlx5_core_dev *mdev = priv->mdev;
u64 tc_maxrate[IEEE_8021QAZ_MAX_TCS];
u16 max_bw_value[MLX5_MAX_NUM_TC];
u8 max_bw_unit[MLX5_MAX_NUM_TC];
u64 upper_limit_100mbps;
u64 upper_limit_gbps;
char delimiter;
u16 type_max;
int i = 0;
int ret;
memset(max_bw_value, 0, sizeof(max_bw_value));
memset(max_bw_unit, 0, sizeof(max_bw_unit));
type_max = MLX5_CAP_QCAM_FEATURE(mdev, qetcr_qshr_max_bw_val_msb) ?
U16_MAX : U8_MAX;
upper_limit_100mbps = type_max * MLX5E_100MB_TO_KB;
upper_limit_gbps = type_max * MLX5E_1GB_TO_KB;
do {
int len;
u64 input_maxrate;
if (i >= MLX5_MAX_NUM_TC)
goto bad_elem_count;
len = strcspn(buf, " ");
/* nul-terminate and parse */
delimiter = buf[len];
((char *)buf)[len] = '\0';
if (sscanf(buf, "%lld", &input_maxrate) != 1
|| input_maxrate < 0) {
netdev_err(priv->netdev, "bad maxrate value: '%s'\n",
buf);
goto out;
}
tc_maxrate[i] = input_maxrate;
buf += len + 1;
i++;
} while (delimiter == ' ');
if (i != MLX5_MAX_NUM_TC)
goto bad_elem_count;
for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
u64 rate = tc_maxrate[i];
if (!tc_maxrate[i]) {
max_bw_unit[i] = MLX5_BW_NO_LIMIT;
continue;
}
if (rate <= upper_limit_100mbps) {
max_bw_value[i] = div_u64(rate, MLX5E_100MB_TO_KB);
max_bw_value[i] = max_bw_value[i] ? max_bw_value[i] : 1;
max_bw_unit[i] = MLX5_100_MBPS_UNIT;
} else if (rate <= upper_limit_gbps) {
max_bw_value[i] = div_u64(rate, MLX5E_1GB_TO_KB);
max_bw_unit[i] = MLX5_GBPS_UNIT;
} else {
netdev_err(priv->netdev,
"tc_%d maxrate %llu Kbps exceeds limit %llu\n",
i, rate, upper_limit_gbps);
return -EINVAL;
}
}
reference link : net/mlx5: Add IFC bits for extended ETS rate limit bandwidth value | Patchew
I think doca-ofed [OFED-internal-26.04-0.8.6] for connectx-7 rnic should support rate limit function.
@spruitt Can you help me to figure out why qetcr_qshr_max_bw_val_msb bit not enable.
Did I need do more configure for ConnectX-7 rnic ?