Hello NVIDIA Support Team,
I am working on testing the Multi-Packet RQ/Striding RQ feature available on ConnectX-4/5/6, as outlined in NVIDIA’s documentation. According to the documentation, this feature is applicable only to Work Queues (WQs) rather than Queue Pairs (QPs).
However, I encountered an issue when attempting to create an RDMA WQ using the mlx5dv_create_wq
function. The system returned an “Operation not supported” error. To troubleshoot further, I also tried using ibv_create_wq
without enabling multi-packet RQ, but the same error occurred. Additionally, I tested on an older setup using ibv_exp_create_wq
on Ubuntu 18.04.6 with MLNX_OFED 4.7, but encountered the same result.
Here are the details of my current setup:
- OS: Ubuntu 22.04.4
- Kernel Version: 6.8.0-45-generic
- MLNX_OFED Version: 24.04.OFED.24.04.0.7.0.1-1
- g++ Version: 11.4.0
I have attached the test code that reproduces the issue for your reference.
#define STRIDING_RQ 1
#include <stdio.h>
#include <infiniband/verbs.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#if STRIDING_RQ
#include <infiniband/mlx5dv.h>
#endif
// g++ -Wall -I/usr/include/infiniband -c test_wq.cc -o test_wq.o
// g++ -o test_wq test_wq.o -libverbs -lrdmacm -lmlx5
// Makefile:
// CC = g++
// CFLAGS = -Wall -I/usr/include/infiniband
// LDFLAGS = -libverbs -lrdmacm -lmlx5
// TARGET = test_wq
// SRCS = test_wq.cc
// OBJS = $(SRCS:.cc=.o)
// all: $(TARGET)
// $(TARGET): $(OBJS)
// $(CC) -o $@ $^ $(LDFLAGS)
// %.o: %.cc
// $(CC) $(CFLAGS) -c $< -o $@
// clean:
// rm -f $(TARGET) $(OBJS)
int main() {
struct ibv_context *context;
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_wq_init_attr wq_init_attr;
#if STRIDING_RQ
struct mlx5dv_wq_init_attr mlx5_wq_attr;
#endif
struct ibv_wq *wq;
int ret = 0;
struct ibv_device **dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
perror("Failed to get IB devices list");
return 1;
}
context = ibv_open_device(dev_list[0]);
if (!context) {
perror("Failed to open device");
ibv_free_device_list(dev_list);
return 1;
}
pd = ibv_alloc_pd(context);
if (!pd) {
perror("Failed to allocate protection domain");
ibv_close_device(context);
ibv_free_device_list(dev_list);
return 1;
}
cq = ibv_create_cq(context, 1024, NULL, NULL, 0);
if (!cq) {
perror("Failed to create completion queue");
ibv_dealloc_pd(pd);
ibv_close_device(context);
ibv_free_device_list(dev_list);
return 1;
}
memset(&wq_init_attr, 0, sizeof(wq_init_attr));
wq_init_attr.wq_type = IBV_WQT_RQ;
wq_init_attr.max_wr = 1024;
wq_init_attr.max_sge = 1;
wq_init_attr.pd = pd;
wq_init_attr.cq = cq;
#if STRIDING_RQ
wq_init_attr.create_flags = 0;
memset(&mlx5_wq_attr, 0, sizeof(mlx5_wq_attr));
mlx5_wq_attr.comp_mask = MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;
mlx5_wq_attr.striding_rq_attrs.single_stride_log_num_of_bytes = 10;
mlx5_wq_attr.striding_rq_attrs.single_wqe_log_num_of_strides = 4;
wq = mlx5dv_create_wq(context, &wq_init_attr, &mlx5_wq_attr);
#else
wq = ibv_create_wq(context, &wq_init_attr);
#endif
if (!wq) {
perror("Failed to create work queue"); // Failed to create work queue: Operation not supported
ret = 1;
goto cleanup;
}
printf("Work queue created successfully!\n");
if (ibv_destroy_wq(wq)) {
perror("Failed to destroy work queue");
ret = 1;
}
cleanup:
if (ibv_destroy_cq(cq)) {
perror("Failed to destroy completion queue");
ret = 1;
}
if (ibv_dealloc_pd(pd)) {
perror("Failed to deallocate protection domain");
ret = 1;
}
ibv_close_device(context);
ibv_free_device_list(dev_list);
return ret;
}
I would greatly appreciate any insights or guidance you could provide to resolve this issue or alternative methods to use the Multi-Packet RQ/Striding RQ feature with a QP if Work Queues are not supported in my case.
Thank you for your assistance and support!