I’m trying to experiment with Ethernet flow control (just global pause frames so far, no PFC). I’ve managed to make it work for the kernel stack using ethtool --set-priv-flags $iface dropless_rq on, but I’m having trouble with ibverbs raw packet QPs. When I set the IBV_WQ_FLAGS_DELAY_DROP flag when creating the WQ I get an EINVAL error, which after some tracing through the kernel module appears to be a BAD_PARAM error from the CREATE_RQ hardware command.
driver: mlx5_core
version: 5.1-2.3.7
firmware-version: 16.27.2008 (MT_0000000011)
C++ source code (run as root, it picks the first IB device):
#include <iostream>
#include <utility>
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <infiniband/verbs.h>
#include <unistd.h>
#define STRINGIFY(x) #x
// Helper macros to print an error and exit if a call fails
#define CHECK_PTR(expr) \
({ \
auto ret = (expr); \
if (ret == nullptr) \
{ \
std::cerr << STRINGIFY(expr) << " failed (errno = " << errno << ")\n"; \
std::exit(1); \
} \
ret; \
})
int main()
{
// Find the device
int num_devices;
ibv_device **devices = CHECK_PTR(ibv_get_device_list(&num_devices));
if (num_devices == 0)
{
std::cerr << "No infiniband devices found\n";
return 1;
}
ibv_device *device = devices[0];
std::cout << "Using " << ibv_get_device_name(device) << '\n';
ibv_context *ctx = CHECK_PTR(ibv_open_device(device));
ibv_free_device_list(devices);
// Allocate PD
ibv_pd *pd = CHECK_PTR(ibv_alloc_pd(ctx));
// Allocate CQ
ibv_cq_init_attr_ex cq_attr = {};
cq_attr.cqe = 1024;
cq_attr.wc_flags = IBV_WC_EX_WITH_BYTE_LEN;
ibv_cq_ex *cq_ex = CHECK_PTR(ibv_create_cq_ex(ctx, &cq_attr));
ibv_cq *cq = ibv_cq_ex_to_cq(cq_ex);
// Allocate WQ
ibv_wq_init_attr wq_init_attr = {};
wq_init_attr.wq_type = IBV_WQT_RQ;
wq_init_attr.max_wr = 1024;
wq_init_attr.max_sge = 1;
wq_init_attr.pd = pd;
wq_init_attr.cq = cq;
wq_init_attr.create_flags = IBV_WQ_FLAGS_DELAY_DROP;
wq_init_attr.comp_mask = IBV_WQ_INIT_ATTR_FLAGS;
CHECK_PTR(ibv_create_wq(ctx, &wq_init_attr));
std::cout << "Successfully created WQ\n";
}