Hi,
I’m working on steering traffic between DPDK application and Linux Kernel using Mellanox Bifurcated Driver (mlx5), I’m using rte_flow
API’s to define flow rules.
Specifically, I’m aiming to direct ICMP traffic to the Linux kernel, while steering all other traffic to the DPDK application.
static ::rte_flow* create_flow(uint16_t port_id, rte_flow_attr& attr, rte_flow_item& pattern, rte_flow_action& actions) {
rte_flow_error error;
auto flow = rte_flow_create(port_id, &attr, &pattern, &actions, &error);
if (!flow) {
spdlog::error("[create flow rule] failed to create flow rule, error: {}", error.message);
}
return flow;
}
bool validate_flow(uint16_t port_id, rte_flow_attr& attr, rte_flow_item& pattern, rte_flow_action& action) {
rte_flow_error error;
auto res = rte_flow_validate(port_id, &attr, &pattern, &action, &error);
if (res < 0) {
spdlog::error("[validate flow rule] validate flow failed, error: {}", error.message);
return false;
}
return true;
}
static void init_flow_rules(uint16_t port_id) {
rte_flow_attr attr = {0};
rte_flow_item pattern[4];
rte_flow_action action[2];
attr.ingress = 1;
memset(pattern, 0, sizeof pattern);
rte_flow_item_eth eth_spec;
rte_flow_item_eth eth_mask;
memset(ð_spec, 0, sizeof eth_spec);
memset(ð_mask, 0, sizeof eth_mask);
pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
pattern[0].spec = ð_spec;
pattern[0].mask = ð_mask;
rte_flow_item_ipv4 ipv4_spec;
rte_flow_item_ipv4 ipv4_mask;
uint32_t ip = ((0<<24) + (0<<16) + (0<<8) + 0);
uint32_t mask = 0xffffffff;
memset(&ipv4_spec, 0, sizeof ipv4_spec);
memset(&ipv4_mask, 0, sizeof ipv4_mask);
ipv4_spec.hdr.dst_addr = htonl(ip);
ipv4_mask.hdr.dst_addr = mask;
ipv4_spec.hdr.src_addr = htonl(ip);
ipv4_mask.hdr.src_addr = mask;
pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
pattern[1].spec = &ipv4_spec;
pattern[1].mask = &ipv4_mask;
rte_flow_item_icmp icmp_spec;
rte_flow_item_icmp icmp_mask;
memset(&icmp_spec, 0, sizeof icmp_spec);
memset(&icmp_mask, 0, sizeof icmp_mask);
icmp_spec.hdr.icmp_type = 8;
pattern[2].type = RTE_FLOW_ITEM_TYPE_ICMP;
pattern[2].spec = &icmp_spec;
pattern[2].mask = &icmp_mask;
pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
action[0].type = RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL;
action[1].type = RTE_FLOW_ACTION_TYPE_END;
if (validate_flow(port_id, attr, pattern[0], action[0])) {
spdlog::info("[init flow rule] validated successfully.");
auto flow = create_flow(port_id, attr, pattern[0], action[0]);
if (flow) {
spdlog::info("[rte flow rule] successfully created a flow rule in port {}", port_id);
}
}
}
on running the above program returns an error cannot create action
.
EAL: Detected CPU lcores: 112
EAL: Detected NUMA nodes: 2
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
mlx5_net: No available register for sampler.
mlx5_net: No available register for sampler.
[2024-10-25 01:30:51.934] [info] [main] Total available ports: 4
port 3 MAC: 04 3f 72 bf 13 87
[2024-10-25 01:30:52.224] [info] [init flow rule] validated successfully.
[2024-10-25 01:30:52.224] [error] [create flow rule] failed to create flow rule, error: hardware refuses to create flow
^C[2024-10-25 01:30:56.671] [info] [signal] signal received 2...
[2024-10-25 01:30:58.263] [info] [main] EAL cleanup.
using the action SEND_TO_KERNEL as documented in dpdk 24.07.
Is the above code correct? The code is validated successfully with rte_flow_validate()
, but failed when attempting to create the flow using rte_flow_create()
, what would be the reason?
Error rte_flow_create()
→ hardware refuses to create flow
Can someone please help me resolve this?
Below listed my system configuration.
cat /etc/os-release
PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
NAME="Debian GNU/Linux"
VERSION_ID="12"
VERSION="12 (bookworm)"
VERSION_CODENAME=bookworm
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"
cat /etc/debian_version
12.7
lspci | grep Mellanox
17:00.0 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
17:00.1 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
98:00.0 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
98:00.1 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5]
ethtool
driver: mlx5_core
version: 24.07-0.6.1
firmware-version: 16.35.1012 (MT_0000000425)
expansion-rom-version:
bus-info: 0000:98:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes
ofed_info
OFED-internal-24.07-0.6.1:
clusterkit:
mlnx_ofed_clusterkit/clusterkit-1.14.462-1.src.rpm
dpcp:
/sw/release/sw_acceleration/dpcp/dpcp-1.1.49-1.src.rpm
fwctl:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
hcoll:
mlnx_ofed_hcol/hcoll-4.8.3228-1.src.rpm
ibarr:
https://github.com/Mellanox/ip2gid master
commit 44ac1948d0d604c723bc36ade0af02c54e7fc7d2
ibdump:
https://github.com/Mellanox/ibdump master
commit d0a4f5aabf21580bee9ba956dfff755b1dd335c3
ibsim:
mlnx_ofed_ibsim/ibsim-0.12.tar.gz
ibutils2:
ibutils2/ibutils2-2.1.1-0.21800.MLNX20240804.ga4352587.tar.gz
iser:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
isert:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
kernel-mft:
mlnx_ofed_mft/kernel-mft-4.29.0-131.src.rpm
knem:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/knem.git mellanox-master
commit 0984cf2a2de70db5c6e6fff375b070eece37c39e
libvma:
vma/source_rpms//libvma-9.8.60-1.src.rpm
libxlio:
/sw/release/sw_acceleration/xlio/libxlio-3.31.2-1.src.rpm
mlnx-dpdk:
https://github.com/Mellanox/dpdk.org mlnx_dpdk_22.11_last_stable
commit 7636f8a75042528dba5873cfac9dde05f3044bec
mlnx-en:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
mlnx-ethtool:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/ethtool.git mlnx_ofed_24_07
commit 557f0ec6d8c6dc9feea69e22e078029e53eb00fd
mlnx-iproute2:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/iproute2.git mlnx_ofed_24_07
commit 38f1b11400ecb0e5873f61a464273099ce2023ff
mlnx-nfsrdma:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
mlnx-nvme:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
mlnx-ofa_kernel:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
mlnx-tools:
https://github.com/Mellanox/mlnx-tools mlnx_ofed
commit 6e00cb0f7354d415e3fdde991c87557fd204e110
mlx-steering-dump:
https://github.com/Mellanox/mlx_steering_dump mlnx_ofed_23_04
commit fc616d9a8f62113b0da6fc5a8948b11177d8461e
mpitests:
mlnx_ofed_mpitest/mpitests-3.2.24-2ffc2d6.src.rpm
multiperf:
https://git-nbu.nvidia.com/r/a/Performance/multiperf rdma-core-support
commit d3fad92dc6984e43cc5377ba0a3126808432ce2d
ofed-docs:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/ofed-docs.git mlnx_ofed-4.0
commit 3d1b0afb7bc190ae5f362223043f76b2b45971cc
openmpi:
mlnx_ofed_ompi_1.8/openmpi-4.1.7a1-1.src.rpm
opensm:
mlnx_ofed_opensm/opensm-5.20.0.MLNX20240804.ef1f438a.tar.gz
openvswitch:
https://gitlab-master.nvidia.com/sdn/ovs doca_2_6
commit 1a6f66e13713ff5c6861a540c6e014f8cd76c139
perftest:
mlnx_ofed_perftest/perftest-24.07.0-0.44.g57725f2.tar.gz
rdma-core:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/rdma-core.git mlnx_ofed_24_07
commit 1018f196dd0004464a8c986f5a9bebc1b03baa0a
rshim:
/sw_mc_soc_release/packages//rshim-2.0.41-0.g5e402e1.src.rpm
sharp:
mlnx_ofed_sharp/sharp-3.8.0.MLNX20240804.aaa5caab.tar.gz
sockperf:
sockperf/sockperf-3.10-0.git5ebd327da983.src.rpm
srp:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/mlnx-ofa_kernel-4.0.git mlnx_ofed_24_07
commit 51d9270cf01b5aaad07d70e42eba87dae4cafd8a
ucx:
mlnx_ofed_ucx/ucx-1.17.0-1.src.rpm
xpmem-lib:
/sw/release/mlnx_ofed/IBHPC/OFED-internal-23.10-0.5.5/SRPMS/xpmem-lib-2.7-0.2310055.src.rpm
xpmem:
https://git-nbu.nvidia.com/r/a/mlnx_ofed/xpmem.git mellanox-master
commit d3cc5b69e3363b14366c439d90fd33ded8c8da8f
Installed Packages:
-------------------
ii ibacm 2407mlnx52-1.2407061 amd64 InfiniBand Communication Manager Assistant (ACM)
ii ibverbs-providers:amd64 2407mlnx52-1.2407061 amd64 User space provider drivers for libibverbs
ii ibverbs-utils 2407mlnx52-1.2407061 amd64 Examples for the libibverbs library
ii libibumad-dev:amd64 2407mlnx52-1.2407061 amd64 Development files for libibumad
ii libibumad3:amd64 2407mlnx52-1.2407061 amd64 InfiniBand Userspace Management Datagram (uMAD) library
ii libibverbs-dev:amd64 2407mlnx52-1.2407061 amd64 Development files for the libibverbs library
ii libibverbs1:amd64 2407mlnx52-1.2407061 amd64 Library for direct userspace use of RDMA (InfiniBand/iWARP)
ii librdmacm-dev:amd64 2407mlnx52-1.2407061 amd64 Development files for the librdmacm library
ii librdmacm1:amd64 2407mlnx52-1.2407061 amd64 Library for managing RDMA connections
ii mlnx-ofed-kernel-dkms 24.07.OFED.24.07.0.6.1.1-1 all DKMS support for mlnx-ofed kernel modules
ii mlnx-ofed-kernel-utils 24.07.OFED.24.07.0.6.1.1-1 amd64 Userspace tools to restart and tune mlnx-ofed kernel modules
ii mlnx-tools 24.07.0-1.2407061 amd64 Userspace tools to restart and tune MLNX_OFED kernel modules
ii python3-pyverbs:amd64 2407mlnx52-1.2407061 amd64 Python bindings for rdma-core
ii rdma-core 2407mlnx52-1.2407061 amd64 RDMA core userspace infrastructure and documentation
ii rdmacm-utils 2407mlnx52-1.2407061 amd64 Examples for the librdmacm library
- Installed OFED by downloading the source from the
Debian 12.5 OFED
MLNX_OFED_SRC-debian-24.07-0.6.1.0.tgz
- installed mlx5 libraries.
ls -l /usr/local/lib/x86_64-linux-gnu/ | grep mlx
(DPDK with pmd enabled)
-rw-r--r-- 1 root root 291574 Oct 8 23:55 librte_common_mlx5.a
lrwxrwxrwx 1 root root 36 Oct 8 23:57 librte_common_mlx5.so -> dpdk/pmds-24.2/librte_common_mlx5.so
lrwxrwxrwx 1 root root 39 Oct 8 23:57 librte_common_mlx5.so.24 -> dpdk/pmds-24.2/librte_common_mlx5.so.24
lrwxrwxrwx 1 root root 41 Oct 8 23:57 librte_common_mlx5.so.24.2 -> dpdk/pmds-24.2/librte_common_mlx5.so.24.2
-rw-r--r-- 1 root root 27630 Oct 8 23:55 librte_compress_mlx5.a
lrwxrwxrwx 1 root root 38 Oct 8 23:57 librte_compress_mlx5.so -> dpdk/pmds-24.2/librte_compress_mlx5.so
lrwxrwxrwx 1 root root 41 Oct 8 23:57 librte_compress_mlx5.so.24 -> dpdk/pmds-24.2/librte_compress_mlx5.so.24
lrwxrwxrwx 1 root root 43 Oct 8 23:57 librte_compress_mlx5.so.24.2 -> dpdk/pmds-24.2/librte_compress_mlx5.so.24.2
-rw-r--r-- 1 root root 70438 Oct 8 23:55 librte_crypto_mlx5.a
lrwxrwxrwx 1 root root 36 Oct 8 23:57 librte_crypto_mlx5.so -> dpdk/pmds-24.2/librte_crypto_mlx5.so
lrwxrwxrwx 1 root root 39 Oct 8 23:57 librte_crypto_mlx5.so.24 -> dpdk/pmds-24.2/librte_crypto_mlx5.so.24
lrwxrwxrwx 1 root root 41 Oct 8 23:57 librte_crypto_mlx5.so.24.2 -> dpdk/pmds-24.2/librte_crypto_mlx5.so.24.2
-rw-r--r-- 1 root root 199576 Oct 8 23:55 librte_net_mlx4.a
lrwxrwxrwx 1 root root 33 Oct 8 23:57 librte_net_mlx4.so -> dpdk/pmds-24.2/librte_net_mlx4.so
lrwxrwxrwx 1 root root 36 Oct 8 23:57 librte_net_mlx4.so.24 -> dpdk/pmds-24.2/librte_net_mlx4.so.24
lrwxrwxrwx 1 root root 38 Oct 8 23:57 librte_net_mlx4.so.24.2 -> dpdk/pmds-24.2/librte_net_mlx4.so.24.2
-rw-r--r-- 1 root root 2847096 Oct 8 23:55 librte_net_mlx5.a
lrwxrwxrwx 1 root root 33 Oct 8 23:57 librte_net_mlx5.so -> dpdk/pmds-24.2/librte_net_mlx5.so
lrwxrwxrwx 1 root root 36 Oct 8 23:57 librte_net_mlx5.so.24 -> dpdk/pmds-24.2/librte_net_mlx5.so.24
lrwxrwxrwx 1 root root 38 Oct 8 23:57 librte_net_mlx5.so.24.2 -> dpdk/pmds-24.2/librte_net_mlx5.so.24.2
-rw-r--r-- 1 root root 41608 Oct 8 23:55 librte_regex_mlx5.a
lrwxrwxrwx 1 root root 35 Oct 8 23:57 librte_regex_mlx5.so -> dpdk/pmds-24.2/librte_regex_mlx5.so
lrwxrwxrwx 1 root root 38 Oct 8 23:57 librte_regex_mlx5.so.24 -> dpdk/pmds-24.2/librte_regex_mlx5.so.24
lrwxrwxrwx 1 root root 40 Oct 8 23:57 librte_regex_mlx5.so.24.2 -> dpdk/pmds-24.2/librte_regex_mlx5.so.24.2
-rw-r--r-- 1 root root 132624 Oct 8 23:55 librte_vdpa_mlx5.a
lrwxrwxrwx 1 root root 34 Oct 8 23:57 librte_vdpa_mlx5.so -> dpdk/pmds-24.2/librte_vdpa_mlx5.so
lrwxrwxrwx 1 root root 37 Oct 8 23:57 librte_vdpa_mlx5.so.24 -> dpdk/pmds-24.2/librte_vdpa_mlx5.so.24
lrwxrwxrwx 1 root root 39 Oct 8 23:57 librte_vdpa_mlx5.so.24.2 -> dpdk/pmds-24.2/librte_vdpa_mlx5.so.24.2
ls -l /usr/lib/x86_64-linux-gnu/ | grep mlx
-rw-r--r-- 1 root root 75686 May 27 06:08 libmlx4.a
lrwxrwxrwx 1 root root 12 May 27 06:08 libmlx4.so -> libmlx4.so.1
lrwxrwxrwx 1 root root 19 May 27 06:08 libmlx4.so.1 -> libmlx4.so.1.0.52.0
-rw-r--r-- 1 root root 51192 May 27 06:08 libmlx4.so.1.0.52.0
-rw-r--r-- 1 root root 777402 May 27 06:08 libmlx5.a
lrwxrwxrwx 1 root root 12 May 27 06:08 libmlx5.so -> libmlx5.so.1
lrwxrwxrwx 1 root root 20 May 27 06:08 libmlx5.so.1 -> libmlx5.so.1.24.52.0
-rw-r--r-- 1 root root 512376 May 27 06:08 libmlx5.so.1.24.52.0