Hi,
I would like to share my observation related to the mlx5_core driver (version 26.01-1.0.0) and the old ConnectX-4 Lx (MT4117) . Not sure if this is the latest one or whether it has already been resolved, but using the below flow one can crash the driver completely:
$ modinfo mlx5_core
filename: /lib/modules/6.8.0-60-generic/updates/dkms/mlx5_core.ko.zst
alias: auxiliary:mlx5_core.eth-rep
alias: auxiliary:mlx5_core.eth
supported: external
basedon: Korg 6.18-rc4
version: 26.01-1.0.0
...
$ sudo mst start
$ sudo mlxconfig -d /dev/mst/mt4117_pciconf0 q
Device #1:
----------
Device type: ConnectX4LX
Name: MCX4121A-XCA_Ax
Description: ConnectX-4 Lx EN network interface card; 10GbE dual-port SFP28; PCIe3.0 x8; ROHS R6
Device: /dev/mst/mt4117_pciconf0
Configurations: Next Boot
PORT_OWNER True(1)
ALLOW_RD_COUNTERS True(1)
RENEG_ON_CHANGE True(1)
TRACER_ENABLE True(1)
FLEX_PARSER_PROFILE_ENABLE 0
FLEX_IPV4_OVER_VXLAN_PORT 0
SWITCH_COMPT_FEATURE_MASK 0x0(0)
PF_NUM_OF_VF_VALID False(0)
NON_PREFETCHABLE_PF_BAR False(0)
VF_VPD_ENABLE False(0)
STRICT_VF_MSIX_NUM False(0)
VF_NODNIC_ENABLE False(0)
NUM_PF_MSIX_VALID True(1)
NUM_OF_VFS 8
NUM_OF_PF 2
SRIOV_EN True(1)
PF_LOG_BAR_SIZE 5
VF_LOG_BAR_SIZE 0
NUM_PF_MSIX 63
NUM_VF_MSIX 11
PCIE_CREDIT_TOKEN_TIMEOUT 0
PCI_BUS0_RESTRICT_SPEED PCI_GEN_1(0)
PCI_BUS0_RESTRICT_ASPM False(0)
PCI_BUS0_RESTRICT_WIDTH PCI_X1(0)
PCI_BUS0_RESTRICT False(0)
PCI_DOWNSTREAM_PORT_OWNER Array[0..15]
ROCE_CC_RTT_TIMESTAMP_FORMAT DEVICE_DEFAULT(0)
ROCE_CONTROL ROCE_ENABLE(2)
PCI_WR_ORDERING per_mkey(0)
MULTI_PORT_VHCA_EN False(0)
LOG_DCR_HASH_TABLE_SIZE 14
MAX_PACKET_LIFETIME 0
DCR_LIFO_SIZE 16384
IB_PROTO_WIDTH_EN_MASK_P1 0
IB_PROTO_WIDTH_EN_MASK_P2 0
KEEP_ETH_LINK_UP_P1 True(1)
KEEP_IB_LINK_UP_P1 False(0)
KEEP_LINK_UP_ON_BOOT_P1 False(0)
KEEP_LINK_UP_ON_STANDBY_P1 False(0)
DO_NOT_CLEAR_PORT_STATS_P1 False(0)
AUTO_POWER_SAVE_LINK_DOWN_P1 False(0)
KEEP_ETH_LINK_UP_P2 True(1)
KEEP_IB_LINK_UP_P2 False(0)
KEEP_LINK_UP_ON_BOOT_P2 False(0)
KEEP_LINK_UP_ON_STANDBY_P2 False(0)
DO_NOT_CLEAR_PORT_STATS_P2 False(0)
AUTO_POWER_SAVE_LINK_DOWN_P2 False(0)
PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0)
PHY_FEC_OVERRIDE_P2 DEVICE_DEFAULT(0)
LLDP_NB_DCBX_P1 False(0)
LLDP_NB_RX_MODE_P1 OFF(0)
LLDP_NB_TX_MODE_P1 OFF(0)
LLDP_NB_DCBX_P2 False(0)
LLDP_NB_RX_MODE_P2 OFF(0)
LLDP_NB_TX_MODE_P2 OFF(0)
DCBX_IEEE_P1 True(1)
DCBX_CEE_P1 True(1)
DCBX_WILLING_P1 True(1)
DCBX_IEEE_P2 True(1)
DCBX_CEE_P2 True(1)
DCBX_WILLING_P2 True(1)
MEMIC_BAR_SIZE 0
MEMIC_SIZE_LIMIT _256KB(1)
DUP_MAC_ACTION_P1 LAST_CFG(0)
SRIOV_IB_ROUTING_MODE_P1 LID(1)
IB_ROUTING_MODE_P1 LID(1)
DUP_MAC_ACTION_P2 LAST_CFG(0)
SRIOV_IB_ROUTING_MODE_P2 LID(1)
IB_ROUTING_MODE_P2 LID(1)
NUM_OF_PLANES_P1 0
LOAD_BALANCE_MODE_P1 DEVICE_DEFAULT(0)
NUM_OF_PLANES_P2 0
LOAD_BALANCE_MODE_P2 DEVICE_DEFAULT(0)
DYNAMIC_VF_MSIX_TABLE False(0)
EXP_ROM_UEFI_ARM_ENABLE False(0)
EXP_ROM_UEFI_x86_ENABLE False(0)
EXP_ROM_PXE_ENABLE True(1)
ADVANCED_PCI_SETTINGS False(0)
ICM_CACHE_MODE DEVICE_DEFAULT(0)
TX_SCHEDULER_BURST 0
PF_SD_GROUP 0
ACCURATE_TX_SCHEDULER False(0)
PARTIAL_RESET_EN False(0)
SW_RECOVERY_ON_ERRORS False(0)
RESET_WITH_HOST_ON_ERRORS False(0)
NUM_OF_VL_P1 _4_VLs(3)
NUM_OF_TC_P1 _8_TCs(0)
NUM_OF_PFC_P1 8
VL15_BUFFER_SIZE_P1 0
NUM_OF_VL_P2 _4_VLs(3)
NUM_OF_TC_P2 _8_TCs(0)
NUM_OF_PFC_P2 8
VL15_BUFFER_SIZE_P2 0
ROCE_NEXT_PROTOCOL 254
ROCE_CC_PRIO_MASK_P1 255
ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0)
ROCE_CC_PRIO_MASK_P2 255
ROCE_CC_CNP_MODERATION_P2 DEVICE_DEFAULT(0)
CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1)
CLAMP_TGT_RATE_P1 False(0)
RPG_TIME_RESET_P1 300
RPG_BYTE_RESET_P1 32767
RPG_THRESHOLD_P1 1
RPG_MAX_RATE_P1 0
RPG_AI_RATE_P1 5
RPG_HAI_RATE_P1 50
RPG_GD_P1 11
RPG_MIN_DEC_FAC_P1 50
RPG_MIN_RATE_P1 1
RATE_TO_SET_ON_FIRST_CNP_P1 0
DCE_TCP_G_P1 1019
DCE_TCP_RTT_P1 1
RATE_REDUCE_MONITOR_PERIOD_P1 4
INITIAL_ALPHA_VALUE_P1 1023
MIN_TIME_BETWEEN_CNPS_P1 4
CNP_802P_PRIO_P1 6
CNP_DSCP_MODE_P1 DEVICE_DEFAULT(0)
CNP_DSCP_P1 48
CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1)
CLAMP_TGT_RATE_P2 False(0)
RPG_TIME_RESET_P2 300
RPG_BYTE_RESET_P2 32767
RPG_THRESHOLD_P2 1
RPG_MAX_RATE_P2 0
RPG_AI_RATE_P2 5
RPG_HAI_RATE_P2 50
RPG_GD_P2 11
RPG_MIN_DEC_FAC_P2 50
RPG_MIN_RATE_P2 1
RATE_TO_SET_ON_FIRST_CNP_P2 0
DCE_TCP_G_P2 1019
DCE_TCP_RTT_P2 1
RATE_REDUCE_MONITOR_PERIOD_P2 4
INITIAL_ALPHA_VALUE_P2 1023
MIN_TIME_BETWEEN_CNPS_P2 4
CNP_802P_PRIO_P2 6
CNP_DSCP_MODE_P2 DEVICE_DEFAULT(0)
CNP_DSCP_P2 48
ROCE_RTT_RESP_DSCP_P1 0
ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0)
ROCE_RTT_RESP_DSCP_P2 0
ROCE_RTT_RESP_DSCP_MODE_P2 DEVICE_DEFAULT(0)
BOOT_VLAN 1
LEGACY_BOOT_PROTOCOL PXE(1)
BOOT_INTERRUPT_DIS False(0)
BOOT_LACP_DIS True(1)
BOOT_VLAN_EN False(0)
IP_VER IPv4(0)
BOOT_UNDI_NETWORK_WAIT 0
BOOT_DBG_LOG False(0)
BOOT_PKEY 0
UEFI_HII_EN True(1)
UEFI_LOGS DISABLED(0)
LOG_MAX_QUEUE 17
CQE_COMPRESSION BALANCED(0)
IP_OVER_VXLAN_EN False(0)
MKEY_BY_NAME False(0)
UCTX_EN True(1)
PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
TUNNEL_ECN_COPY_DISABLE False(0)
LRO_LOG_TIMEOUT0 6
LRO_LOG_TIMEOUT1 7
LRO_LOG_TIMEOUT2 8
LRO_LOG_TIMEOUT3 13
INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0)
TPH_STEERING_TAG_TABLE DEVICE_DEFAULT(0)
SAFE_MODE_THRESHOLD 10
SAFE_MODE_ENABLE True(1)
$ sudo flint -d 0b:00.0 q
Image type: FS3
FW Version: 14.32.1010
FW Release Date: 1.12.2021
Product Version: 14.32.1010
Rom Info: type=UEFI version=14.25.17 cpu=AMD64,AARCH64
type=PXE version=3.6.502 cpu=AMD64
Description: UID GuidsNumber
Base GUID: ec0d9a0300d99b52 4
Base MAC: ec0d9ad99b52 4
Image VSD: N/A
Device VSD: N/A
PSID: MT_2420110004
Security Attributes: N/A
#
# My custom script with ping test between the two interfaces (each in a separate namespace, ports still in legacy mode)
#
=== Testing connectivity ===
Ping from ns1 to ns2...
PING 192.168.168.2 (192.168.168.2) 56(84) bytes of data.
64 bytes from 192.168.168.2: icmp_seq=1 ttl=64 time=0.152 ms
64 bytes from 192.168.168.2: icmp_seq=2 ttl=64 time=0.089 ms
64 bytes from 192.168.168.2: icmp_seq=3 ttl=64 time=0.077 ms
$ sudo devlink dev eswitch set pci/0000:0b:00.0 mode switchdev
$ sudo devlink dev eswitch set pci/0000:0b:00.1 mode switchdev
$ sudo devlink dev eswitch set pci/0000:0b:00.0 inline-mode network
$ sudo devlink dev eswitch set pci/0000:0b:00.1 inline-mode network
# Enable VFs
$ echo '2' | sudo tee /sys/bus/pci/devices/0000:0b:00.0/sriov_numvfs
$ echo '2' | sudo tee /sys/bus/pci/devices/0000:0b:00.1/sriov_numvfs
# Unbind VFs
$ echo '0b:00.2' > sudo tee /sys/bus/pci/drivers/mlx5_core/unbind
$ echo '0b:00.3' > sudo tee /sys/bus/pci/drivers/mlx5_core/unbind
$ echo '0b:01.2' > sudo tee /sys/bus/pci/drivers/mlx5_core/unbind
$ echo '0b:01.3' > sudo tee /sys/bus/pci/drivers/mlx5_core/unbind
$ sudo ethtool -K enp11s0f1r0 hw-tc-offload on
$ sudo ethtool -K enp11s0f1r1 hw-tc-offload on
$ sudo -i
# cd /opt/mellanox/iproute2/sbin/
# devlink dev eswitch show pci/0000:0b:00.0
pci/0000:0b:00.0: mode switchdev inline-mode network encap-mode basic
# ./tc filter del dev enp11s0f1r0 ingress
# ./tc filter del dev enp11s0f1r1 ingress
# ./tc qdisc add dev enp11s0f1r0 ingress
# ./tc qdisc add dev enp11s0f1r1 ingress
# ./tc filter add dev enp11s0f1r0 ingress protocol ip flower \
skip_sw \
dst_ip 10.0.2.50 \
action mirred egress redirect dev enp11s0f1r1
Killed
#
# Plan: Try to go revert to legacy mode and get the ping test running again
#
$ echo '0' | sudo tee /sys/bus/pci/devices/0000:0b:00.0/sriov_numvfs
$ echo '0' | sudo tee /sys/bus/pci/devices/0000:0b:00.1/sriov_numvfs
$ sudo ethtool -K enp11s0f0np0 hw-tc-offload off
$ sudo ethtool -K enp11s0f1np1 hw-tc-offload off
$ sudo devlink dev eswitch set pci/0000:0b:00.0 mode legacy
$ sudo devlink dev eswitch set pci/0000:0b:00.1 mode legacy
#
# Reality (part I)
#
$ echo '0' | sudo tee /sys/bus/pci/devices/0000:0b:00.0/sriov_numvfs # Hangs here and cannot continue
[ 4421.321717] mlx5_core 0000:0b:00.0: E-Switch: Unload vfs: mode(OFFLOADS), nvfs(2), necvfs(0), active vports(2)
[ 4431.369465] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4441.569811] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4451.770174] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4461.778504] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4471.930849] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4482.075351] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4492.243500] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4502.395831] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4512.564335] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4522.796491] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4533.028802] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4543.213104] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4553.429420] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4563.645712] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
[ 4573.822015] unregister_netdevice: waiting for enp11s0f0r0 to become free. Usage count = 2
#
# Reality (part II)
#
$ sudo devlink dev eswitch set pci/0000:0b:00.0 mode legacy # Hangs here and cannot continue as well
#
# Reality (part III) - system requires power off or hard reboot
#
Inside dmesg the crash looks as follows:
Dump:
[ 219.405758] Mirror/redirect action on
[ 219.407188] BUG: kernel NULL pointer dereference, address: 0000000000000000
[ 219.407191] #PF: supervisor read access in kernel mode
[ 219.407194] #PF: error_code(0x0000) - not-present page
[ 219.407195] PGD 0 P4D 0
[ 219.407199] Oops: 0000 [#1] PREEMPT SMP NOPTI
[ 219.407202] CPU: 3 PID: 8284 Comm: tc Tainted: G OE 6.8.0-60-generic #63-Ubuntu
[ 219.407205] Hardware name: ASUS System Product Name/Pro B650M-CT, BIOS 3024 08/02/2024
[ 219.407206] RIP: 0010:mlx5_fs_pool_release_index+0x2e/0x130 [mlx5_core]
[ 219.407271] Code: 00 55 48 89 e5 41 56 4c 8d 77 18 41 55 49 89 f5 41 54 53 4c 8b 26 48 89 fb 4c 89 f7 e8 7b 06 2f d5 49 63 45 08 49 8b 54 24 18 <48> 0f a3 02 0f 82 ed 00 00 00 49 8b 54 24 18 f0 48 0f ab 02 83 43
[ 219.407273] RSP: 0018:ffff9785e859b2b8 EFLAGS: 00010246
[ 219.407276] RAX: 0000000000000000 RBX: ffff8c72408bee90 RCX: 0000000000000000
[ 219.407278] RDX: 0000000000000000 RSI: ffff9785e859b2e8 RDI: 0000000000000000
[ 219.407279] RBP: ffff9785e859b2d8 R08: 0000000000000000 R09: 0000000000000000
[ 219.407281] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8c72503acea0
[ 219.407282] R13: ffff9785e859b2e8 R14: ffff8c72408beea8 R15: 0000000000000000
[ 219.407284] FS: 000073d4d8251740(0000) GS:ffff8c799db80000(0000) knlGS:0000000000000000
[ 219.407286] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 219.407288] CR2: 0000000000000000 CR3: 00000001c9c02000 CR4: 0000000000f50ef0
[ 219.407289] PKRU: 55555554
[ 219.407291] Call Trace:
[ 219.407293] <TASK>
[ 219.407296] ? show_regs+0x6d/0x80
[ 219.407301] ? __die+0x24/0x80
[ 219.407304] ? page_fault_oops+0x99/0x1b0
[ 219.407309] ? do_user_addr_fault+0x2ea/0x670
[ 219.407313] ? exc_page_fault+0x83/0x1b0
[ 219.407317] ? asm_exc_page_fault+0x27/0x30
[ 219.407323] ? mlx5_fs_pool_release_index+0x2e/0x130 [mlx5_core]
[ 219.407373] ? mlx5_fs_pool_release_index+0x25/0x130 [mlx5_core]
[ 219.407422] mlx5_fc_release+0xc9/0x110 [mlx5_core]
[ 219.407476] mlx5_fc_destroy+0x42/0x70 [mlx5_core]
[ 219.407526] mlx5_free_flow_attr_actions+0x24e/0x2b0 [mlx5_core]
[ 219.407594] mlx5e_tc_del_fdb_flow+0x1d1/0x310 [mlx5_core]
[ 219.407650] mlx5e_tc_del_flow+0x60/0x2c0 [mlx5_core]
[ 219.407698] mlx5e_flow_put+0x3a/0x90 [mlx5_core]
[ 219.407746] __mlx5e_add_fdb_flow+0x10d/0x4b0 [mlx5_core]
[ 219.407795] mlx5e_configure_flower+0x7bf/0x1470 [mlx5_core]
[ 219.407843] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.407845] ? finish_task_switch.isra.0+0x93/0x300
[ 219.407851] mlx5e_rep_setup_tc_cls_flower+0x37/0xa0 [mlx5_core]
[ 219.407901] mlx5e_rep_setup_tc_cb+0x88/0x1a0 [mlx5_core]
[ 219.407948] tc_setup_cb_add+0xea/0x210
[ 219.407953] fl_hw_replace_filter+0x157/0x210 [cls_flower]
[ 219.407959] fl_change+0xcce/0xfe0 [cls_flower]
[ 219.407962] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.407964] ? clsact_chain_head_change+0x17/0x30 [sch_ingress]
[ 219.407969] tc_new_tfilter+0x400/0xaf0
[ 219.407971] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.407978] ? __pfx_tc_new_tfilter+0x10/0x10
[ 219.407980] rtnetlink_rcv_msg+0x369/0x440
[ 219.407984] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.407986] ? ext4_match+0x102/0x150
[ 219.407990] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.407993] ? __pfx_rtnetlink_rcv_msg+0x10/0x10
[ 219.407995] netlink_rcv_skb+0x5a/0x110
[ 219.408001] rtnetlink_rcv+0x15/0x30
[ 219.408004] netlink_unicast+0x24a/0x390
[ 219.408007] netlink_sendmsg+0x214/0x470
[ 219.408011] ____sys_sendmsg+0x38b/0x3f0
[ 219.408014] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408018] ___sys_sendmsg+0x9a/0xf0
[ 219.408026] __sys_sendmsg+0x89/0xf0
[ 219.408031] __x64_sys_sendmsg+0x1d/0x30
[ 219.408033] x64_sys_call+0x91e/0x25a0
[ 219.408036] do_syscall_64+0x7f/0x180
[ 219.408041] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408043] ? __count_memcg_events+0x6b/0x120
[ 219.408046] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408049] ? count_memcg_events.constprop.0+0x2a/0x50
[ 219.408052] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408055] ? handle_mm_fault+0xad/0x380
[ 219.408058] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408060] ? do_user_addr_fault+0x334/0x670
[ 219.408063] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408065] ? irqentry_exit_to_user_mode+0x7b/0x260
[ 219.408068] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408070] ? irqentry_exit+0x43/0x50
[ 219.408073] ? srso_alias_return_thunk+0x5/0xfbef5
[ 219.408075] ? exc_page_fault+0x94/0x1b0
[ 219.408077] entry_SYSCALL_64_after_hwframe+0x78/0x80
[ 219.408080] RIP: 0033:0x73d4d812c034
[ 219.408096] Code: 15 e9 6d 0d 00 f7 d8 64 89 02 b8 ff ff ff ff eb bf 0f 1f 44 00 00 f3 0f 1e fa 80 3d 15 f0 0d 00 00 74 13 b8 2e 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 4c c3 0f 1f 00 55 48 89 e5 48 83 ec 20 89 55
[ 219.408098] RSP: 002b:00007ffef5767fd8 EFLAGS: 00000202 ORIG_RAX: 000000000000002e
[ 219.408101] RAX: ffffffffffffffda RBX: 00005a8cdc5f2780 RCX: 000073d4d812c034
[ 219.408103] RDX: 0000000000000000 RSI: 00007ffef5768040 RDI: 0000000000000003
[ 219.408104] RBP: 00007ffef57680b0 R08: 0000000000000010 R09: 0000000000000064
[ 219.408106] R10: 0000000000000002 R11: 0000000000000202 R12: 00007ffef5768120
[ 219.408107] R13: 00005a8cdc5f84e0 R14: 0000000000000066 R15: 000000006a2b065d
[ 219.408111] </TASK>
[ 219.408112] Modules linked in: act_mirred cls_flower sch_ingress mst_pciconf(OE) xt_nat veth nf_conntrack_netlink xfrm_user xfrm_algo vxlan ip6_udp_tunnel udp_tunnel snd_seq_dummy snd_hrtimer bridge stp llc overlay qrtr rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) nvidia_uvm(OE) nft_chain_nat xt_MASQUERADE nf_nat xt_addrtype ipt_REJECT nf_reject_ipv4 xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 xt_CHECKSUM xt_tcpudp nft_compat nf_tables libcrc32c binfmt_misc nls_iso8859_1 nvidia_drm(OE) intel_rapl_msr intel_rapl_common nvidia_modeset(OE) snd_hda_codec_realtek snd_hda_codec_generic snd_hda_codec_hdmi edac_mce_amd kvm_amd snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi snd_hda_codec kvm snd_hda_core snd_hwdep snd_pcm irqbypass crct10dif_pclmul polyval_clmulni polyval_generic ghash_clmulni_intel sha256_ssse3 snd_seq_midi sha1_ssse3 nvidia(OE) snd_seq_midi_event aesni_intel snd_rawmidi crypto_simd eeepc_wmi mfd_aaeon cryptd asus_wmi snd_seq rapl ledtrig_audio snd_seq_device
[ 219.408173] sparse_keymap snd_timer platform_profile wmi_bmof snd mlx5_fwctl(OE) k10temp ccp i2c_piix4 soundcore fwctl(OE) ecc input_leds joydev gpio_amdpt mac_hid sch_fq_codel msr parport_pc ppdev lp parport efi_pstore nfnetlink dmi_sysfs ip_tables x_tables autofs4 mlx5_ib(OE) ib_uverbs(OE) macsec ib_core(OE) hid_generic mlx5_core(OE) usbhid hid mlxfw(OE) psample nvme video crc32_pclmul mlxdevm(OE) r8169 ahci tls libahci realtek xhci_pci nvme_core mlx_compat(OE) xhci_pci_renesas nvme_auth pci_hyperv_intf wmi [last unloaded: mst_pci(OE)]
[ 219.408216] CR2: 0000000000000000
[ 219.408218] ---[ end trace 0000000000000000 ]---
[ 219.527836] RIP: 0010:mlx5_fs_pool_release_index+0x2e/0x130 [mlx5_core]
[ 219.527903] Code: 00 55 48 89 e5 41 56 4c 8d 77 18 41 55 49 89 f5 41 54 53 4c 8b 26 48 89 fb 4c 89 f7 e8 7b 06 2f d5 49 63 45 08 49 8b 54 24 18 <48> 0f a3 02 0f 82 ed 00 00 00 49 8b 54 24 18 f0 48 0f ab 02 83 43
[ 219.527905] RSP: 0018:ffff9785e859b2b8 EFLAGS: 00010246
[ 219.527909] RAX: 0000000000000000 RBX: ffff8c72408bee90 RCX: 0000000000000000
[ 219.527911] RDX: 0000000000000000 RSI: ffff9785e859b2e8 RDI: 0000000000000000
[ 219.527912] RBP: ffff9785e859b2d8 R08: 0000000000000000 R09: 0000000000000000
[ 219.527913] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8c72503acea0
[ 219.527915] R13: ffff9785e859b2e8 R14: ffff8c72408beea8 R15: 0000000000000000
[ 219.527917] FS: 000073d4d8251740(0000) GS:ffff8c799db80000(0000) knlGS:0000000000000000
[ 219.527919] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 219.527920] CR2: 0000000000000000 CR3: 00000001c9c02000 CR4: 0000000000f50ef0
[ 219.527922] PKRU: 55555554
[ 219.527924] note: tc[8284] exited with irqs disabled