Hi,
I am working with an NVIDIA BlueField-2 P-Series DPU (25GbE Dual-Port, PCIe Gen4 x8, Crypto and Secure Boot enabled).
I have configured multiple Subfunctions (SFs) using mlxdevm and devlink tools on the ARM side of the DPU. Interfaces such as en3f0pf0sf0 through en3f0pf0sf7 appear and are functional inside the DPU, and I am able to add them to OVS bridges.
However, after rebooting the host system (x86 Linux/ESXi), only the SF0 interface (en3f0pf0sf0) is persistently exposed to the host as a VF or usable network interface. The other SF interfaces (en3f0pf0sf1 to en3f0pf0sf`) are not exposed and disappear.
My goal is to have all created SFs, especially SF1 through SF7, persistently exposed to the host as VFs so that they can be configured with IP addresses and managed by the host OS.
Below is some relevant information about my environment and configuration:
Environment:
- DPU: BlueField-2 P-Series
- Firmware version:
- ARM OS: Ubuntu 22.04
- Host OS: Ubuntu 22.04 with DOCA
- OVS version: 3.0.0-0056-25.04-based-3.3.5
Commands and outputs:
Relevant details:
- Total SFs available: 8
- Created 8 SFs (sf0 to sf7)
- After reboot, only SF0 is present on the host. The other SFs are missing, and OVS reports errors such as “could not open network device en3f0pf0sfX (No such device)” for SF1 through SF7.
Could you please advise on what configuration or steps are required to expose all created SFs persistently to the host, and ensure they behave as proper VFs for networking purposes?
Could you please clarify why SF creation is limited to PF0? Is this expected behavior or is there a configuration step I am missing?
root@bf2:/home/ubuntu# mst status
MST modules:
MST PCI module is not loaded
MST PCI configuration module loaded
MST devices:
/dev/mst/mt41686_pciconf0 - PCI configuration cycles access.
domain:bus:dev.fn=0000:03:00.0 addr.reg=88 data.reg=92 cr_bar.gw_offset=-1
Chip revision is: 01
Device type: BlueField2
Name: MBF2H332A-AECO_Ax_Bx
Description: BlueField-2 P-Series DPU 25GbE Dual-Port SFP56; PCIe Gen4 x8; Crypto and Secure Boot Enabled; 16GB on-board DDR; 1GbE OOB management; HHHL
Device: /dev/mst/mt41686_pciconf0
Configurations: Next Boot
LARGE_MTU_TWEAK_64 False(0)
AES_XTS_TWEAK_INC_64 False(0)
CRYPTO_POLICY UNRESTRICTED(1)
NVME_EMULATION_ENABLE False(0)
NVME_EMULATION_NUM_VF 0
NVME_EMULATION_NUM_PF 1
NVME_EMULATION_VENDOR_ID 5555
NVME_EMULATION_DEVICE_ID 24577
NVME_EMULATION_CLASS_CODE 67586
NVME_EMULATION_REVISION_ID 0
NVME_EMULATION_SUBSYSTEM_VENDOR_ID 0
NVME_EMULATION_SUBSYSTEM_ID 0
NVME_EMULATION_NUM_MSIX 0
NVME_EMULATION_MAX_QUEUE_DEPTH 0
PCI_SWITCH_EMULATION_NUM_PORT 0
VIRTIO_EMULATION_HOTPLUG_TRANS False(0)
PCI_SWITCH_EMULATION_ENABLE False(0)
UPT_EMULATION_ENABLE True(1)
VIRTIO_BLK_EMULATION_VF_PCI_LAYOUT VIRTIO_1_X(0)
VIRTIO_BLK_EMULATION_PF_PCI_LAYOUT VIRTIO_1_X(0)
VIRTIO_BLK_EMULATION_ENABLE False(0)
VIRTIO_BLK_EMULATION_NUM_VF 0
VIRTIO_BLK_EMULATION_NUM_PF 0
VIRTIO_BLK_EMU_SUBS_VENDOR_ID 6900
VIRTIO_BLK_EMULATION_SUBSYSTEM_ID 4162
VIRTIO_BLK_EMULATION_NUM_MSIX 2
VIRTIO_NET_EMULATION_VF_PCI_LAYOUT VIRTIO_1_X(0)
VIRTIO_NET_EMULATION_PF_PCI_LAYOUT VIRTIO_1_X(0)
VIRTIO_NET_EMULATION_ENABLE False(0)
VIRTIO_NET_EMULATION_NUM_VF 0
VIRTIO_NET_EMULATION_NUM_PF 0
VIRTIO_NET_EMU_SUBSYSTEM_VENDOR_ID 6900
VIRTIO_NET_EMULATION_SUBSYSTEM_ID 4161
VIRTIO_NET_EMULATION_NUM_MSIX 2
HOST_PRIV_FLASH_ACCESS DEVICE_DEFAULT(0)
HOST_PRIV_NV_HOST DEVICE_DEFAULT(0)
HOST_PRIV_NV_PORT DEVICE_DEFAULT(0)
HOST_PRIV_NV_GLOBAL DEVICE_DEFAULT(0)
HOST_PRIV_NV_INTERNAL_CPU DEVICE_DEFAULT(0)
HOST_PRIV_PCC_UPDATE DEVICE_DEFAULT(0)
HOST_PRIV_FW_UPDATE DEVICE_DEFAULT(0)
HOST_PRIV_NIC_RESET DEVICE_DEFAULT(0)
FLEX_PARSER_PROFILE_ENABLE 0
PROG_PARSE_GRAPH False(0)
FLEX_IPV4_OVER_VXLAN_PORT 0
SWITCH_COMPT_FEATURE_MASK 0x0(0)
PF_BAR2_SIZE 0
DPU_RESET_NOTIFICATION_ENABLED DISABLED(0)
INTERNAL_CPU_RSHIM ENABLED(0)
PF_NUM_OF_VF_VALID True(1)
NON_PREFETCHABLE_PF_BAR False(0)
VF_VPD_ENABLE False(0)
PF_NUM_PF_MSIX_VALID False(0)
PER_PF_NUM_SF True(1)
STRICT_VF_MSIX_NUM False(0)
VF_NODNIC_ENABLE False(0)
NUM_PF_MSIX_VALID True(1)
NUM_OF_VFS 0
NUM_OF_PF 0
PF_BAR2_ENABLE False(0)
HIDE_PORT2_PF True(1)
SRIOV_EN False(0)
PF_LOG_BAR_SIZE 5
VF_LOG_BAR_SIZE 1
NUM_PF_MSIX 127
NUM_VF_MSIX 11
PCIE_CREDIT_TOKEN_TIMEOUT 0
PCI_DOWNSTREAM_PORT_OWNER Array[0..15]
ROCE_CC_DCQCN_COMPATIBILITY_MODE DEVICE_DEFAULT(0)
ROCE_CC_LEGACY_DCQCN False(0)
HOST_CHAINING_MODE DISABLED(0)
HOST_CHAINING_CACHE_DISABLE False(0)
HOST_CHAINING_DESCRIPTORS Array[0..7]
HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7]
RDE_DISABLE False(0)
PLDM_FW_UPDATE_DISABLE False(0)
MCTP_I3C_DISABLE False(0)
RBT_DISABLE False(0)
PCIE_SMBUS_DISABLE False(0)
PCIE_IN_BAND_VDM_DISABLE False(0)
SM_DISABLE False(0)
LOG_DCR_HASH_TABLE_SIZE 11
MAX_PACKET_LIFETIME 0
DCR_LIFO_SIZE 16384
INTERNAL_CPU_MODEL EMBEDDED_CPU(1)
INTERNAL_CPU_PAGE_SUPPLIER ECPF(0)
INTERNAL_CPU_ESWITCH_MANAGER ECPF(0)
INTERNAL_CPU_IB_VPORT0 ECPF(0)
INTERNAL_CPU_OFFLOAD_ENGINE ENABLED(0)
ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7]
ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7]
MEMIC_BAR_SIZE 0
MEMIC_SIZE_LIMIT _256KB(1)
PCC_INT_NP_RTT_DSCP 26
PCC_INT_NP_RTT_DSCP_EN False(0)
PCC_INT_NP_RTT_DATA_MODE RTT_V0(64)
PCC_INT_EN False(0)
PCC_INT_SYSTEM_RTT 0
STEERING_CACHE_REFRESH 0
TX_SCHEDULER_LOCALITY_MODE DEVICE_DEFAULT(0)
MULTI_PCI_RESOURCE_SHARING DEVICE_DEFAULT(0)
ICM_CACHE_MODE DEVICE_DEFAULT(0)
TX_SCHEDULER_FWS_REACTIVITY DIRECT(1)
HAIRPIN_DATA_BUFFER_LOCK False(0)
TLS_OPTIMIZE False(0)
TX_SCHEDULER_BURST 0
ZERO_TOUCH_TUNING_ENABLE False(0)
TX_PCI_DATA_FETCH_LATENCY 40
RT_PPS_ENABLED_ON_POWERUP False(0)
LAG_RESOURCE_ALLOCATION PRE_ALLOCATION(1)
PHY_COUNT_LINK_UP_DELAY DELAY_NONE(0)
ACCURATE_TX_SCHEDULER False(0)
PARTIAL_RESET_EN False(0)
RESET_WITH_HOST_ON_ERRORS False(0)
USER_PROGRAMMABLE_CC False(0)
ROCE_NEXT_PROTOCOL 254
LOG_MAX_QUEUE 17
LOG_MAX_OUTSTANDING_READ_ATOMIC 0
MULTIPATH_DSCP DEVICE_DEFAULT(0)
CQE_COMPRESSION BALANCED(0)
IP_OVER_VXLAN_EN False(0)
MKEY_BY_NAME False(0)
PRIO_TAG_REQUIRED_EN False(0)
UCTX_EN True(1)
REAL_TIME_CLOCK_ENABLE False(0)
RDMA_SELECTIVE_REPEAT_EN False(0)
PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
TUNNEL_ECN_COPY_DISABLE False(0)
LRO_LOG_TIMEOUT0 6
LRO_LOG_TIMEOUT1 7
LRO_LOG_TIMEOUT2 8
LRO_LOG_TIMEOUT3 13
LOG_TX_PSN_WINDOW 7
VF_MIGRATION_MODE DEVICE_DEFAULT(0)
LOG_MAX_OUTSTANDING_WQE 7
ROCE_ADAPTIVE_ROUTING_EN False(0)
TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0)
INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0)
KEEP_ETH_LINK_UP_P1 True(1)
KEEP_IB_LINK_UP_P1 False(0)
KEEP_LINK_UP_ON_BOOT_P1 False(0)
KEEP_LINK_UP_ON_STANDBY_P1 False(0)
DO_NOT_CLEAR_PORT_STATS_P1 False(0)
AUTO_POWER_SAVE_LINK_DOWN_P1 False(0)
KEEP_ETH_LINK_UP_P2 True(1)
KEEP_IB_LINK_UP_P2 False(0)
KEEP_LINK_UP_ON_BOOT_P2 False(0)
KEEP_LINK_UP_ON_STANDBY_P2 False(0)
DO_NOT_CLEAR_PORT_STATS_P2 False(0)
AUTO_POWER_SAVE_LINK_DOWN_P2 False(0)
PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0)
PHY_RATE_MASK_OVERRIDE_P1 False(0)
PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0)
PHY_AUTO_NEG_P2 DEVICE_DEFAULT(0)
PHY_RATE_MASK_OVERRIDE_P2 False(0)
PHY_FEC_OVERRIDE_P2 DEVICE_DEFAULT(0)
LLDP_NB_DCBX_P1 True(1)
LLDP_NB_RX_MODE_P1 ALL(2)
LLDP_NB_TX_MODE_P1 ALL(2)
LLDP_NB_DCBX_P2 True(1)
LLDP_NB_RX_MODE_P2 ALL(2)
LLDP_NB_TX_MODE_P2 ALL(2)
DCBX_IEEE_P1 False(0)
DCBX_CEE_P1 False(0)
DCBX_WILLING_P1 False(0)
DCBX_IEEE_P2 False(0)
DCBX_CEE_P2 False(0)
DCBX_WILLING_P2 False(0)
DUP_MAC_ACTION_P1 LAST_CFG(0)
MPFS_MC_LOOPBACK_DISABLE_P1 False(0)
MPFS_UC_LOOPBACK_DISABLE_P1 False(0)
UNKNOWN_UPLINK_MAC_FLOOD_P1 False(0)
SRIOV_IB_ROUTING_MODE_P1 LID(1)
IB_ROUTING_MODE_P1 LID(1)
DUP_MAC_ACTION_P2 LAST_CFG(0)
MPFS_MC_LOOPBACK_DISABLE_P2 False(0)
MPFS_UC_LOOPBACK_DISABLE_P2 False(0)
UNKNOWN_UPLINK_MAC_FLOOD_P2 False(0)
SRIOV_IB_ROUTING_MODE_P2 LID(1)
IB_ROUTING_MODE_P2 LID(1)
NUM_OF_VL_P1 _4_VLs(3)
NUM_OF_TC_P1 _8_TCs(0)
NUM_OF_PFC_P1 8
VL15_BUFFER_SIZE_P1 0
QOS_TRUST_STATE_P1 TRUST_PCP(1)
ETS_SCHED_MODE_P1 device_default(0)
VL_BUFFER_ALLOCATION_P1 FULL_DYNAMIC(1)
NUM_OF_VL_P2 _4_VLs(3)
NUM_OF_TC_P2 _8_TCs(0)
NUM_OF_PFC_P2 8
VL15_BUFFER_SIZE_P2 0
QOS_TRUST_STATE_P2 TRUST_PCP(1)
ETS_SCHED_MODE_P2 device_default(0)
VL_BUFFER_ALLOCATION_P2 FULL_DYNAMIC(1)
ROCE_CC_PRIO_MASK_P1 255
ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0)
ROCE_CC_PRIO_MASK_P2 255
ROCE_CC_CNP_MODERATION_P2 DEVICE_DEFAULT(0)
CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1)
CLAMP_TGT_RATE_P1 False(0)
RPG_TIME_RESET_P1 300
RPG_BYTE_RESET_P1 32767
RPG_THRESHOLD_P1 1
RPG_MAX_RATE_P1 0
RPG_AI_RATE_P1 5
RPG_HAI_RATE_P1 50
RPG_GD_P1 11
RPG_MIN_DEC_FAC_P1 50
RPG_MIN_RATE_P1 1
RATE_TO_SET_ON_FIRST_CNP_P1 0
DCE_TCP_G_P1 1019
DCE_TCP_RTT_P1 1
RATE_REDUCE_MONITOR_PERIOD_P1 4
INITIAL_ALPHA_VALUE_P1 1023
MIN_TIME_BETWEEN_CNPS_P1 4
CNP_802P_PRIO_P1 6
CNP_DSCP_P1 48
CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1)
CLAMP_TGT_RATE_P2 False(0)
RPG_TIME_RESET_P2 300
RPG_BYTE_RESET_P2 32767
RPG_THRESHOLD_P2 1
RPG_MAX_RATE_P2 0
RPG_AI_RATE_P2 5
RPG_HAI_RATE_P2 50
RPG_GD_P2 11
RPG_MIN_DEC_FAC_P2 50
RPG_MIN_RATE_P2 1
RATE_TO_SET_ON_FIRST_CNP_P2 0
DCE_TCP_G_P2 1019
DCE_TCP_RTT_P2 1
RATE_REDUCE_MONITOR_PERIOD_P2 4
INITIAL_ALPHA_VALUE_P2 1023
MIN_TIME_BETWEEN_CNPS_P2 4
CNP_802P_PRIO_P2 6
CNP_DSCP_P2 48
ROCE_RTT_RESP_DSCP_P1 0
ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0)
ROCE_RTT_RESP_DSCP_P2 0
ROCE_RTT_RESP_DSCP_MODE_P2 DEVICE_DEFAULT(0)
ECPF_ESWITCH_MANAGER ECPF(1)
ECPF_PAGE_SUPPLIER ECPF(1)
PORT_OWNER True(1)
ALLOW_RD_COUNTERS True(1)
RENEG_ON_CHANGE True(1)
TRACER_ENABLE True(1)
SILENT_MODE False(0)
MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0)
ROCE_CONTROL ROCE_ENABLE(2)
PCI_WR_ORDERING per_mkey(0)
MULTI_PORT_VHCA_EN False(0)
PF_TOTAL_SF 8
PF_SD_GROUP 0
PF_DEVICE_ID_ENABLE False(0)
PF_SF_BAR_SIZE 10
PF_NUM_PF_MSIX 63
PF_NUM_OF_VF 0
PF_DEVICE_ID 41686
BOOT_VLAN 1
LEGACY_BOOT_PROTOCOL PXE(1)
BOOT_INTERRUPT_DIS False(0)
BOOT_LACP_DIS True(1)
BOOT_VLAN_EN False(0)
IP_VER IPv4(0)
BOOT_UNDI_NETWORK_WAIT 0
BOOT_DBG_LOG False(0)
BOOT_PKEY 0
UEFI_HII_EN True(1)
UEFI_LOGS DISABLED(0)
MANAGEMENT_PF_MODE MNG_PF_ENABLE(1)
P2P_ORDERING_MODE DEVICE_DEFAULT(0)
EXP_ROM_VIRTIO_NET_PXE_ENABLE False(0)
EXP_ROM_VIRTIO_NET_UEFI_ARM_ENABLE False(0)
EXP_ROM_VIRTIO_NET_UEFI_x86_ENABLE False(0)
EXP_ROM_VIRTIO_BLK_UEFI_ARM_ENABLE False(0)
EXP_ROM_VIRTIO_BLK_UEFI_x86_ENABLE False(0)
EXP_ROM_NVME_UEFI_x86_ENABLE True(1)
ATS_ENABLED False(0)
DYNAMIC_VF_MSIX_TABLE False(0)
EXP_ROM_UEFI_ARM_ENABLE True(1)
EXP_ROM_UEFI_x86_ENABLE True(1)
EXP_ROM_PXE_ENABLE True(1)
ADVANCED_PCI_SETTINGS False(0)
SAFE_MODE_THRESHOLD 10
SAFE_MODE_ENABLE True(1)
root@bf2:/home/ubuntu#
opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 0
/opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 1
/opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 2
/opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum /opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 4
/opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 5
/opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 6
/opt/mellanox/iproute2/sbin/mlxdevm port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 7
344 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98304 hw_addr 02:11:22:33:44:01
345 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98305 hw_addr 02:11:22:33:44:02
346 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98306 hw_addr 02:11:22:33:44:03
347 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98307 hw_addr 02:11:22:33:44:04
348 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98308 hw_addr 02:11:22:33:44:05
349 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98309 hw_addr 02:11:22:33:44:06
350 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98310 hw_addr 02:11:22:33:44:07
351 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98311 hw_addr 02:11:22:33:44:08
352 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98304 trust on
353 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98305 trust on
354 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98306 trust on
355 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98307 trust on
356 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98308 trust on
357 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98309 trust on
358 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/983010 trust on
359 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98310 trust on
360 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98311 trust on
361 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98304 state active
362 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98305 state active
363 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98306 state active
364 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98307 state active
365 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98308 state active
366 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98309 state active
367 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98310 state active
368 /opt/mellanox/iproute2/sbin/mlxdevm port function set pci/0000:03:00.0/98311 state active
370 cat /sys/bus/auxiliary/devices/mlx5_core.sf.2/sfnum
371 cat /sys/bus/auxiliary/devices/mlx5_core.sf.3/sfnum
372 cat /sys/bus/auxiliary/devices/mlx5_core.sf.4/sfnum
373 cat /sys/bus/auxiliary/devices/mlx5_core.sf.5/sfnum
374 cat /sys/bus/auxiliary/devices/mlx5_core.sf.6/sfnum
375 cat /sys/bus/auxiliary/devices/mlx5_core.sf.7/sfnum
376 cat /sys/bus/auxiliary/devices/mlx5_core.sf.8/sfnum
377 cat /sys/bus/auxiliary/devices/mlx5_core.sf.9/sfnum
378 echo mlx5_core.sf.2 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
380 echo mlx5_core.sf.3 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
382 echo mlx5_core.sf.4 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
383 echo mlx5_core.sf.5 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
384 echo mlx5_core.sf.6 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
387 echo mlx5_core.sf.7 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
388 echo mlx5_core.sf.8 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
389 echo mlx5_core.sf.9 > /sys/bus/auxiliary/drivers/mlx5_core.sf_cfg/unbind
390 echo mlx5_core.sf.2 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
391 echo mlx5_core.sf.3 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
392 echo mlx5_core.sf.4 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
echo mlx5_core.sf.5 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
echo mlx5_core.sf.6 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
echo mlx5_core.sf.7 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
echo mlx5_core.sf.8 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
echo mlx5_core.sf.9 > /sys/bus/auxiliary/drivers/mlx5_core.sf/bind
sudo ovs-vsctl add-br sf_switch
sudo ovs-vsctl add-port sf_switch p0
sudo ovs-vsctl add-port sf_switch en3f0pf0sf0
sudo ovs-vsctl add-port sf_switch en3f0pf0sf1
sudo ovs-vsctl add-port sf_switch en3f0pf0sf2
sudo ovs-vsctl add-port sf_switch en3f0pf0sf3
sudo ovs-vsctl add-port sf_switch en3f0pf0sf4
sudo ovs-vsctl add-port sf_switch en3f0pf0sf5
sudo ovs-vsctl add-port sf_switch en3f0pf0sf6
sudo ovs-vsctl add-port sf_switch en3f0pf0sf7
sudo ip link set en3f0pf0sf0 up
sudo ip link set en3f0pf0sf1 up
sudo ip link set en3f0pf0sf2 up
sudo ip link set en3f0pf0sf3 up
sudo ip link set en3f0pf0sf4 up
sudo ip link set en3f0pf0sf5 up
sudo ip link set en3f0pf0sf6 up
sudo ip link set en3f0pf0sf7 up
sudo ip link set p0 up
before reboot:
root@bf2:/home/ubuntu# sudo ovs-vsctl show
2106a279-27bf-44d0-aba5-c4b9d6e6ce44
Bridge sf_switch
Port en3f0pf0sf2
Interface en3f0pf0sf2
Port sf_switch
Interface sf_switch
type: internal
Port en3f0pf0sf3
Interface en3f0pf0sf3
Port en3f0pf0sf6
Interface en3f0pf0sf6
Port en3f0pf0sf5
Interface en3f0pf0sf5
Port en3f0pf0sf0
Interface en3f0pf0sf0
Port en3f0pf0sf7
Interface en3f0pf0sf7
Port en3f0pf0sf1
Interface en3f0pf0sf1
Port en3f0pf0sf4
Interface en3f0pf0sf4
Port p0
Interface p0
ovs_version: "3.0.0-0056-25
oot@bf2:/home/ubuntu# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: oob_net0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 08:c0:eb:b2:43:96 brd ff:ff:ff:ff:ff:ff
altname enamlnxbf17i0
inet 172.28.20.117/24 brd 172.28.20.255 scope global dynamic noprefixroute oob_net0
valid_lft 3030sec preferred_lft 3030sec
inet6 fe80::ac0:ebff:feb2:4396/64 scope link
valid_lft forever preferred_lft forever
3: tmfifo_net0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:1a:ca:ff:ff:01 brd ff:ff:ff:ff:ff:ff
inet 192.168.100.2/30 brd 192.168.100.3 scope global noprefixroute tmfifo_net0
valid_lft forever preferred_lft forever
inet6 fe80::21a:caff:feff:ff01/64 scope link
valid_lft forever preferred_lft forever
4: p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 08:c0:eb:b2:43:8e brd ff:ff:ff:ff:ff:ff
altname enp3s0f0np0
inet6 fe80::ac0:ebff:feb2:438e/64 scope link
valid_lft forever preferred_lft forever
5: p1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 08:c0:eb:b2:43:8f brd ff:ff:ff:ff:ff:ff
altname enp3s0f1np1
inet6 fe80::ac0:ebff:feb2:438f/64 scope link
valid_lft forever preferred_lft forever
8: en3f0pf0sf0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether a6:cd:09:fe:76:d1 brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf0
inet 192.168.50.1/24 scope global en3f0pf0sf0
valid_lft forever preferred_lft forever
9: en3f0pf0sf1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether a2:cb:16:27:68:65 brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf1
inet 192.168.50.2/24 scope global en3f0pf0sf1
valid_lft forever preferred_lft forever
inet6 fe80::a0cb:16ff:fe27:6865/64 scope link
valid_lft forever preferred_lft forever
10: en3f0pf0sf2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 02:c8:a1:25:36:3e brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf2
inet6 fe80::c8:a1ff:fe25:363e/64 scope link
valid_lft forever preferred_lft forever
11: en3f0pf0sf3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 1a:ee:11:99:df:2c brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf3
inet6 fe80::18ee:11ff:fe99:df2c/64 scope link
valid_lft forever preferred_lft forever
12: en3f0pf0sf4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether c2:26:9d:bd:33:0c brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf4
inet6 fe80::c026:9dff:febd:330c/64 scope link
valid_lft forever preferred_lft forever
13: en3f0pf0sf5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 36:58:c5:eb:eb:27 brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf5
inet6 fe80::3458:c5ff:feeb:eb27/64 scope link
valid_lft forever preferred_lft forever
14: en3f0pf0sf6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 8a:ac:87:d7:88:87 brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf6
inet6 fe80::88ac:87ff:fed7:8887/64 scope link
valid_lft forever preferred_lft forever
15: en3f0pf0sf7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 06:27:f0:ef:a1:97 brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf7
inet6 fe80::427:f0ff:feef:a197/64 scope link
valid_lft forever preferred_lft forever
16: enp3s0f0s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:01 brd ff:ff:ff:ff:ff:ff
17: enp3s0f0s1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:02 brd ff:ff:ff:ff:ff:ff
18: enp3s0f0s2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:03 brd ff:ff:ff:ff:ff:ff
19: enp3s0f0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:04 brd ff:ff:ff:ff:ff:ff
20: enp3s0f0s4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:05 brd ff:ff:ff:ff:ff:ff
21: enp3s0f0s5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:06 brd ff:ff:ff:ff:ff:ff
22: enp3s0f0s6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:07 brd ff:ff:ff:ff:ff:ff
23: enp3s0f0s7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:11:22:33:44:08 brd ff:ff:ff:ff:ff:ff
24: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 7a:21:81:d9:77:6f brd ff:ff:ff:ff:ff:ff
25: sf_switch: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 08:c0:eb:b2:43:8e brd ff:ff:ff:ff:ff:ff
root@bf2:/home/ubuntu# devlink dev show
pci/0000:03:00.0
pci/0000:03:00.1
auxiliary/mlx5_core.sf.2
auxiliary/mlx5_core.sf.3
auxiliary/mlx5_core.sf.4
auxiliary/mlx5_core.sf.5
auxiliary/mlx5_core.sf.6
auxiliary/mlx5_core.sf.7
auxiliary/mlx5_core.sf.8
auxiliary/mlx5_core.sf.9
after reboot
oot@bf2:/home/ubuntu# sudo ovs-vsctl show
2106a279-27bf-44d0-aba5-c4b9d6e6ce44
Bridge sf_switch
Port en3f0pf0sf2
Interface en3f0pf0sf2
error: “could not open network device en3f0pf0sf2 (No such device)”
Port sf_switch
Interface sf_switch
type: internal
Port en3f0pf0sf3
Interface en3f0pf0sf3
error: “could not open network device en3f0pf0sf3 (No such device)”
Port en3f0pf0sf6
Interface en3f0pf0sf6
error: “could not open network device en3f0pf0sf6 (No such device)”
Port en3f0pf0sf5
Interface en3f0pf0sf5
error: “could not open network device en3f0pf0sf5 (No such device)”
Port en3f0pf0sf0
Interface en3f0pf0sf0
Port en3f0pf0sf7
Interface en3f0pf0sf7
error: “could not open network device en3f0pf0sf7 (No such device)”
Port en3f0pf0sf1
Interface en3f0pf0sf1
error: “could not open network device en3f0pf0sf1 (No such device)”
Port en3f0pf0sf4
Interface en3f0pf0sf4
error: “could not open network device en3f0pf0sf4 (No such device)”
Port p0
Interface p0
ovs_version: “3.0.0-0056-25.04-based-3.3.5”
root@bf2:/home/ubuntu# devlink dev show
pci/0000:03:00.0
pci/0000:03:00.1
auxiliary/mlx5_core.sf.2
root@bf2:/home/ubuntu# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: oob_net0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 08:c0:eb:b2:43:96 brd ff:ff:ff:ff:ff:ff
altname enamlnxbf17i0
inet 172.28.20.117/24 brd 172.28.20.255 scope global dynamic noprefixroute oob_net0
valid_lft 3586sec preferred_lft 3586sec
inet6 fdb5:768b:3f44:7146:ac0:ebff:feb2:4396/64 scope global dynamic mngtmpaddr
valid_lft 299sec preferred_lft 299sec
inet6 fe80::ac0:ebff:feb2:4396/64 scope link
valid_lft forever preferred_lft forever
3: tmfifo_net0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:1a:ca:ff:ff:01 brd ff:ff:ff:ff:ff:ff
inet 192.168.100.2/30 brd 192.168.100.3 scope global noprefixroute tmfifo_net0
valid_lft forever preferred_lft forever
inet6 fe80::21a:caff:feff:ff01/64 scope link
valid_lft forever preferred_lft forever
4: p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 08:c0:eb:b2:43:8e brd ff:ff:ff:ff:ff:ff
altname enp3s0f0np0
inet6 fe80::ac0:ebff:feb2:438e/64 scope link
valid_lft forever preferred_lft forever
5: p1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 08:c0:eb:b2:43:8f brd ff:ff:ff:ff:ff:ff
altname enp3s0f1np1
inet6 fe80::ac0:ebff:feb2:438f/64 scope link
valid_lft forever preferred_lft forever
6: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 2a:9f:d7:b2:ef:6d brd ff:ff:ff:ff:ff:ff
7: pf0hpf: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 56:b5:1f:31:e1:9d brd ff:ff:ff:ff:ff:ff
altname enp3s0f0nc1pf0
inet6 fe80::54b5:1fff:fe31:e19d/64 scope link
valid_lft forever preferred_lft forever
8: sf_switch: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 08:c0:eb:b2:43:8e brd ff:ff:ff:ff:ff:ff
9: en3f0pf0sf0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ovs-system state UP group default qlen 1000
link/ether 16:81:97:e5:97:d4 brd ff:ff:ff:ff:ff:ff
altname enp3s0f0npf0sf0
10: enp3s0f0s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 02:72:5f:6e:3a:9a brd ff:ff:ff:ff:ff:ff
inet6 fe80::72:5fff:fe6e:3a9a/64 scope link
valid_lft forever preferred_lft forever
root@bf2:/home/ubuntu#