I’ve been testing Bluefield2 in the 2x25G (from Ebay) and 2x100G config (MBF2M516A-CEEO_Ax_Bx) and performance is horrible. With iperf3 I’m getting 25Mbit/s and when I stop stop nl2doca then I get 500-1000 Mbit/s. I’m on a Proxmox host with doca-all, BFB and HBN 3.1.0 installed. I’ve followed basically the HBN Service Configuration and doing EVPN with two SN2010 switches. The host also has a ConnectX 4 Lx with SRIOV and I’m able to fill up the 25G link with that. But I really like how I’m able to assign the different VFs to a port on the eswitch and then still have the redundant uplink of the server without having LACP. I’ve configured the port to the server as an access port in a specific VLAN but I also have trunk ports. What I noticed is that OVS is still very active as well, but I haven’t seen anywhere if that is expected or not.I do believe all the CPU engine etc… settings are correct.
Device type: BlueField2Name: MBF2M516A-CEEO_Ax_BxDescription: BlueField-2 E-Series DPU 100GbE Dual-Port QSFP56; PCIe Gen4 x16; Crypto Enabled; 16GB on-board DDR; 1GbE OOB management; FHHLDevice: /dev/mst/mt41686_pciconf0
Configurations: Next BootACCURATE_TX_SCHEDULER False(0)ADVANCED_PCI_SETTINGS False(0)AES_XTS_TWEAK_INC_64 False(0)ALLOW_RD_COUNTERS True(1)ATS_ENABLED False(0)AUTO_POWER_SAVE_LINK_DOWN_P1 False(0)AUTO_POWER_SAVE_LINK_DOWN_P2 False(0)BOOT_DBG_LOG False(0)BOOT_INTERRUPT_DIS False(0)BOOT_LACP_DIS True(1)BOOT_PKEY 0BOOT_UNDI_NETWORK_WAIT 0BOOT_VLAN_EN False(0)BOOT_VLAN 1CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1)CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1)CLAMP_TGT_RATE_P1 False(0)CLAMP_TGT_RATE_P2 False(0)CNP_802P_PRIO_P1 6CNP_802P_PRIO_P2 6CNP_DSCP_P1 48CNP_DSCP_P2 48CQE_COMPRESSION BALANCED(0)CRYPTO_POLICY UNRESTRICTED(1)DCBX_CEE_P1 True(1)DCBX_CEE_P2 True(1)DCBX_IEEE_P1 True(1)DCBX_IEEE_P2 True(1)DCBX_WILLING_P1 True(1)DCBX_WILLING_P2 True(1)DCE_TCP_G_P1 1019DCE_TCP_G_P2 1019DCE_TCP_RTT_P1 1DCE_TCP_RTT_P2 1DCR_LIFO_SIZE 16384DO_NOT_CLEAR_PORT_STATS_P1 False(0)DO_NOT_CLEAR_PORT_STATS_P2 False(0)DPU_RESET_NOTIFICATION_ENABLED DISABLED(0)DUP_MAC_ACTION_P1 LAST_CFG(0)DUP_MAC_ACTION_P2 LAST_CFG(0)DYNAMIC_VF_MSIX_TABLE False(0)ECPF_ESWITCH_MANAGER ECPF(1)ECPF_PAGE_SUPPLIER ECPF(1)ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7]ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7]ETS_SCHED_MODE_P1 device_default(0)ETS_SCHED_MODE_P2 device_default(0)EXP_ROM_NVME_UEFI_x86_ENABLE True(1)EXP_ROM_PXE_ENABLE True(1)EXP_ROM_UEFI_ARM_ENABLE True(1)EXP_ROM_UEFI_x86_ENABLE True(1)EXP_ROM_VIRTIO_BLK_UEFI_ARM_ENABLE False(0)EXP_ROM_VIRTIO_BLK_UEFI_x86_ENABLE False(0)EXP_ROM_VIRTIO_NET_PXE_ENABLE False(0)EXP_ROM_VIRTIO_NET_UEFI_ARM_ENABLE False(0)EXP_ROM_VIRTIO_NET_UEFI_x86_ENABLE False(0)FLEX_IPV4_OVER_VXLAN_PORT 0FLEX_PARSER_PROFILE_ENABLE 0HAIRPIN_DATA_BUFFER_LOCK False(0)HIDE_PORT2_PF False(0)HOST_CHAINING_CACHE_DISABLE False(0)HOST_CHAINING_DESCRIPTORS Array[0..7]HOST_CHAINING_MODE DISABLED(0)HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7]HOST_PRIV_FLASH_ACCESS DEVICE_DEFAULT(0)HOST_PRIV_FW_UPDATE DEVICE_DEFAULT(0)HOST_PRIV_NIC_RESET DEVICE_DEFAULT(0)HOST_PRIV_NV_GLOBAL DEVICE_DEFAULT(0)HOST_PRIV_NV_HOST DEVICE_DEFAULT(0)HOST_PRIV_NV_INTERNAL_CPU DEVICE_DEFAULT(0)HOST_PRIV_NV_PORT DEVICE_DEFAULT(0)HOST_PRIV_PCC_UPDATE DEVICE_DEFAULT(0)IB_ROUTING_MODE_P1 LID(1)IB_ROUTING_MODE_P2 LID(1)ICM_CACHE_MODE DEVICE_DEFAULT(0)INITIAL_ALPHA_VALUE_P1 1023INITIAL_ALPHA_VALUE_P2 1023INTERNAL_CPU_ESWITCH_MANAGER ECPF(0)INTERNAL_CPU_IB_VPORT0 ECPF(0)INTERNAL_CPU_MODEL EMBEDDED_CPU(1)INTERNAL_CPU_OFFLOAD_ENGINE ENABLED(0)INTERNAL_CPU_PAGE_SUPPLIER ECPF(0)INTERNAL_CPU_RSHIM ENABLED(0)INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0)IP_OVER_VXLAN_EN False(0)IP_VER IPv4(0)KEEP_ETH_LINK_UP_P1 True(1)KEEP_ETH_LINK_UP_P2 True(1)KEEP_IB_LINK_UP_P1 False(0)KEEP_IB_LINK_UP_P2 False(0)KEEP_LINK_UP_ON_BOOT_P1 False(0)KEEP_LINK_UP_ON_BOOT_P2 False(0)KEEP_LINK_UP_ON_STANDBY_P1 False(0)KEEP_LINK_UP_ON_STANDBY_P2 False(0)LAG_RESOURCE_ALLOCATION PRE_ALLOCATION(1)LARGE_MTU_TWEAK_64 False(0)LEGACY_BOOT_PROTOCOL PXE(1)LLDP_NB_DCBX_P1 False(0)LLDP_NB_DCBX_P2 False(0)LLDP_NB_RX_MODE_P1 OFF(0)LLDP_NB_RX_MODE_P2 OFF(0)LLDP_NB_TX_MODE_P1 OFF(0)LLDP_NB_TX_MODE_P2 OFF(0)LOG_DCR_HASH_TABLE_SIZE 11LOG_MAX_OUTSTANDING_READ_ATOMIC 0LOG_MAX_OUTSTANDING_WQE 7LOG_MAX_QUEUE 17LOG_TX_PSN_WINDOW 7LRO_LOG_TIMEOUT0 6LRO_LOG_TIMEOUT1 7LRO_LOG_TIMEOUT2 8LRO_LOG_TIMEOUT3 13MANAGEMENT_PF_MODE DEVICE_DEFAULT(0)MAX_PACKET_LIFETIME 0MCTP_I3C_DISABLE False(0)MEMIC_BAR_SIZE 0MEMIC_SIZE_LIMIT _256KB(1)MIN_TIME_BETWEEN_CNPS_P1 4MIN_TIME_BETWEEN_CNPS_P2 4MKEY_BY_NAME False(0)MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0)MPFS_MC_LOOPBACK_DISABLE_P1 False(0)MPFS_MC_LOOPBACK_DISABLE_P2 False(0)MPFS_UC_LOOPBACK_DISABLE_P1 False(0)MPFS_UC_LOOPBACK_DISABLE_P2 False(0)MULTIPATH_DSCP DEVICE_DEFAULT(0)MULTI_PCI_RESOURCE_SHARING DEVICE_DEFAULT(0)MULTI_PORT_VHCA_EN False(0)NON_PREFETCHABLE_PF_BAR False(0)NUM_OF_PFC_P1 8NUM_OF_PFC_P2 8NUM_OF_PF 2NUM_OF_TC_P1 _8_TCs(0)NUM_OF_TC_P2 _8_TCs(0)NUM_OF_VFS 16NUM_OF_VL_P1 _4_VLs(3)NUM_OF_VL_P2 _4_VLs(3)NUM_PF_MSIX 63NUM_PF_MSIX_VALID False(0)NUM_VF_MSIX 11NVME_EMULATION_CLASS_CODE 67586NVME_EMULATION_DEVICE_ID 24577NVME_EMULATION_ENABLE False(0)NVME_EMULATION_MAX_QUEUE_DEPTH 0NVME_EMULATION_NUM_MSIX 0NVME_EMULATION_NUM_PF 1NVME_EMULATION_NUM_VF 0NVME_EMULATION_REVISION_ID 0NVME_EMULATION_SUBSYSTEM_ID 0NVME_EMULATION_SUBSYSTEM_VENDOR_ID 0NVME_EMULATION_VENDOR_ID 5555P2P_ORDERING_MODE DEVICE_DEFAULT(0)PARTIAL_RESET_EN False(0)PCC_INT_EN False(0)PCC_INT_NP_RTT_DATA_MODE RTT_V0(64)PCC_INT_NP_RTT_DSCP_EN False(0)PCC_INT_NP_RTT_DSCP 26PCC_INT_SYSTEM_RTT 0PCIE_CREDIT_TOKEN_TIMEOUT 0PCIE_IN_BAND_VDM_DISABLE False(0)PCIE_SMBUS_DISABLE False(0)PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)PCI_DOWNSTREAM_PORT_OWNER Array[0..15]PCI_SWITCH_EMULATION_ENABLE False(0)PCI_SWITCH_EMULATION_NUM_PORT 0PCI_WR_ORDERING per_mkey(0)PER_PF_NUM_SF True(1)PF_BAR2_ENABLE False(0)PF_BAR2_SIZE 3PF_DEVICE_ID_ENABLE False(0)PF_DEVICE_ID 41686PF_LOG_BAR_SIZE 5PF_NUM_OF_VF 32PF_NUM_OF_VF_VALID True(1)PF_NUM_PF_MSIX 228PF_NUM_PF_MSIX_VALID True(1)PF_SD_GROUP 0PF_SF_BAR_SIZE 10PF_TOTAL_SF 128PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0)PHY_AUTO_NEG_P2 DEVICE_DEFAULT(0)PHY_COUNT_LINK_UP_DELAY DELAY_NONE(0)PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0)PHY_FEC_OVERRIDE_P2 DEVICE_DEFAULT(0)PHY_RATE_MASK_OVERRIDE_P1 False(0)PHY_RATE_MASK_OVERRIDE_P2 False(0)PLDM_FW_UPDATE_DISABLE False(0)PORT_OWNER True(1)PRIO_TAG_REQUIRED_EN False(0)PROG_PARSE_GRAPH False(0)QOS_TRUST_STATE_P1 TRUST_PCP(1)QOS_TRUST_STATE_P2 TRUST_PCP(1)RATE_REDUCE_MONITOR_PERIOD_P1 4RATE_REDUCE_MONITOR_PERIOD_P2 4RATE_TO_SET_ON_FIRST_CNP_P1 0RATE_TO_SET_ON_FIRST_CNP_P2 0RBT_DISABLE False(0)RDE_DISABLE False(0)RDMA_SELECTIVE_REPEAT_EN False(0)REAL_TIME_CLOCK_ENABLE False(0)RENEG_ON_CHANGE True(1)RESET_WITH_HOST_ON_ERRORS False(0)ROCE_ADAPTIVE_ROUTING_EN False(0)ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0)ROCE_CC_CNP_MODERATION_P2 DEVICE_DEFAULT(0)ROCE_CC_DCQCN_COMPATIBILITY_MODE DEVICE_DEFAULT(0)ROCE_CC_LEGACY_DCQCN False(0)ROCE_CC_PRIO_MASK_P1 255ROCE_CC_PRIO_MASK_P2 255ROCE_CONTROL ROCE_ENABLE(2)ROCE_NEXT_PROTOCOL 254ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0)ROCE_RTT_RESP_DSCP_MODE_P2 DEVICE_DEFAULT(0)ROCE_RTT_RESP_DSCP_P1 0ROCE_RTT_RESP_DSCP_P2 0RPG_AI_RATE_P1 5RPG_AI_RATE_P2 5RPG_BYTE_RESET_P1 32767RPG_BYTE_RESET_P2 32767RPG_GD_P1 11RPG_GD_P2 11RPG_HAI_RATE_P1 50RPG_HAI_RATE_P2 50RPG_MAX_RATE_P1 0RPG_MAX_RATE_P2 0RPG_MIN_DEC_FAC_P1 50RPG_MIN_DEC_FAC_P2 50RPG_MIN_RATE_P1 1RPG_MIN_RATE_P2 1RPG_THRESHOLD_P1 1RPG_THRESHOLD_P2 1RPG_TIME_RESET_P1 300RPG_TIME_RESET_P2 300RT_PPS_ENABLED_ON_POWERUP False(0)SAFE_MODE_ENABLE True(1)SAFE_MODE_THRESHOLD 10SILENT_MODE False(0)SM_DISABLE False(0)SRIOV_EN True(1)SRIOV_IB_ROUTING_MODE_P1 LID(1)SRIOV_IB_ROUTING_MODE_P2 LID(1)STEERING_CACHE_REFRESH 0STRICT_VF_MSIX_NUM False(0)SWITCH_COMPT_FEATURE_MASK 0x0(0)TLS_OPTIMIZE False(0)TRACER_ENABLE True(1)TUNNEL_ECN_COPY_DISABLE False(0)TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0)TX_PCI_DATA_FETCH_LATENCY 40TX_SCHEDULER_BURST 0TX_SCHEDULER_FWS_REACTIVITY DIRECT(1)TX_SCHEDULER_LOCALITY_MODE DEVICE_DEFAULT(0)UCTX_EN True(1)UEFI_HII_EN True(1)UEFI_LOGS DISABLED(0)UNKNOWN_UPLINK_MAC_FLOOD_P1 False(0)UNKNOWN_UPLINK_MAC_FLOOD_P2 False(0)UPT_EMULATION_ENABLE False(0)USER_PROGRAMMABLE_CC False(0)VF_LOG_BAR_SIZE 1VF_MIGRATION_MODE DEVICE_DEFAULT(0)VF_NODNIC_ENABLE False(0)VF_VPD_ENABLE False(0)VIRTIO_BLK_EMULATION_ENABLE False(0)VIRTIO_BLK_EMULATION_NUM_MSIX 2VIRTIO_BLK_EMULATION_NUM_PF 0VIRTIO_BLK_EMULATION_NUM_VF 0VIRTIO_BLK_EMULATION_PF_PCI_LAYOUT VIRTIO_1_X(0)VIRTIO_BLK_EMULATION_SUBSYSTEM_ID 4162VIRTIO_BLK_EMULATION_VF_PCI_LAYOUT VIRTIO_1_X(0)VIRTIO_BLK_EMU_SUBS_VENDOR_ID 6900VIRTIO_EMULATION_HOTPLUG_TRANS False(0)VIRTIO_NET_EMULATION_ENABLE False(0)VIRTIO_NET_EMULATION_NUM_MSIX 2VIRTIO_NET_EMULATION_NUM_PF 0VIRTIO_NET_EMULATION_NUM_VF 0VIRTIO_NET_EMULATION_PF_PCI_LAYOUT VIRTIO_1_X(0)VIRTIO_NET_EMULATION_SUBSYSTEM_ID 4161VIRTIO_NET_EMULATION_VF_PCI_LAYOUT VIRTIO_1_X(0)VIRTIO_NET_EMU_SUBSYSTEM_VENDOR_ID 6900VL15_BUFFER_SIZE_P1 0VL15_BUFFER_SIZE_P2 0VL_BUFFER_ALLOCATION_P1 FULL_DYNAMIC(1)VL_BUFFER_ALLOCATION_P2 FULL_DYNAMIC(1)ZERO_TOUCH_TUNING_ENABLE False(0)