Hi, when looking at doca’s example application pcc, I see the following:
#ifdef TIME_SYNC
/* if there is a time sync between the nodes we can get one way delay */
uint32_t end_rtt = doca_pcc_dev_get_rtt_req_recv_timestamp(event);
#else
We setup a clock synchronization such that the nic hardware clocks and clock_realtime are synced between two nodes with the smart nic, synced to within hundreds of nanoseconds. When we send probes using ibverbs, we are getting about 900 ns for one way delay.
When running the pcc application, we are getting a reasonable rtt of 2 microseconds.
However, we are getting roughly -179ms one way delays, when we use the result of doca_pcc_dev_get_rtt_req_recv_timestamp(event).
What are we missing? Why are we getting such desynced clocks on the dpu?
Thanks a lot!
Here is the full mlxconfig output:
ubuntu@oob4:~$ sudo mlxconfig -y -d mlx5_1 query
Device #1:
----------
Device type: BlueField3
Name: 900-9D3B6-00CV-A_Ax
Description: NVIDIA BlueField-3 B3220 P-Series FHHL DPU; 200GbE (default mode) / NDR200 IB; Dual-port QSFP112; PCIe Gen5.0 x16 with x16 PCIe extension option; 16 Arm cores; 32GB on-board DDR; integrated BMC; Crypto Enabled
Device: mlx5_1
Configurations: Next Boot
MODULE_SPLIT_M0 Array[0..15]
MODULE_SPLIT_M1 Array[0..15]
MEMIC_BAR_SIZE 0
MEMIC_SIZE_LIMIT _256KB(1)
MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2)
HOST_CHAINING_MODE DISABLED(0)
HOST_CHAINING_CACHE_DISABLE False(0)
HOST_CHAINING_DESCRIPTORS Array[0..7]
HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7]
INTERNAL_CPU_MODEL EMBEDDED_CPU(1)
INTERNAL_CPU_PAGE_SUPPLIER ECPF(0)
INTERNAL_CPU_ESWITCH_MANAGER ECPF(0)
INTERNAL_CPU_IB_VPORT0 ECPF(0)
INTERNAL_CPU_OFFLOAD_ENGINE ENABLED(0)
FLEX_PARSER_PROFILE_ENABLE 0
PROG_PARSE_GRAPH False(0)
FLEX_IPV4_OVER_VXLAN_PORT 0
ROCE_NEXT_PROTOCOL 254
ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7]
ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7]
DPA_AUTHENTICATION False(0)
PF_BAR2_SIZE 3
INTERNAL_CPU_RSHIM ENABLED(0)
PF_NUM_OF_VF_VALID False(0)
NON_PREFETCHABLE_PF_BAR False(0)
VF_VPD_ENABLE False(0)
PF_NUM_PF_MSIX_VALID False(0)
PER_PF_NUM_SF False(0)
STRICT_VF_MSIX_NUM False(0)
VF_NODNIC_ENABLE False(0)
NUM_PF_MSIX_VALID True(1)
NUM_OF_VFS 16
NUM_OF_PF 2
PF_BAR2_ENABLE True(1)
HIDE_PORT2_PF False(0)
SRIOV_EN True(1)
PF_LOG_BAR_SIZE 5
VF_LOG_BAR_SIZE 1
NUM_PF_MSIX 63
NUM_VF_MSIX 11
INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0)
PCIE_CREDIT_TOKEN_TIMEOUT 0
RT_PPS_ENABLED_ON_POWERUP False(0)
LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0)
ACCURATE_TX_SCHEDULER False(0)
PARTIAL_RESET_EN False(0)
RESET_WITH_HOST_ON_ERRORS False(0)
NVME_EMULATION_ENABLE False(0)
NVME_EMULATION_NUM_VF 0
NVME_EMULATION_NUM_PF 1
NVME_EMULATION_VENDOR_ID 5555
NVME_EMULATION_DEVICE_ID 24577
NVME_EMULATION_CLASS_CODE 67586
NVME_EMULATION_REVISION_ID 0
NVME_EMULATION_SUBSYSTEM_VENDOR_ID 0
NVME_EMULATION_SUBSYSTEM_ID 0
NVME_EMULATION_NUM_MSIX 0
NVME_EMULATION_NUM_VF_MSIX 0
NVME_EMULATION_MAX_QUEUE_DEPTH 0
PCI_SWITCH_EMULATION_NUM_PORT 16
PCI_SWITCH_EMULATION_ENABLE False(0)
VIRTIO_NET_EMULATION_ENABLE False(0)
VIRTIO_NET_EMULATION_NUM_VF 0
VIRTIO_NET_EMULATION_NUM_PF 0
VIRTIO_NET_EMU_SUBSYSTEM_VENDOR_ID 6900
VIRTIO_NET_EMULATION_SUBSYSTEM_ID 4161
VIRTIO_NET_EMULATION_NUM_MSIX 2
VIRTIO_NET_EMULATION_NUM_VF_MSIX 0
VIRTIO_BLK_EMULATION_ENABLE False(0)
VIRTIO_BLK_EMULATION_NUM_VF 0
VIRTIO_BLK_EMULATION_NUM_PF 0
VIRTIO_BLK_EMU_SUBSYSTEM_VENDOR_ID 6900
VIRTIO_BLK_EMULATION_SUBSYSTEM_ID 4162
VIRTIO_BLK_EMULATION_NUM_MSIX 2
VIRTIO_BLK_EMULATION_NUM_VF_MSIX 0
PCI_DOWNSTREAM_PORT_OWNER Array[0..15]
VIRTIO_FS_EMULATION_ENABLE False(0)
VIRTIO_FS_EMULATION_NUM_VF 0
VIRTIO_FS_EMULATION_NUM_PF 0
VIRTIO_FS_EMU_SUBSYSTEM_VENDOR_ID 6900
VIRTIO_FS_EMULATION_SUBSYSTEM_ID 4186
VIRTIO_FS_EMULATION_NUM_MSIX 2
VIRTIO_FS_EMULATION_NUM_VF_MSIX 0
STRAP_SD_OR_MH False(0)
OFF_BOARD_SERIALIZER False(0)
PCI_BUS01_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS01_SWITCH_INDEX 0
PCI_BUS01_SPEED PCI_GEN_1(0)
PCI_BUS01_ASPM False(0)
PCI_BUS01_WIDTH PCI_INACTIVE(0)
PCI_BUS00_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS00_SWITCH_INDEX 0
PCI_BUS00_SPEED PCI_GEN_1(0)
PCI_BUS00_ASPM False(0)
PCI_BUS00_WIDTH PCI_INACTIVE(0)
PCI_BUS03_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS03_SWITCH_INDEX 0
PCI_BUS03_SPEED PCI_GEN_1(0)
PCI_BUS03_ASPM False(0)
PCI_BUS03_WIDTH PCI_INACTIVE(0)
PCI_BUS02_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS02_SWITCH_INDEX 0
PCI_BUS02_SPEED PCI_GEN_1(0)
PCI_BUS02_ASPM False(0)
PCI_BUS02_WIDTH PCI_INACTIVE(0)
PCI_BUS05_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS05_SWITCH_INDEX 0
PCI_BUS05_SPEED PCI_GEN_1(0)
PCI_BUS05_ASPM False(0)
PCI_BUS05_WIDTH PCI_INACTIVE(0)
PCI_BUS04_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS04_SWITCH_INDEX 0
PCI_BUS04_SPEED PCI_GEN_1(0)
PCI_BUS04_ASPM False(0)
PCI_BUS04_WIDTH PCI_INACTIVE(0)
PCI_BUS07_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS07_SWITCH_INDEX 0
PCI_BUS07_SPEED PCI_GEN_1(0)
PCI_BUS07_ASPM False(0)
PCI_BUS07_WIDTH PCI_INACTIVE(0)
PCI_BUS06_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS06_SWITCH_INDEX 0
PCI_BUS06_SPEED PCI_GEN_1(0)
PCI_BUS06_ASPM False(0)
PCI_BUS06_WIDTH PCI_INACTIVE(0)
PCI_BUS11_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS11_SWITCH_INDEX 0
PCI_BUS11_SPEED PCI_GEN_1(0)
PCI_BUS11_ASPM False(0)
PCI_BUS11_WIDTH PCI_INACTIVE(0)
PCI_BUS10_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS10_SWITCH_INDEX 0
PCI_BUS10_SPEED PCI_GEN_1(0)
PCI_BUS10_ASPM False(0)
PCI_BUS10_WIDTH PCI_INACTIVE(0)
PCI_BUS13_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS13_SWITCH_INDEX 0
PCI_BUS13_SPEED PCI_GEN_1(0)
PCI_BUS13_ASPM False(0)
PCI_BUS13_WIDTH PCI_INACTIVE(0)
PCI_BUS12_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS12_SWITCH_INDEX 0
PCI_BUS12_SPEED PCI_GEN_1(0)
PCI_BUS12_ASPM False(0)
PCI_BUS12_WIDTH PCI_INACTIVE(0)
PCI_BUS15_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS15_SWITCH_INDEX 0
PCI_BUS15_SPEED PCI_GEN_1(0)
PCI_BUS15_ASPM False(0)
PCI_BUS15_WIDTH PCI_INACTIVE(0)
PCI_BUS14_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS14_SWITCH_INDEX 0
PCI_BUS14_SPEED PCI_GEN_1(0)
PCI_BUS14_ASPM False(0)
PCI_BUS14_WIDTH PCI_INACTIVE(0)
PCI_BUS17_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS17_SWITCH_INDEX 0
PCI_BUS17_SPEED PCI_GEN_1(0)
PCI_BUS17_ASPM False(0)
PCI_BUS17_WIDTH PCI_INACTIVE(0)
PCI_BUS16_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS16_SWITCH_INDEX 0
PCI_BUS16_SPEED PCI_GEN_1(0)
PCI_BUS16_ASPM False(0)
PCI_BUS16_WIDTH PCI_INACTIVE(0)
PCI_BUS21_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS21_SWITCH_INDEX 0
PCI_BUS21_SPEED PCI_GEN_1(0)
PCI_BUS21_ASPM False(0)
PCI_BUS21_WIDTH PCI_INACTIVE(0)
PCI_BUS20_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS20_SWITCH_INDEX 0
PCI_BUS20_SPEED PCI_GEN_1(0)
PCI_BUS20_ASPM False(0)
PCI_BUS20_WIDTH PCI_INACTIVE(0)
PCI_BUS23_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS23_SWITCH_INDEX 0
PCI_BUS23_SPEED PCI_GEN_1(0)
PCI_BUS23_ASPM False(0)
PCI_BUS23_WIDTH PCI_INACTIVE(0)
PCI_BUS22_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS22_SWITCH_INDEX 0
PCI_BUS22_SPEED PCI_GEN_1(0)
PCI_BUS22_ASPM False(0)
PCI_BUS22_WIDTH PCI_INACTIVE(0)
PCI_BUS25_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS25_SWITCH_INDEX 0
PCI_BUS25_SPEED PCI_GEN_1(0)
PCI_BUS25_ASPM False(0)
PCI_BUS25_WIDTH PCI_INACTIVE(0)
PCI_BUS24_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS24_SWITCH_INDEX 0
PCI_BUS24_SPEED PCI_GEN_1(0)
PCI_BUS24_ASPM False(0)
PCI_BUS24_WIDTH PCI_INACTIVE(0)
PCI_BUS27_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS27_SWITCH_INDEX 0
PCI_BUS27_SPEED PCI_GEN_1(0)
PCI_BUS27_ASPM False(0)
PCI_BUS27_WIDTH PCI_INACTIVE(0)
PCI_BUS26_HIERARCHY_TYPE PCIE_ENDPOINT(0)
PCI_BUS26_SWITCH_INDEX 0
PCI_BUS26_SPEED PCI_GEN_1(0)
PCI_BUS26_ASPM False(0)
PCI_BUS26_WIDTH PCI_INACTIVE(0)
PCI_SWITCH1_UPSTRAEM_PORT_PEX 0
PCI_SWITCH1_UPSTRAEM_PORT_BUS 0
PCI_SWITCH0_UPSTRAEM_PORT_PEX 0
PCI_SWITCH0_UPSTRAEM_PORT_BUS 0
PCI_SWITCH2_UPSTRAEM_PORT_PEX 0
PCI_SWITCH2_UPSTRAEM_PORT_BUS 0
CQE_COMPRESSION BALANCED(0)
IP_OVER_VXLAN_EN False(0)
MKEY_BY_NAME False(0)
PRIO_TAG_REQUIRED_EN False(0)
UCTX_EN True(1)
REAL_TIME_CLOCK_ENABLE False(0)
RDMA_SELECTIVE_REPEAT_EN False(0)
PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
TUNNEL_ECN_COPY_DISABLE False(0)
LRO_LOG_TIMEOUT0 6
LRO_LOG_TIMEOUT1 7
LRO_LOG_TIMEOUT2 8
LRO_LOG_TIMEOUT3 13
LOG_TX_PSN_WINDOW 9
VF_MIGRATION_MODE DEVICE_DEFAULT(0)
LOG_MAX_OUTSTANDING_WQE 7
ROCE_ADAPTIVE_ROUTING_EN False(0)
TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0)
USER_PROGRAMMABLE_CC True(1)
PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0)
PCC_INT_NP_RTT_DSCP 26
PCC_INT_NP_RTT_DSCP_EN False(0)
PCC_INT_NP_RTT_DATA_MODE RTT_V0(64)
PCC_INT_EN False(0)
PCC_INT_SYSTEM_RTT 1
STEERING_CACHE_REFRESH 0
TX_SCHEDULER_LOCALITY_MODE DEVICE_DEFAULT(0)
ICM_CACHE_MODE DEVICE_DEFAULT(0)
TX_SCHEDULER_FWS_REACTIVITY DIRECT(1)
HAIRPIN_DATA_BUFFER_LOCK False(0)
TX_SCHEDULER_BURST 0
ZERO_TOUCH_TUNING_ENABLE False(0)
TX_PCI_DATA_FETCH_LATENCY 40
ROCE_CC_DCQCN_COMPATIBILITY_MODE DEVICE_DEFAULT(0)
ROCE_CC_LEGACY_DCQCN_SW False(0)
LOG_MAX_QUEUE 17
LOG_MAX_OUTSTANDING_READ_ATOMIC 0
MULTIPATH_DSCP DEVICE_DEFAULT(0)
SWP_L4_CHECKSUM_MODE DEVICE_DEFAULT(0)
UPT_EMULATION_ENABLE False(0)
LARGE_MTU_TWEAK_64 False(0)
AES_XTS_TWEAK_INC_64 False(0)
CRYPTO_POLICY UNRESTRICTED(1)
RDE_DISABLE False(0)
PLDM_FW_UPDATE_DISABLE False(0)
RBT_DISABLE False(0)
PCIE_SMBUS_DISABLE False(0)
PCIE_IN_BAND_VDM_DISABLE False(0)
HOST_PRIV_FLASH_ACCESS DEVICE_DEFAULT(0)
HOST_PRIV_NV_HOST DEVICE_DEFAULT(0)
HOST_PRIV_NV_PORT DEVICE_DEFAULT(0)
HOST_PRIV_NV_GLOBAL DEVICE_DEFAULT(0)
HOST_PRIV_NV_INTERNAL_CPU DEVICE_DEFAULT(0)
HOST_PRIV_PCC_UPDATE DEVICE_DEFAULT(0)
HOST_PRIV_FW_UPDATE DEVICE_DEFAULT(0)
HOST_PRIV_NIC_RESET DEVICE_DEFAULT(0)
LOG_DCR_HASH_TABLE_SIZE 11
MAX_PACKET_LIFETIME 0
DCR_LIFO_SIZE 16384
LINK_TYPE_P1 ETH(2)
LINK_TYPE_P2 ETH(2)
ROCE_CC_PRIO_MASK_P1 255
ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0)
ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0)
IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0)
ROCE_CC_PRIO_MASK_P2 255
ROCE_CC_CNP_MODERATION_P2 DEVICE_DEFAULT(0)
ROCE_CC_SHAPER_COALESCE_P2 DEVICE_DEFAULT(0)
IB_CC_SHAPER_COALESCE_P2 DEVICE_DEFAULT(0)
CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1)
CLAMP_TGT_RATE_P1 False(0)
RPG_TIME_RESET_P1 300
RPG_BYTE_RESET_P1 32767
RPG_THRESHOLD_P1 1
RPG_MAX_RATE_P1 0
RPG_AI_RATE_P1 5
RPG_HAI_RATE_P1 50
RPG_GD_P1 11
RPG_MIN_DEC_FAC_P1 50
RPG_MIN_RATE_P1 1
RATE_TO_SET_ON_FIRST_CNP_P1 0
DCE_TCP_G_P1 1019
DCE_TCP_RTT_P1 1
RATE_REDUCE_MONITOR_PERIOD_P1 4
INITIAL_ALPHA_VALUE_P1 1023
MIN_TIME_BETWEEN_CNPS_P1 4
CNP_802P_PRIO_P1 6
CNP_DSCP_P1 48
CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1)
CLAMP_TGT_RATE_P2 False(0)
RPG_TIME_RESET_P2 300
RPG_BYTE_RESET_P2 32767
RPG_THRESHOLD_P2 1
RPG_MAX_RATE_P2 0
RPG_AI_RATE_P2 5
RPG_HAI_RATE_P2 50
RPG_GD_P2 11
RPG_MIN_DEC_FAC_P2 50
RPG_MIN_RATE_P2 1
RATE_TO_SET_ON_FIRST_CNP_P2 0
DCE_TCP_G_P2 1019
DCE_TCP_RTT_P2 1
RATE_REDUCE_MONITOR_PERIOD_P2 4
INITIAL_ALPHA_VALUE_P2 1023
MIN_TIME_BETWEEN_CNPS_P2 4
CNP_802P_PRIO_P2 6
CNP_DSCP_P2 48
LLDP_NB_DCBX_P1 False(0)
LLDP_NB_RX_MODE_P1 OFF(0)
LLDP_NB_TX_MODE_P1 OFF(0)
LLDP_NB_DCBX_P2 False(0)
LLDP_NB_RX_MODE_P2 OFF(0)
LLDP_NB_TX_MODE_P2 OFF(0)
ROCE_RTT_RESP_DSCP_P1 0
ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0)
ROCE_RTT_RESP_DSCP_P2 0
ROCE_RTT_RESP_DSCP_MODE_P2 DEVICE_DEFAULT(0)
DCBX_IEEE_P1 True(1)
DCBX_CEE_P1 True(1)
DCBX_WILLING_P1 True(1)
DCBX_IEEE_P2 True(1)
DCBX_CEE_P2 True(1)
DCBX_WILLING_P2 True(1)
KEEP_ETH_LINK_UP_P1 True(1)
KEEP_IB_LINK_UP_P1 False(0)
KEEP_LINK_UP_ON_BOOT_P1 False(0)
KEEP_LINK_UP_ON_STANDBY_P1 False(0)
DO_NOT_CLEAR_PORT_STATS_P1 False(0)
AUTO_POWER_SAVE_LINK_DOWN_P1 False(0)
KEEP_ETH_LINK_UP_P2 True(1)
KEEP_IB_LINK_UP_P2 False(0)
KEEP_LINK_UP_ON_BOOT_P2 False(0)
KEEP_LINK_UP_ON_STANDBY_P2 False(0)
DO_NOT_CLEAR_PORT_STATS_P2 False(0)
AUTO_POWER_SAVE_LINK_DOWN_P2 False(0)
NUM_OF_VL_P1 _4_VLs(3)
NUM_OF_TC_P1 _8_TCs(0)
NUM_OF_PFC_P1 8
VL15_BUFFER_SIZE_P1 0
QOS_TRUST_STATE_P1 TRUST_PCP(1)
ETS_SCHED_MODE_P1 device_default(0)
VL_BUFFER_ALLOCATION_P1 FULL_DYNAMIC(1)
NUM_OF_VL_P2 _4_VLs(3)
NUM_OF_TC_P2 _8_TCs(0)
NUM_OF_PFC_P2 8
VL15_BUFFER_SIZE_P2 0
QOS_TRUST_STATE_P2 TRUST_PCP(1)
ETS_SCHED_MODE_P2 device_default(0)
VL_BUFFER_ALLOCATION_P2 FULL_DYNAMIC(1)
DUP_MAC_ACTION_P1 LAST_CFG(0)
MPFS_MC_LOOPBACK_DISABLE_P1 False(0)
MPFS_UC_LOOPBACK_DISABLE_P1 False(0)
UNKNOWN_UPLINK_MAC_FLOOD_P1 False(0)
SRIOV_IB_ROUTING_MODE_P1 LID(1)
IB_ROUTING_MODE_P1 LID(1)
DUP_MAC_ACTION_P2 LAST_CFG(0)
MPFS_MC_LOOPBACK_DISABLE_P2 False(0)
MPFS_UC_LOOPBACK_DISABLE_P2 False(0)
UNKNOWN_UPLINK_MAC_FLOOD_P2 False(0)
SRIOV_IB_ROUTING_MODE_P2 LID(1)
IB_ROUTING_MODE_P2 LID(1)
PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0)
PHY_RATE_MASK_OVERRIDE_P1 False(0)
PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0)
PHY_AUTO_NEG_P2 DEVICE_DEFAULT(0)
PHY_RATE_MASK_OVERRIDE_P2 False(0)
PHY_FEC_OVERRIDE_P2 DEVICE_DEFAULT(0)
PF_TOTAL_SF 0
PF_SD_GROUP 0
PF_DEVICE_ID_ENABLE False(0)
PF_SF_BAR_SIZE 0
PF_NUM_PF_MSIX 63
PF_DEVICE_ID 41692
SILENT_MODE False(0)
MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0)
ROCE_CONTROL ROCE_ENABLE(2)
PCI_WR_ORDERING per_mkey(0)
MULTI_PORT_VHCA_EN False(0)
ECPF_ESWITCH_MANAGER ECPF(1)
ECPF_PAGE_SUPPLIER ECPF(1)
PORT_OWNER True(1)
ALLOW_RD_COUNTERS True(1)
RENEG_ON_CHANGE True(1)
TRACER_ENABLE True(1)
IP_VER IPv4(0)
BOOT_UNDI_NETWORK_WAIT 0
UEFI_HII_EN True(1)
BOOT_DBG_LOG False(0)
UEFI_LOGS DISABLED(0)
BOOT_VLAN 1
LEGACY_BOOT_PROTOCOL PXE(1)
BOOT_INTERRUPT_DIS False(0)
BOOT_LACP_DIS True(1)
BOOT_VLAN_EN False(0)
BOOT_PKEY 0
BAR_PAGE_ALIGNMENT DEVICE_DEFAULT(0)
MANAGEMENT_PF_MODE DEVICE_DEFAULT(0)
P2P_ORDERING_MODE DEVICE_DEFAULT(0)
EXP_ROM_VIRTIO_NET_PXE_ENABLE False(0)
EXP_ROM_VIRTIO_NET_UEFI_ARM_ENABLE False(0)
EXP_ROM_VIRTIO_NET_UEFI_x86_ENABLE False(0)
EXP_ROM_VIRTIO_BLK_UEFI_ARM_ENABLE False(0)
EXP_ROM_VIRTIO_BLK_UEFI_x86_ENABLE False(0)
EXP_ROM_NVME_UEFI_x86_ENABLE True(1)
ATS_ENABLED False(0)
DYNAMIC_VF_MSIX_TABLE False(0)
EXP_ROM_UEFI_ARM_ENABLE True(1)
EXP_ROM_UEFI_x86_ENABLE True(1)
EXP_ROM_PXE_ENABLE True(1)
ADVANCED_PCI_SETTINGS False(0)
SAFE_MODE_THRESHOLD 10
SAFE_MODE_ENABLE True(1)