Hello,
I am trying to implement a DNS Filter (UDP/TCP port 53) using a Bluefield-3 DPU Running DOCA 3.1.0.
The DPU runs the following software suite:
BlueField3
NVIDIA BlueField-3 B3220 P-Series FHHL DPU; 200GbE (default mode) / NDR200 IB; Dual-port QSFP112; PCIe Gen5.0 x16 with x16 PCIe extension option; 16 Arm cores; 32GB on-board DDR; integrated BMC; Crypto Enabled
900-9D3B6-00CV-A_Ax
BlueField3
Part Number:
Description:
NVIDIA BlueField-3 B3220 P-Series FHHL DPU; 200GbE (default mode) / NDR200 IB; Dual-port QSFP112; PCIe Gen5.0 x16 with x16 PCIe extension option; 16 Arm cores; 32GB on-board DDR; integrated BMC; Crypto Enabled
900-9D3B6-00CV-A_Ax
PSID:
0000000884
PCI Device Name:
/dev/mst/mt41692_pciconf0
Base MAC:
c25736ddbc0
Versions:
Current
Available
FW
32.46.1006
N/A
PXE
3.8.0100
N/A
14.39.0013
N/A
UEFI Virtio blk
22.4.0014
N/A
JEFI Virtio net
21.4.0013
N/A
I am trying to build as a starting test a simple sample that does the following:
PC_1 connected to p1 of the DPU. PC_1 has IP 10.10.10.2
PC_0 connected to p0 of the DPU. PC_2 has IP 10.10.10.4
The first simple test is to create an hairpin between p1 and p0 so that PC_1 can ping PC_0.
+---------------------------+
| NVIDIA BlueField-3 DPU |
| |
PC_1 | Port 1 Port 0 | PC_0
10.10.10.2 ---> | [ eth1 ] [ eth0 ] | <--- 10.10.10.4
| |
+---------------------------+
I first tried to create an hairpin between p0 and p1 using DOCA_FLOW_FWD_PORT (hardware hairpin) to no avail. The is found at the bottom of this post.
OVS was shutdown to make sure it does not steal the ports from DOCA but that did not work. I then realize that for
some reasons my version of kernel does not support multiport switching even though I had downloaded DOCA 3.1.0 full
release a few month ago. Would that be a reason why it does not work.
I am really hesitant to download the new DOCA 3.2.1 to avoid breaking other software.
Is there another way to implement hairpin between p0 and p1 simply via software without reconfigure the kernel.
I have another fundamental question regarding all the Scalable Functions and representors. How to test whether the
configuration of the DPU allows DOCA FLow to control the physical ports instead of the Linux Kernel of the ARM CPUs
or OVSwitch. Even when I shutdown and disable OVSwitch the DOCA FLow app does not see any traffic.
Thanks
Code:
flow_drop_vnf_main.c
/*
* Copyright (c) 2022-2025 NVIDIA CORPORATION AND AFFILIATES. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <stdlib.h>
#include <doca_argp.h>
#include <doca_flow.h>
#include <doca_log.h>
#include <flow_common.h>
#include <dpdk_utils.h>
DOCA_LOG_REGISTER(FLOW_DROP::MAIN);
/* Sample's Logic */
doca_error_t flow_drop(int nb_queues);
/*
* Sample main function
*
* @argc [in]: command line arguments size
* @argv [in]: array of command line arguments
* @return: EXIT_SUCCESS on success and EXIT_FAILURE otherwise
*/
int main(int argc, char **argv)
{
doca_error_t result;
struct doca_log_backend *sdk_log;
int exit_status = EXIT_FAILURE;
struct flow_dev_ctx flow_dev_ctx = {};
struct application_dpdk_config dpdk_config = {
.port_config.nb_ports = 2,
.port_config.nb_queues = 1,
};
/* Register a logger backend */
result = doca_log_backend_create_standard();
if (result != DOCA_SUCCESS)
goto sample_exit;
/* Register a logger backend for internal SDK errors and warnings */
result = doca_log_backend_create_with_file_sdk(stderr, &sdk_log);
if (result != DOCA_SUCCESS)
goto sample_exit;
result = doca_log_backend_set_sdk_level(sdk_log, DOCA_LOG_LEVEL_WARNING);
if (result != DOCA_SUCCESS)
goto sample_exit;
DOCA_LOG_INFO("Starting the sample");
result = doca_argp_init(NULL, &flow_dev_ctx);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to init ARGP resources: %s", doca_error_get_descr(result));
goto sample_exit;
}
result = register_flow_device_params(NULL);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to register flow device params: %s", doca_error_get_descr(result));
goto argp_cleanup;
}
doca_argp_set_dpdk_program(flow_init_dpdk);
result = doca_argp_start(argc, argv);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to parse sample input: %s", doca_error_get_descr(result));
goto argp_cleanup;
}
result = init_doca_flow_devs(&flow_dev_ctx);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to init flow devices: %s", doca_error_get_descr(result));
goto argp_cleanup;
}
/* update queues and ports */
result = dpdk_queues_and_ports_init(&dpdk_config);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to update ports and queues");
goto dpdk_cleanup;
}
/* run sample */
result = flow_drop(dpdk_config.port_config.nb_queues);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("flow_drop() encountered an error: %s", doca_error_get_descr(result));
goto dpdk_ports_queues_cleanup;
}
exit_status = EXIT_SUCCESS;
dpdk_ports_queues_cleanup:
dpdk_queues_and_ports_fini(&dpdk_config);
dpdk_cleanup:
dpdk_fini_with_devs(dpdk_config.port_config.nb_ports);
argp_cleanup:
doca_argp_destroy();
sample_exit:
if (exit_status == EXIT_SUCCESS)
DOCA_LOG_INFO("Sample finished successfully");
else
DOCA_LOG_INFO("Sample finished with errors");
return exit_status;
}
flow_drop_vnf_sample.c
/*
* Copyright (c) 2022-2025 NVIDIA CORPORATION AND AFFILIATES. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <string.h>
#include <unistd.h>
#include <doca_log.h>
#include <doca_flow.h>
#include <flow_common.h>
typedef enum {
PORT_FWD_PIPE_ENTRY = 0,
} pipe_entry_index;
DOCA_LOG_REGISTER(FLOW_DROP);
/*
* Create DOCA Flow pipe that forwards all the traffic to the other port
*
* @port [in]: port of the pipe
* @port_id [in]: port ID of the pipe
* @pipe [out]: created pipe pointer
* @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise.
*/
static doca_error_t create_port_fwd_pipe(struct doca_flow_port *port, int port_id, struct doca_flow_pipe **pipe)
{
struct doca_flow_match match;
struct doca_flow_monitor monitor;
struct doca_flow_actions actions, *actions_arr[NB_ACTIONS_ARR];
struct doca_flow_fwd fwd;
struct doca_flow_pipe_cfg *pipe_cfg;
doca_error_t result;
memset(&match, 0, sizeof(match));
memset(&monitor, 0, sizeof(monitor));
memset(&actions, 0, sizeof(actions));
memset(&fwd, 0, sizeof(fwd));
monitor.counter_type = DOCA_FLOW_RESOURCE_TYPE_NON_SHARED;
actions_arr[0] = &actions;
result = doca_flow_pipe_cfg_create(&pipe_cfg, port);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to create doca_flow_pipe_cfg: %s", doca_error_get_descr(result));
return result;
}
result = set_flow_pipe_cfg(pipe_cfg, "PORT_FWD_PIPE", DOCA_FLOW_PIPE_BASIC, false);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to set doca_flow_pipe_cfg: %s", doca_error_get_descr(result));
goto destroy_pipe_cfg;
}
result = doca_flow_pipe_cfg_set_match(pipe_cfg, &match, NULL);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to set doca_flow_pipe_cfg match: %s", doca_error_get_descr(result));
goto destroy_pipe_cfg;
}
result = doca_flow_pipe_cfg_set_actions(pipe_cfg, actions_arr, NULL, NULL, NB_ACTIONS_ARR);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to set doca_flow_pipe_cfg actions: %s", doca_error_get_descr(result));
goto destroy_pipe_cfg;
}
result = doca_flow_pipe_cfg_set_monitor(pipe_cfg, &monitor);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to set doca_flow_pipe_cfg monitor: %s", doca_error_get_descr(result));
goto destroy_pipe_cfg;
}
/* forwarding traffic to other port */
fwd.type = DOCA_FLOW_FWD_PORT;
fwd.port_id = port_id ^ 1;
result = doca_flow_pipe_create(pipe_cfg, &fwd, NULL, pipe);
destroy_pipe_cfg:
doca_flow_pipe_cfg_destroy(pipe_cfg);
return result;
}
/*
* Add DOCA Flow pipe entry to the port forwarding pipe
*
* @pipe [in]: pipe of the entry
* @status [in]: user context for adding entry
* @entry [out]: created entry pointer
* @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise.
*/
static doca_error_t add_pipe_entry(struct doca_flow_pipe *pipe,
struct entries_status *status,
struct doca_flow_pipe_entry **entry)
{
struct doca_flow_match match;
/*
* All fields are not changeable, thus we need to add only 1 entry, all values will be
* inherited from the pipe creation
*/
memset(&match, 0, sizeof(match));
return doca_flow_pipe_add_entry(0, pipe, &match, NULL, NULL, NULL, 0, status, entry);
}
/*
* Run flow_drop sample (simplified - port forwarding only)
*
* @nb_queues [in]: number of queues the sample will use
* @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise.
*/
doca_error_t flow_drop(int nb_queues)
{
const int nb_ports = 2;
struct flow_resources resource = {0};
uint32_t nr_shared_resources[SHARED_RESOURCE_NUM_VALUES] = {0};
struct doca_flow_port *ports[nb_ports];
uint32_t actions_mem_size[nb_ports];
struct doca_flow_pipe *port_fwd_pipe;
/* Total number of entries - 1 for port forwarding pipe */
const int nb_entries = 1;
struct doca_flow_pipe_entry *entry[nb_ports][nb_entries];
struct doca_flow_resource_query query_stats;
struct entries_status status;
doca_error_t result;
int port_id;
resource.nr_counters = nb_ports * nb_entries;
result = init_doca_flow(nb_queues, "vnf,hws", &resource, nr_shared_resources);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to init DOCA Flow: %s", doca_error_get_descr(result));
return result;
}
ARRAY_INIT(actions_mem_size, ACTIONS_MEM_SIZE(nb_entries));
result = init_doca_flow_vnf_ports(nb_ports, ports, actions_mem_size);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to init DOCA ports: %s", doca_error_get_descr(result));
doca_flow_destroy();
return result;
}
for (port_id = 0; port_id < nb_ports; port_id++) {
memset(&status, 0, sizeof(status));
result = create_port_fwd_pipe(ports[port_id], port_id, &port_fwd_pipe);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to create port forwarding pipe: %s", doca_error_get_descr(result));
stop_doca_flow_ports(nb_ports, ports);
doca_flow_destroy();
return result;
}
result = add_pipe_entry(port_fwd_pipe, &status, &entry[port_id][PORT_FWD_PIPE_ENTRY]);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to add port forwarding entry: %s", doca_error_get_descr(result));
stop_doca_flow_ports(nb_ports, ports);
doca_flow_destroy();
return result;
}
result = doca_flow_entries_process(ports[port_id], 0, DEFAULT_TIMEOUT_US, nb_entries);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to process entries: %s", doca_error_get_descr(result));
stop_doca_flow_ports(nb_ports, ports);
doca_flow_destroy();
return result;
}
if (status.nb_processed != nb_entries || status.failure) {
DOCA_LOG_ERR("Failed to process entries");
stop_doca_flow_ports(nb_ports, ports);
doca_flow_destroy();
return DOCA_ERROR_BAD_STATE;
}
}
/* wait few seconds for packets to arrive so query will not return zero */
DOCA_LOG_INFO("Wait few seconds for packets to arrive");
sleep(5);
DOCA_LOG_INFO("===================================================");
for (port_id = 0; port_id < nb_ports; port_id++) {
result = doca_flow_resource_query_entry(entry[port_id][PORT_FWD_PIPE_ENTRY], &query_stats);
if (result != DOCA_SUCCESS) {
DOCA_LOG_ERR("Failed to query entry: %s", doca_error_get_descr(result));
stop_doca_flow_ports(nb_ports, ports);
doca_flow_destroy();
return result;
}
DOCA_LOG_INFO("Port %d - Port forwarding pipe:", port_id);
DOCA_LOG_INFO("\tTotal bytes: %ld", query_stats.counter.total_bytes);
DOCA_LOG_INFO("\tTotal packets: %ld", query_stats.counter.total_pkts);
DOCA_LOG_INFO("===================================================");
}
result = stop_doca_flow_ports(nb_ports, ports);
doca_flow_destroy();
return result;
}