DPDK IP 重组 API 返回 NULL

Question

I am new to DPDK, currently testing IP reassemble API and I am having difficulties. Below is the C++ code which I wrote to test the IP reassemble. I took the reference from the examples list provided from dpdk itself. dpdk version which I am using is 20.08 in debian machine. dpdk user guide mentions the API works on src add, dst add and packet ID, even though all three data are proper still the API returns NULL. Any kind of help will be much appreciated. Thanks in advance.

Search the Fragment Table for entry with packet’s <IPv4 Source Address, IPv4 Destination Address, Packet ID>.

If the entry is found, then check if that entry already timed-out. If yes, then free all previously received fragments, and remove information about them from the entry.

If no entry with such key is found, then try to create a new one by one of two ways:

Use as empty entry.

Delete a timed-out entry, free mbufs associated with it mbufs and store a new entry with specified key in it.

Update the entry with new fragment information and check if a packet can be reassembled (the packet’s entry contains all fragments).

If yes, then, reassemble the packet, mark table’s entry as empty and return the reassembled mbuf to the caller.

If no, then return a NULL to the caller.

CONFIG_RTE_LIBRTE_IP_FRAG=y

CONFIG_RTE_LIBRTE_IP_FRAG_DEBUG=y

CONFIG_RTE_LIBRTE_IP_FRAG_MAX_FRAG=100

CONFIG_RTE_LIBRTE_IP_FRAG_TBL_STAT=n

#define DEF_FLOW_NUM    0x1000
#define DEF_FLOW_TTL    MS_PER_S
#define IP_FRAG_TBL_BUCKET_ENTRIES  128
static uint32_t max_flow_num = DEF_FLOW_NUM;
static uint32_t max_flow_ttl = DEF_FLOW_TTL;

struct lcore_queue_conf {
    struct rte_ip_frag_tbl *frag_tbl;
    struct rte_mempool *pool;
    struct rte_ip_frag_death_row death_row; 
}__rte_cache_aligned; 
static struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];


static inline int setup_queue_tbl(struct lcore_queue_conf *qconf)
{

    uint64_t frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S * max_flow_ttl;
    
    qconf->frag_tbl = rte_ip_frag_table_create(max_flow_num, IP_FRAG_TBL_BUCKET_ENTRIES, max_flow_num, frag_cycles, rte_socket_id());

    if((qconf->frag_tbl) == NULL){ 
        RTE_LOG(ERR, IP_RSMBL, "Table Failed.");
        return -1;
    }   

    qconf->pool=rte_pktmbuf_pool_create("BUFFER", POOL_SIZE*2, POOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());

    if(qconf->pool== NULL){ 
    RTE_LOG(ERR, IP_RSMBL, "Mem Pool Failed.");
    return -1;
    }

    return 0;
}


static inline void reassemble(struct rte_mbuf *reassemblepkt, struct lcore_queue_conf *qconf, uint64_t cur_tsc)
{
    struct rte_mbuf *mo;
    struct rte_ether_hdr *eth_hdr;
    struct rte_ipv4_hdr *ip_hdr;

    eth_hdr = rte_pktmbuf_mtod(reassemblepkt, struct rte_ether_hdr *);
    ip_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);

    if (rte_ipv4_frag_pkt_is_fragmented(ip_hdr)){
        reassemblepkt->l2_len = sizeof(*eth_hdr);
        reassemblepkt->l3_len = sizeof(*ip_hdr);
        
        int ip_len;
        ip_len = rte_be_to_cpu_16(ip_hdr->total_length);

        mo = rte_ipv4_frag_reassemble_packet(qconf->frag_tbl, &qconf->death_row, reassemblepkt, cur_tsc, ip_hdr);
            
        if (mo == NULL){
        cout << "Total Length: " << ip_len << ", l3 length: " << reassemblepkt->l3_len << " ,Packet ID: " << ip_hdr->packet_id << " , src add:" << ip_hdr->src_addr << " , dst add:" << ip_hdr->dst_addr << endl; 
        RTE_LOG(ERR, IP_RSMBL, "Reassemble Failed.\n");
        }   

        if ((mo != reassemblepkt) && (mo != NULL)){
            cout << "Reassemble is success." << endl;
            reassemblepkt = mo;
        }
    }
}

static int
lcore_main(struct lcore_queue_conf *qconf)
{
    int rx, rec;
    struct rte_mbuf *bufs[BUFFER_LENGTH];
    uint64_t cur_tsc;   
    int i;

    RTE_ETH_FOREACH_DEV(port){
    cout << "RX Thread: Socket ID: " << rte_socket_id() << endl;
    cout << "RX Thread: lcore count: " << dec << rte_lcore_count() << endl;
    cout << "RX Thread: lcore ID: " << rte_lcore_id() << endl; 
    cout << "RX Thread Started." << endl;
    cout << "=====================================================" << endl;
    }   
        
        while(!quit_signal) {
            cur_tsc = rte_rdtsc();
            RTE_ETH_FOREACH_DEV(port){
                            rx=rte_eth_rx_burst(port, 0, bufs, BUFFER_LENGTH);                          
                            if(unlikely(rx == 0 ))
                                continue;
                            if(rx){ 
                                for(i=0; i<rx; i++)
                                reassemble(bufs[i], qconf, cur_tsc);

                                rte_ip_frag_free_death_row(&qconf->death_row, PREFETCH_OFFSET);
                                rec = rte_ring_enqueue_burst(Myring, (void **)bufs, rx, NULL);  
                            }                       
            }
        }
        return 0;
}


int main(int argc, char *argv[])
{

    int ret;
    uint16_t portcheck;
    DPDKPORT p1;
    struct lcore_queue_conf *qconf;


     /* catch ctrl-c so we can print on exit */
    signal(SIGINT, int_handler);

    /* EAL setup */
    ret=rte_eal_init(argc, argv);
    cout << "=====================================================" << endl;
    if(ret < 0)
        cout << "EAL initialising failed." <<  strerror(-ret) << endl;
    else
        cout << "EAL initialisation success." << endl;

    qconf = &lcore_queue_conf[rte_get_master_lcore()];

    if(setup_queue_tbl(qconf) != 0)
    rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno));
.
.
.
.

RTE_ETH_FOREACH_DEV(portcheck){
    if(p1.eth_init(portcheck, qconf->pool) != 0)
        rte_exit(EXIT_FAILURE, "Ethernet port initialisation failed.");
    }


    /* Master core call */  
    lcore_main(qconf);

    return 0;
}

The Output as follows.

EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL:   Invalid NUMA socket, default to 0
EAL:   Invalid NUMA socket, default to 0
EAL:   using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL:   Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
=====================================================
EAL initialisation success.
USER1: rte_ip_frag_table_create: allocated of 201326720 bytes at socket 0
PORT 0: Ethernet configuration success.
TX queue configuration success.
RX queue configuration success.
PORT 0: NIC started successfully.
PORT 0: Enabled promiscuous mode.
MAC Addr b4:96:91:3f:21:b6
=====================================================
RX Thread: Socket ID: 0
RX Thread: lcore count: 12
RX Thread: lcore ID: 0
RX Thread Started.
=====================================================
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831203304 , dst add:11304
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 32, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 32, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295

I added printf inside the ip_reassemble example program and same thing is observed. unable to figure it out why is it happening. When I open the input pcap in Wireshark, Wireshark is able to reassemble properly. Please anyone suggest me a pcap or a way to check if IP reassemble is working fine.

/* process this fragment. */
        mo = rte_ipv4_frag_reassemble_packet(tbl, dr, m, tms, ip_hdr);
        if (mo == NULL) {
            /* no packet to send out. */
            printf("IP reassemble failed\n");
            return;
        }

/* we have our packet reassembled. */
if (mo != m) {
    printf("IP reassemble success\n");
    m = mo;
    eth_hdr = rte_pktmbuf_mtod(m,
        struct rte_ether_hdr *);
    ip_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
}

root@user:/home/user/dpdk/examples/ip_reassembly# ./build/ip_reassembly -l 1 -- -p 1 -q 2
EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL:   Invalid NUMA socket, default to 0
EAL:   Invalid NUMA socket, default to 0
EAL:   using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL:   Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
0x7ffdd2cbd50e
IP_RSMBL: Creating LPM table on socket 0
IP_RSMBL: Creating LPM6 table on socket 0
USER1: rte_ip_frag_table_create: allocated of 25165952 bytes at socket 0
Initializing port 0 ... Port 0 modified RSS hash function based on hardware support,requested:0xa38c configured:0x8104
 Address:B4:96:91:3F:21:B6
txq=1,0,0 

IP_RSMBL: Socket 0: adding route 100.10.0.0/16 (port 0)
IP_RSMBL: Socket 0: adding route 100.20.0.0/16 (port 1)
IP_RSMBL: Socket 0: adding route 100.30.0.0/16 (port 2)
IP_RSMBL: Socket 0: adding route 100.40.0.0/16 (port 3)
IP_RSMBL: Socket 0: adding route 100.50.0.0/16 (port 4)
IP_RSMBL: Socket 0: adding route 100.60.0.0/16 (port 5)
IP_RSMBL: Socket 0: adding route 100.70.0.0/16 (port 6)
IP_RSMBL: Socket 0: adding route 100.80.0.0/16 (port 7)
IP_RSMBL: Socket 0: adding route 0101:0101:0101:0101:0101:0101:0101:0101/48 (port 0)
IP_RSMBL: Socket 0: adding route 0201:0101:0101:0101:0101:0101:0101:0101/48 (port 1)
IP_RSMBL: Socket 0: adding route 0301:0101:0101:0101:0101:0101:0101:0101/48 (port 2)
IP_RSMBL: Socket 0: adding route 0401:0101:0101:0101:0101:0101:0101:0101/48 (port 3)
IP_RSMBL: Socket 0: adding route 0501:0101:0101:0101:0101:0101:0101:0101/48 (port 4)
IP_RSMBL: Socket 0: adding route 0601:0101:0101:0101:0101:0101:0101:0101/48 (port 5)
IP_RSMBL: Socket 0: adding route 0701:0101:0101:0101:0101:0101:0101:0101/48 (port 6)
IP_RSMBL: Socket 0: adding route 0801:0101:0101:0101:0101:0101:0101:0101/48 (port 7)

Checking link status.......................................
done
Port0 Link Up. Speed 10000 Mbps - full-duplex
IP_RSMBL: entering main loop on lcore 1
IP_RSMBL:  -- lcoreid=1 portid=0
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
.
.
.
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
^C -- lcoreid=1 portid=0 frag tbl stat:
max entries:    4096;
entries in use: 11;
finds/inserts:  0;
entries added:  0;
entries deleted by timeout: 0;
entries reused by timeout:  0;
total add failures: 0;
add no-space failures:  0;
add hash-collisions failures:   0;
TX bursts:  0
TX packets _queued: 0
TX packets dropped: 0
TX packets send:    0
received signal: 2, exiting

[EDIT-2]: Either Last fragment or first fragment length is 0. I dont know this has anything to do with below details but just mentioning.

NIC(10GB NIC) Ethernet controller: Intel Corporation Ethernet Controller 10G X550T

Architecture:        x86_64
CPU op-mode(s):      32-bit, 64-bit
Byte Order:          Little Endian
Address sizes:       39 bits physical, 48 bits virtual
CPU(s):              12
On-line CPU(s) list: 0-11
Thread(s) per core:  2
Core(s) per socket:  6
Socket(s):           1
NUMA node(s):        1
Vendor ID:           GenuineIntel
CPU family:          6
Model:               158
Model name:          Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz
Stepping:            10
CPU MHz:             3185.543
CPU max MHz:         3200.0000
CPU min MHz:         800.0000
BogoMIPS:            6384.00
Virtualization:      VT-x
L1d cache:           32K
L1i cache:           32K
L2 cache:            256K
L3 cache:            12288K
NUMA node0 CPU(s):   0-11
Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm arat pln pts hwp hwp_notify hwp_act_window hwp_epp md_clear flush_l1d

Output of your code..

    EAL: Detected 12 lcore(s)
    EAL: Detected 1 NUMA nodes
    EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
    EAL: Selected IOVA mode 'VA'
    EAL: Probing VFIO support...
    EAL: VFIO support initialized
    EAL:   Invalid NUMA socket, default to 0
    EAL:   Invalid NUMA socket, default to 0
    EAL:   using IOMMU type 1 (Type 1)
    EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
    EAL:   Invalid NUMA socket, default to 0
    EAL: No legacy callbacks, legacy socket not created
    USER1: rte_ip_frag_table_create: allocated of 201326720 bytes at socket 0
    Port 0 MAC: b4 96 91 3f 21 b6
    RX Thread: Socket ID: 0
    RX Thread: lcore count: 12
    RX Thread: lcore ID: 0
    
    
    mb: 0x172980e80
    fp: 0x17e05c5c0
    offset: 0
    , IPLen: 24
    , ipflag: 8192
    mb_end: (nil)
    ERR, IP_RSMBL, Reassemble Failed.
    mb: 0x172980540
    fp: 0x17e05c5c0
    offset: 24
    , IPLen: 24
    , ipflag: 8192
    mb_end: (nil)
    ERR, IP_RSMBL, Reassemble Failed.
    mb: 0x17297fc00
    fp: 0x17e05c5c0
    offset: 48
    , IPLen: 24
    , ipflag: 8192
    mb_end: (nil)
    ERR, IP_RSMBL, Reassemble Failed.
    mb: 0x17297f2c0
    fp: 0x17e05c5c0
    offset: 72
    , IPLen: 24
    , ipflag: 8192
    idx: 4, frags: 0x4ip_frag_process:145 invalid fragmented packet:
    ipv4_frag_pkt: 0x17e05c5c0, key: <ffffffff3b28a8c0, 0xb500>, total_size: 4294967295, frag_size: 96, last_idx: 4
    first fragment: ofs: 0, len: 24
    last fragment: ofs: 0, len: 0
    
    mb_end: (nil)
    ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b99680
fp: 0x17551e5c0
offset: 96
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b98d40
fp: 0x17551e5c0
offset: 120
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b98400
fp: 0x17551e5c0
offset: 144
, IPLen: 12
, ipflag: 0
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172c8fb00
fp: 0x17d5885c0
offset: 8
, IPLen: 2028
, ipflag: 0
idx: -1, frags: 0x4ip_frag_process:145 invalid fragmented packet:
ipv4_frag_pkt: 0x17d5885c0, key: <2c28a8c0bbe8, 0x406>, total_size: 2036, frag_size: 4056, last_idx: 2
first fragment: ofs: 0, len: 0
last fragment: ofs: 8, len: 2028

mb_end: (nil)

Answer 1

DPDK API rte_ipv4_frag_reassemble_packet returns NULL in 2 ocassion

1. an error occurred
2. not all fragments of the packet are collected yet

Based on the code and logs shared it looks like you are

1. sending the last fragment multiple times.
2. setting time out as cur_tsc

Note:

• the easiest way to test your packet is run it against ip_reassembly example and cross check the variance.
• if (mo == NULL) it only means not sufficient fragments are received.

[edit-1] Hence I request to model your code as dpdk example ip_reassembly since assuming rte_ipv4_frag_reassemble_packet returning NULL is not always a failure.

[edit-2] cleaning up the code and adding missing libraries I am able to get this working with right set of fragment packets

Reassemble is success.dump mbuf at 0x1736966c0, iova=7b3696740, buf_len=2176
  pkt_len=5421, ol_flags=10, nb_segs=4, in_port=0
  segment at 0x1736966c0, data=0x1736967c0, data_len=1514
  Dump data at [0x1736967c0], len=64
00000000: 00 1D 09 94 65 38 68 5B 35 C0 61 B6 08 00 45 00 | ....e8h[5.a...E.
00000010: 15 1F F5 AF 00 00 40 11 00 00 83 B3 C4 DC 83 B3 | ......@.........
00000020: C4 2E 18 DB 18 DB 15 0B DC E2 06 FD 14 FF 07 29 | ...............)
00000030: 08 07 65 78 61 6D 70 6C 65 08 07 74 65 73 74 41 | ..example..testA
  segment at 0x1733a8040, data=0x1733a8162, data_len=1480
  segment at 0x1733a8980, data=0x1733a8aa2, data_len=1480
  segment at 0x1734dde40, data=0x1734ddf62, data_len=947

[edit-3] traffic generator: ./app/x86_64-native-linuxapp-gcc/pktgen -l 1-4 -- -s 0:test.pcap -P -m [2].0

application: (code snippet edited and compiled as C program) https://pastebin.pl/view/91e533e3

1. build: gcc $(PKG_CONFIG_PATH=[path-to-dpdk-pkgconfig] pkg-config --static --cflags libdpdk) dpdk.c -Wl,-Bstatic $(PKG_CONFIG_PATH=[path-to-dpdk-pkgconfig] pkg-config --static --libs libdpdk)
2. run: sudo LD_LIBRARY_PATH=/path-to-dpdk-sahred-library/ ./a.out

[edit-4] based on the live debug, issue appears to be the hub switch connecting the 2 machines. Packets are dropped or not at all forwarded. Requested to have a direct stable connection to check the logic on the @Nirmal machine. Ran the same example and show cased the output in my machine.