#include <argp.h>
#include <string.h>
#include <arpa/inet.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/in.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include "bench.h"
#include "bench_bpf_timing.h"
#include "xdp_lb_bench.skel.h"
#include "xdp_lb_bench_common.h"
#include "bpf_util.h"
#define IP4(a, b, c, d) (((__u32)(a) << 24) | ((__u32)(b) << 16) | ((__u32)(c) << 8) | (__u32)(d))
#define IP6(a, b, c, d) { (__u32)(a), (__u32)(b), (__u32)(c), (__u32)(d) }
#define TNL_DST IP4(192, 168, 1, 2)
#define REAL_INDEX 1
#define REAL_INDEX_V6 2
#define MAX_PKT_SIZE 256
#define IP_MF 0x2000
static const __u32 tnl_dst_v6[4] = { 0xfd000000, 0, 0, 2 };
static const __u8 lb_mac[ETH_ALEN] = {0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff};
static const __u8 client_mac[ETH_ALEN] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66};
static const __u8 router_mac[ETH_ALEN] = {0xde, 0xad, 0xbe, 0xef, 0x00, 0x01};
enum scenario_id {
S_TCP_V4_LRU_HIT,
S_TCP_V4_CH,
S_TCP_V6_LRU_HIT,
S_TCP_V6_CH,
S_UDP_V4_LRU_HIT,
S_UDP_V6_LRU_HIT,
S_TCP_V4V6_LRU_HIT,
S_TCP_V4_LRU_DIVERSE,
S_TCP_V4_CH_DIVERSE,
S_TCP_V6_LRU_DIVERSE,
S_TCP_V6_CH_DIVERSE,
S_UDP_V4_LRU_DIVERSE,
S_TCP_V4_LRU_MISS,
S_UDP_V4_LRU_MISS,
S_TCP_V4_LRU_WARMUP,
S_TCP_V4_SYN,
S_TCP_V4_RST_MISS,
S_PASS_V4_NO_VIP,
S_PASS_V6_NO_VIP,
S_PASS_V4_ICMP,
S_PASS_NON_IP,
S_DROP_V4_FRAG,
S_DROP_V4_OPTIONS,
S_DROP_V6_FRAG,
NUM_SCENARIOS,
};
enum lru_miss_type {
LRU_MISS_AUTO = 0,
LRU_MISS_NONE,
LRU_MISS_ALL,
LRU_MISS_FIRST,
};
#define S_BASE_ENCAP_V4 \
.expected_retval = XDP_TX, .expect_encap = true, \
.tunnel_dst = TNL_DST
#define S_BASE_ENCAP_V6 \
.expected_retval = XDP_TX, .expect_encap = true, \
.is_v6 = true, .encap_v6_outer = true, \
.tunnel_dst_v6 = { 0xfd000000, 0, 0, 2 }
#define S_BASE_ENCAP_V4V6 \
.expected_retval = XDP_TX, .expect_encap = true, \
.encap_v6_outer = true, \
.tunnel_dst_v6 = { 0xfd000000, 0, 0, 2 }
struct test_scenario {
const char *name;
const char *description;
int expected_retval;
bool expect_encap;
bool is_v6;
__u32 vip_addr;
__u32 src_addr;
__u32 tunnel_dst;
__u32 vip_addr_v6[4];
__u32 src_addr_v6[4];
__u32 tunnel_dst_v6[4];
__u16 dst_port;
__u16 src_port;
__u8 ip_proto;
__u32 vip_flags;
__u32 vip_num;
bool prepopulate_lru;
bool set_frag;
__u16 eth_proto;
bool encap_v6_outer;
__u32 flow_mask;
bool cold_lru;
bool set_syn;
bool set_rst;
bool set_ip_options;
__u32 fixed_batch_iters;
enum lru_miss_type lru_miss;
};
static const struct test_scenario scenarios[NUM_SCENARIOS] = {
[S_TCP_V4_LRU_HIT] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-lru-hit",
.description = "IPv4 TCP, LRU hit, IPIP encap",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 2, 1), .src_port = 12345,
.prepopulate_lru = true, .lru_miss = LRU_MISS_NONE,
},
[S_TCP_V4_CH] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-ch",
.description = "IPv4 TCP, CH (LRU bypass), IPIP encap",
.vip_addr = IP4(10, 10, 1, 2), .dst_port = 80,
.src_addr = IP4(10, 10, 2, 2), .src_port = 54321,
.vip_flags = F_LRU_BYPASS, .vip_num = 1,
.lru_miss = LRU_MISS_ALL,
},
[S_TCP_V6_LRU_HIT] = {
S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP,
.name = "tcp-v6-lru-hit",
.description = "IPv6 TCP, LRU hit, IP6IP6 encap",
.vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 80,
.src_addr_v6 = IP6(0xfd000200, 0, 0, 1), .src_port = 12345,
.vip_num = 10,
.prepopulate_lru = true, .lru_miss = LRU_MISS_NONE,
},
[S_TCP_V6_CH] = {
S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP,
.name = "tcp-v6-ch",
.description = "IPv6 TCP, CH (LRU bypass), IP6IP6 encap",
.vip_addr_v6 = IP6(0xfd000100, 0, 0, 2), .dst_port = 80,
.src_addr_v6 = IP6(0xfd000200, 0, 0, 2), .src_port = 54321,
.vip_flags = F_LRU_BYPASS, .vip_num = 12,
.lru_miss = LRU_MISS_ALL,
},
[S_UDP_V4_LRU_HIT] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_UDP,
.name = "udp-v4-lru-hit",
.description = "IPv4 UDP, LRU hit, IPIP encap",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 443,
.src_addr = IP4(10, 10, 3, 1), .src_port = 11111,
.vip_num = 2,
.prepopulate_lru = true, .lru_miss = LRU_MISS_NONE,
},
[S_UDP_V6_LRU_HIT] = {
S_BASE_ENCAP_V6, .ip_proto = IPPROTO_UDP,
.name = "udp-v6-lru-hit",
.description = "IPv6 UDP, LRU hit, IP6IP6 encap",
.vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 443,
.src_addr_v6 = IP6(0xfd000200, 0, 0, 3), .src_port = 22222,
.vip_num = 14,
.prepopulate_lru = true, .lru_miss = LRU_MISS_NONE,
},
[S_TCP_V4V6_LRU_HIT] = {
S_BASE_ENCAP_V4V6, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4v6-lru-hit",
.description = "IPv4 TCP, LRU hit, IPv4-in-IPv6 encap",
.vip_addr = IP4(10, 10, 1, 4), .dst_port = 80,
.src_addr = IP4(10, 10, 2, 4), .src_port = 12347,
.vip_num = 13,
.prepopulate_lru = true, .lru_miss = LRU_MISS_NONE,
},
[S_TCP_V4_LRU_DIVERSE] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-lru-diverse",
.description = "IPv4 TCP, diverse flows, warm LRU",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 2, 1), .src_port = 12345,
.prepopulate_lru = true, .flow_mask = 0xFFF,
.lru_miss = LRU_MISS_NONE,
},
[S_TCP_V4_CH_DIVERSE] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-ch-diverse",
.description = "IPv4 TCP, diverse flows, CH (LRU bypass)",
.vip_addr = IP4(10, 10, 1, 2), .dst_port = 80,
.src_addr = IP4(10, 10, 2, 2), .src_port = 54321,
.vip_flags = F_LRU_BYPASS, .vip_num = 1,
.flow_mask = 0xFFF, .lru_miss = LRU_MISS_ALL,
},
[S_TCP_V6_LRU_DIVERSE] = {
S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP,
.name = "tcp-v6-lru-diverse",
.description = "IPv6 TCP, diverse flows, warm LRU",
.vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 80,
.src_addr_v6 = IP6(0xfd000200, 0, 0, 1), .src_port = 12345,
.vip_num = 10,
.prepopulate_lru = true, .flow_mask = 0xFFF,
.lru_miss = LRU_MISS_NONE,
},
[S_TCP_V6_CH_DIVERSE] = {
S_BASE_ENCAP_V6, .ip_proto = IPPROTO_TCP,
.name = "tcp-v6-ch-diverse",
.description = "IPv6 TCP, diverse flows, CH (LRU bypass)",
.vip_addr_v6 = IP6(0xfd000100, 0, 0, 2), .dst_port = 80,
.src_addr_v6 = IP6(0xfd000200, 0, 0, 2), .src_port = 54321,
.vip_flags = F_LRU_BYPASS, .vip_num = 12,
.flow_mask = 0xFFF, .lru_miss = LRU_MISS_ALL,
},
[S_UDP_V4_LRU_DIVERSE] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_UDP,
.name = "udp-v4-lru-diverse",
.description = "IPv4 UDP, diverse flows, warm LRU",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 443,
.src_addr = IP4(10, 10, 3, 1), .src_port = 11111,
.vip_num = 2,
.prepopulate_lru = true, .flow_mask = 0xFFF,
.lru_miss = LRU_MISS_NONE,
},
[S_TCP_V4_LRU_MISS] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-lru-miss",
.description = "IPv4 TCP, LRU miss (16M flow space), CH lookup",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 2, 1), .src_port = 12345,
.flow_mask = 0xFFFFFF, .cold_lru = true,
.lru_miss = LRU_MISS_FIRST,
},
[S_UDP_V4_LRU_MISS] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_UDP,
.name = "udp-v4-lru-miss",
.description = "IPv4 UDP, LRU miss (16M flow space), CH lookup",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 443,
.src_addr = IP4(10, 10, 3, 1), .src_port = 11111,
.vip_num = 2,
.flow_mask = 0xFFFFFF, .cold_lru = true,
.lru_miss = LRU_MISS_FIRST,
},
[S_TCP_V4_LRU_WARMUP] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-lru-warmup",
.description = "IPv4 TCP, 4K flows, ~50% LRU miss",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 2, 1), .src_port = 12345,
.flow_mask = 0xFFF, .cold_lru = true,
.fixed_batch_iters = 6500,
.lru_miss = LRU_MISS_FIRST,
},
[S_TCP_V4_SYN] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-syn",
.description = "IPv4 TCP SYN, skip LRU, CH + LRU insert",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 8, 2), .src_port = 60001,
.set_syn = true, .lru_miss = LRU_MISS_ALL,
},
[S_TCP_V4_RST_MISS] = {
S_BASE_ENCAP_V4, .ip_proto = IPPROTO_TCP,
.name = "tcp-v4-rst-miss",
.description = "IPv4 TCP RST, CH lookup, no LRU insert",
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 8, 1), .src_port = 60000,
.flow_mask = 0xFFFFFF, .cold_lru = true,
.set_rst = true, .lru_miss = LRU_MISS_ALL,
},
[S_PASS_V4_NO_VIP] = {
.name = "pass-v4-no-vip",
.description = "IPv4 TCP, unknown VIP, XDP_PASS",
.expected_retval = XDP_PASS,
.ip_proto = IPPROTO_TCP,
.vip_addr = IP4(10, 10, 9, 9), .dst_port = 80,
.src_addr = IP4(10, 10, 4, 1), .src_port = 33333,
},
[S_PASS_V6_NO_VIP] = {
.name = "pass-v6-no-vip",
.description = "IPv6 TCP, unknown VIP, XDP_PASS",
.expected_retval = XDP_PASS, .is_v6 = true,
.ip_proto = IPPROTO_TCP,
.vip_addr_v6 = IP6(0xfd009900, 0, 0, 1), .dst_port = 80,
.src_addr_v6 = IP6(0xfd000400, 0, 0, 1), .src_port = 33333,
},
[S_PASS_V4_ICMP] = {
.name = "pass-v4-icmp",
.description = "IPv4 ICMP, non-TCP/UDP protocol, XDP_PASS",
.expected_retval = XDP_PASS,
.ip_proto = IPPROTO_ICMP,
.vip_addr = IP4(10, 10, 1, 1),
.src_addr = IP4(10, 10, 6, 1),
},
[S_PASS_NON_IP] = {
.name = "pass-non-ip",
.description = "Non-IP (ARP), earliest XDP_PASS exit",
.expected_retval = XDP_PASS,
.eth_proto = ETH_P_ARP,
},
[S_DROP_V4_FRAG] = {
.name = "drop-v4-frag",
.description = "IPv4 fragmented, XDP_DROP",
.expected_retval = XDP_DROP, .ip_proto = IPPROTO_TCP,
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 5, 1), .src_port = 44444,
.set_frag = true,
},
[S_DROP_V4_OPTIONS] = {
.name = "drop-v4-options",
.description = "IPv4 with IP options (ihl>5), XDP_DROP",
.expected_retval = XDP_DROP, .ip_proto = IPPROTO_TCP,
.vip_addr = IP4(10, 10, 1, 1), .dst_port = 80,
.src_addr = IP4(10, 10, 7, 1), .src_port = 55555,
.set_ip_options = true,
},
[S_DROP_V6_FRAG] = {
.name = "drop-v6-frag",
.description = "IPv6 fragment extension header, XDP_DROP",
.expected_retval = XDP_DROP, .is_v6 = true,
.ip_proto = IPPROTO_TCP,
.vip_addr_v6 = IP6(0xfd000100, 0, 0, 1), .dst_port = 80,
.src_addr_v6 = IP6(0xfd000500, 0, 0, 1), .src_port = 44444,
.set_frag = true,
},
};
#define MAX_ENCAP_SIZE (MAX_PKT_SIZE + sizeof(struct ipv6hdr))
static __u8 pkt_buf[NUM_SCENARIOS][MAX_PKT_SIZE];
static __u32 pkt_len[NUM_SCENARIOS];
static __u8 expected_buf[NUM_SCENARIOS][MAX_ENCAP_SIZE];
static __u32 expected_len[NUM_SCENARIOS];
static int lru_inner_fds[BENCH_NR_CPUS];
static int nr_inner_maps;
static struct ctx {
struct xdp_lb_bench *skel;
struct bpf_bench_timing timing;
int prog_fd;
} ctx;
static struct {
int scenario;
bool machine_readable;
} args = {
.scenario = -1,
};
static __u16 ip_checksum(const void *hdr, int len)
{
const __u16 *p = hdr;
__u32 csum = 0;
int i;
for (i = 0; i < len / 2; i++)
csum += p[i];
while (csum >> 16)
csum = (csum & 0xffff) + (csum >> 16);
return ~csum;
}
static void htonl_v6(__be32 dst[4], const __u32 src[4])
{
int i;
for (i = 0; i < 4; i++)
dst[i] = htonl(src[i]);
}
static void build_flow_key(struct flow_key *fk, const struct test_scenario *sc)
{
memset(fk, 0, sizeof(*fk));
if (sc->is_v6) {
htonl_v6(fk->srcv6, sc->src_addr_v6);
htonl_v6(fk->dstv6, sc->vip_addr_v6);
} else {
fk->src = htonl(sc->src_addr);
fk->dst = htonl(sc->vip_addr);
}
fk->proto = sc->ip_proto;
fk->port16[0] = htons(sc->src_port);
fk->port16[1] = htons(sc->dst_port);
}
static void build_l4(const struct test_scenario *sc, __u8 *p, __u32 *off)
{
if (sc->ip_proto == IPPROTO_TCP) {
struct tcphdr tcp = {};
tcp.source = htons(sc->src_port);
tcp.dest = htons(sc->dst_port);
tcp.doff = 5;
tcp.syn = sc->set_syn ? 1 : 0;
tcp.rst = sc->set_rst ? 1 : 0;
tcp.window = htons(8192);
memcpy(p + *off, &tcp, sizeof(tcp));
*off += sizeof(tcp);
} else if (sc->ip_proto == IPPROTO_UDP) {
struct udphdr udp = {};
udp.source = htons(sc->src_port);
udp.dest = htons(sc->dst_port);
udp.len = htons(sizeof(udp) + 16);
memcpy(p + *off, &udp, sizeof(udp));
*off += sizeof(udp);
}
}
static void build_packet(int idx)
{
const struct test_scenario *sc = &scenarios[idx];
__u8 *p = pkt_buf[idx];
struct ethhdr eth = {};
__u16 proto;
__u32 off = 0;
memcpy(eth.h_dest, lb_mac, ETH_ALEN);
memcpy(eth.h_source, client_mac, ETH_ALEN);
if (sc->eth_proto)
proto = sc->eth_proto;
else if (sc->is_v6)
proto = ETH_P_IPV6;
else
proto = ETH_P_IP;
eth.h_proto = htons(proto);
memcpy(p, ð, sizeof(eth));
off += sizeof(eth);
if (proto != ETH_P_IP && proto != ETH_P_IPV6) {
memcpy(p + off, "bench___payload!", 16);
off += 16;
pkt_len[idx] = off;
return;
}
if (sc->is_v6) {
struct ipv6hdr ip6h = {};
__u32 ip6_off = off;
ip6h.version = 6;
ip6h.nexthdr = sc->set_frag ? 44 : sc->ip_proto;
ip6h.hop_limit = 64;
htonl_v6((__be32 *)&ip6h.saddr, sc->src_addr_v6);
htonl_v6((__be32 *)&ip6h.daddr, sc->vip_addr_v6);
off += sizeof(ip6h);
if (sc->set_frag) {
memset(p + off, 0, 8);
p[off] = sc->ip_proto;
off += 8;
}
build_l4(sc, p, &off);
memcpy(p + off, "bench___payload!", 16);
off += 16;
ip6h.payload_len = htons(off - ip6_off - sizeof(ip6h));
memcpy(p + ip6_off, &ip6h, sizeof(ip6h));
} else {
struct iphdr iph = {};
__u32 ip_off = off;
iph.version = 4;
iph.ihl = sc->set_ip_options ? 6 : 5;
iph.ttl = 64;
iph.protocol = sc->ip_proto;
iph.saddr = htonl(sc->src_addr);
iph.daddr = htonl(sc->vip_addr);
iph.frag_off = sc->set_frag ? htons(IP_MF) : 0;
off += sizeof(iph);
if (sc->set_ip_options) {
__u32 nop = htonl(0x01010101);
memcpy(p + off, &nop, sizeof(nop));
off += sizeof(nop);
}
build_l4(sc, p, &off);
memcpy(p + off, "bench___payload!", 16);
off += 16;
iph.tot_len = htons(off - ip_off);
iph.check = ip_checksum(&iph, sizeof(iph));
memcpy(p + ip_off, &iph, sizeof(iph));
}
pkt_len[idx] = off;
}
static void populate_vip(struct xdp_lb_bench *skel, const struct test_scenario *sc)
{
struct vip_definition key = {};
struct vip_meta val = {};
int err;
if (sc->is_v6)
htonl_v6(key.vipv6, sc->vip_addr_v6);
else
key.vip = htonl(sc->vip_addr);
key.port = htons(sc->dst_port);
key.proto = sc->ip_proto;
val.flags = sc->vip_flags;
val.vip_num = sc->vip_num;
err = bpf_map_update_elem(bpf_map__fd(skel->maps.vip_map), &key, &val, BPF_ANY);
if (err) {
fprintf(stderr, "vip_map [%s]: %s\n", sc->name, strerror(errno));
exit(1);
}
}
static void create_per_cpu_lru_maps(struct xdp_lb_bench *skel)
{
int outer_fd = bpf_map__fd(skel->maps.lru_mapping);
unsigned int nr_cpus = bpf_num_possible_cpus();
int i, inner_fd, err;
__u32 cpu;
if (nr_cpus > BENCH_NR_CPUS)
nr_cpus = BENCH_NR_CPUS;
for (i = 0; i < (int)nr_cpus; i++) {
LIBBPF_OPTS(bpf_map_create_opts, opts);
inner_fd = bpf_map_create(BPF_MAP_TYPE_LRU_HASH, "lru_inner",
sizeof(struct flow_key),
sizeof(struct real_pos_lru),
DEFAULT_LRU_SIZE, &opts);
if (inner_fd < 0) {
fprintf(stderr, "lru_inner[%d]: %s\n", i, strerror(errno));
exit(1);
}
cpu = i;
err = bpf_map_update_elem(outer_fd, &cpu, &inner_fd, BPF_ANY);
if (err) {
fprintf(stderr, "lru_mapping[%d]: %s\n", i, strerror(errno));
close(inner_fd);
exit(1);
}
lru_inner_fds[i] = inner_fd;
}
nr_inner_maps = nr_cpus;
}
static __u64 ktime_get_ns(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (__u64)ts.tv_sec * 1000000000ULL + ts.tv_nsec;
}
static void populate_lru(const struct test_scenario *sc, __u32 real_idx)
{
struct real_pos_lru lru = { .pos = real_idx };
struct flow_key fk;
int i, err;
if (sc->ip_proto == IPPROTO_UDP)
lru.atime = ktime_get_ns();
build_flow_key(&fk, sc);
for (i = 0; i < nr_inner_maps; i++) {
err = bpf_map_update_elem(lru_inner_fds[i], &fk, &lru, BPF_ANY);
if (err) {
fprintf(stderr, "lru_inner[%d] [%s]: %s\n", i, sc->name,
strerror(errno));
exit(1);
}
}
}
static void populate_maps(struct xdp_lb_bench *skel)
{
struct real_definition real_v4 = {};
struct real_definition real_v6 = {};
struct ctl_value cval = {};
__u32 key, real_idx = REAL_INDEX;
int ch_fd, err, i;
if (scenarios[args.scenario].expect_encap)
populate_vip(skel, &scenarios[args.scenario]);
ch_fd = bpf_map__fd(skel->maps.ch_rings);
for (i = 0; i < CH_RINGS_SIZE; i++) {
__u32 k = i;
err = bpf_map_update_elem(ch_fd, &k, &real_idx, BPF_ANY);
if (err) {
fprintf(stderr, "ch_rings[%d]: %s\n", i, strerror(errno));
exit(1);
}
}
memcpy(cval.mac, router_mac, ETH_ALEN);
key = 0;
err = bpf_map_update_elem(bpf_map__fd(skel->maps.ctl_array), &key, &cval, BPF_ANY);
if (err) {
fprintf(stderr, "ctl_array: %s\n", strerror(errno));
exit(1);
}
key = REAL_INDEX;
real_v4.dst = htonl(TNL_DST);
htonl_v6(real_v4.dstv6, tnl_dst_v6);
err = bpf_map_update_elem(bpf_map__fd(skel->maps.reals), &key, &real_v4, BPF_ANY);
if (err) {
fprintf(stderr, "reals[%d]: %s\n", REAL_INDEX, strerror(errno));
exit(1);
}
key = REAL_INDEX_V6;
htonl_v6(real_v6.dstv6, tnl_dst_v6);
real_v6.flags = F_IPV6;
err = bpf_map_update_elem(bpf_map__fd(skel->maps.reals), &key, &real_v6, BPF_ANY);
if (err) {
fprintf(stderr, "reals[%d]: %s\n", REAL_INDEX_V6, strerror(errno));
exit(1);
}
create_per_cpu_lru_maps(skel);
if (scenarios[args.scenario].prepopulate_lru) {
const struct test_scenario *sc = &scenarios[args.scenario];
__u32 ridx = sc->encap_v6_outer ? REAL_INDEX_V6 : REAL_INDEX;
populate_lru(sc, ridx);
}
if (scenarios[args.scenario].expect_encap) {
const struct test_scenario *sc = &scenarios[args.scenario];
struct vip_definition miss_vip = {};
if (sc->is_v6)
htonl_v6(miss_vip.vipv6, sc->vip_addr_v6);
else
miss_vip.vip = htonl(sc->vip_addr);
miss_vip.port = htons(sc->dst_port);
miss_vip.proto = sc->ip_proto;
key = 0;
err = bpf_map_update_elem(bpf_map__fd(skel->maps.vip_miss_stats),
&key, &miss_vip, BPF_ANY);
if (err) {
fprintf(stderr, "vip_miss_stats: %s\n", strerror(errno));
exit(1);
}
}
}
static void build_expected_packet(int idx)
{
const struct test_scenario *sc = &scenarios[idx];
__u8 *p = expected_buf[idx];
struct ethhdr eth = {};
const __u8 *in = pkt_buf[idx];
__u32 in_len = pkt_len[idx];
__u32 off = 0;
__u32 inner_len = in_len - sizeof(struct ethhdr);
if (sc->expected_retval == XDP_DROP) {
expected_len[idx] = 0;
return;
}
if (sc->expected_retval == XDP_PASS) {
memcpy(p, in, in_len);
expected_len[idx] = in_len;
return;
}
memcpy(eth.h_dest, router_mac, ETH_ALEN);
memcpy(eth.h_source, lb_mac, ETH_ALEN);
eth.h_proto = htons(sc->encap_v6_outer ? ETH_P_IPV6 : ETH_P_IP);
memcpy(p, ð, sizeof(eth));
off += sizeof(eth);
if (sc->encap_v6_outer) {
struct ipv6hdr ip6h = {};
__u8 nexthdr = sc->is_v6 ? IPPROTO_IPV6 : IPPROTO_IPIP;
ip6h.version = 6;
ip6h.nexthdr = nexthdr;
ip6h.payload_len = htons(inner_len);
ip6h.hop_limit = 64;
create_encap_ipv6_src(htons(sc->src_port),
sc->is_v6 ? htonl(sc->src_addr_v6[0])
: htonl(sc->src_addr),
(__be32 *)&ip6h.saddr);
htonl_v6((__be32 *)&ip6h.daddr, sc->tunnel_dst_v6);
memcpy(p + off, &ip6h, sizeof(ip6h));
off += sizeof(ip6h);
} else {
struct iphdr iph = {};
iph.version = 4;
iph.ihl = sizeof(iph) >> 2;
iph.protocol = IPPROTO_IPIP;
iph.tot_len = htons(inner_len + sizeof(iph));
iph.ttl = 64;
iph.saddr = create_encap_ipv4_src(htons(sc->src_port),
htonl(sc->src_addr));
iph.daddr = htonl(sc->tunnel_dst);
iph.check = ip_checksum(&iph, sizeof(iph));
memcpy(p + off, &iph, sizeof(iph));
off += sizeof(iph);
}
memcpy(p + off, in + sizeof(struct ethhdr), inner_len);
off += inner_len;
expected_len[idx] = off;
}
static void print_hex_diff(const char *name, const __u8 *got, __u32 got_len, const __u8 *exp,
__u32 exp_len)
{
__u32 max_len = got_len > exp_len ? got_len : exp_len;
__u32 i, ndiffs = 0;
fprintf(stderr, " [%s] got %u bytes, expected %u bytes\n",
name, got_len, exp_len);
for (i = 0; i < max_len && ndiffs < 8; i++) {
__u8 g = i < got_len ? got[i] : 0;
__u8 e = i < exp_len ? exp[i] : 0;
if (g != e || i >= got_len || i >= exp_len) {
fprintf(stderr, " offset 0x%03x: got 0x%02x expected 0x%02x\n",
i, g, e);
ndiffs++;
}
}
if (ndiffs >= 8 && i < max_len)
fprintf(stderr, " ... (more differences)\n");
}
static void read_stat(int stats_fd, __u32 key, __u64 *v1_out, __u64 *v2_out)
{
struct lb_stats values[BENCH_NR_CPUS];
unsigned int nr_cpus = bpf_num_possible_cpus();
__u64 v1 = 0, v2 = 0;
unsigned int i;
if (nr_cpus > BENCH_NR_CPUS)
nr_cpus = BENCH_NR_CPUS;
if (bpf_map_lookup_elem(stats_fd, &key, values) == 0) {
for (i = 0; i < nr_cpus; i++) {
v1 += values[i].v1;
v2 += values[i].v2;
}
}
*v1_out = v1;
*v2_out = v2;
}
static void reset_stats(int stats_fd)
{
struct lb_stats zeros[BENCH_NR_CPUS];
__u32 key;
memset(zeros, 0, sizeof(zeros));
for (key = 0; key < STATS_SIZE; key++)
bpf_map_update_elem(stats_fd, &key, zeros, BPF_ANY);
}
static bool validate_counters(int idx)
{
const struct test_scenario *sc = &scenarios[idx];
int stats_fd = bpf_map__fd(ctx.skel->maps.stats);
__u64 xdp_tx, xdp_pass, xdp_drop, lru_pkts, lru_misses, tcp_misses;
__u64 expected_misses;
__u64 dummy;
__u64 n = ctx.timing.batch_iters + 1;
bool pass = true;
read_stat(stats_fd, STATS_XDP_TX, &xdp_tx, &dummy);
read_stat(stats_fd, STATS_XDP_PASS, &xdp_pass, &dummy);
read_stat(stats_fd, STATS_XDP_DROP, &xdp_drop, &dummy);
read_stat(stats_fd, STATS_LRU, &lru_pkts, &lru_misses);
read_stat(stats_fd, STATS_LRU_MISS, &tcp_misses, &dummy);
if (sc->expected_retval == XDP_TX && xdp_tx != n) {
fprintf(stderr, " [%s] COUNTER FAIL: STATS_XDP_TX=%llu, expected %llu\n", sc->name,
(unsigned long long)xdp_tx, (unsigned long long)n);
pass = false;
}
if (sc->expected_retval == XDP_PASS && xdp_pass != n) {
fprintf(stderr, " [%s] COUNTER FAIL: STATS_XDP_PASS=%llu, expected %llu\n",
sc->name, (unsigned long long)xdp_pass, (unsigned long long)n);
pass = false;
}
if (sc->expected_retval == XDP_DROP && xdp_drop != n) {
fprintf(stderr, " [%s] COUNTER FAIL: STATS_XDP_DROP=%llu, expected %llu\n",
sc->name, (unsigned long long)xdp_drop, (unsigned long long)n);
pass = false;
}
if (!sc->expect_encap)
goto out;
if (lru_pkts != n) {
fprintf(stderr, " [%s] COUNTER FAIL: STATS_LRU.v1=%llu, expected %llu\n",
sc->name, (unsigned long long)lru_pkts, (unsigned long long)n);
pass = false;
}
switch (sc->lru_miss) {
case LRU_MISS_NONE:
expected_misses = 0;
break;
case LRU_MISS_ALL:
expected_misses = n;
break;
case LRU_MISS_FIRST:
expected_misses = 1;
break;
default:
if (sc->prepopulate_lru && !sc->set_syn)
expected_misses = 0;
else if (sc->set_syn || sc->set_rst ||
(sc->vip_flags & F_LRU_BYPASS))
expected_misses = n;
else if (sc->cold_lru)
expected_misses = 1;
else
expected_misses = n;
break;
}
if (lru_misses != expected_misses) {
fprintf(stderr, " [%s] COUNTER FAIL: LRU misses=%llu, expected %llu\n",
sc->name, (unsigned long long)lru_misses,
(unsigned long long)expected_misses);
pass = false;
}
if (sc->ip_proto == IPPROTO_TCP && lru_misses > 0) {
if (tcp_misses != lru_misses) {
fprintf(stderr, " [%s] COUNTER FAIL: TCP LRU misses=%llu, expected %llu\n",
sc->name, (unsigned long long)tcp_misses,
(unsigned long long)lru_misses);
pass = false;
}
}
out:
reset_stats(stats_fd);
return pass;
}
static const char *xdp_action_str(int action)
{
switch (action) {
case XDP_DROP: return "XDP_DROP";
case XDP_PASS: return "XDP_PASS";
case XDP_TX: return "XDP_TX";
default: return "UNKNOWN";
}
}
static bool validate_scenario(int idx)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
const struct test_scenario *sc = &scenarios[idx];
__u8 out[MAX_ENCAP_SIZE];
int err;
topts.data_in = pkt_buf[idx];
topts.data_size_in = pkt_len[idx];
topts.data_out = out;
topts.data_size_out = sizeof(out);
topts.repeat = 1;
err = bpf_prog_test_run_opts(ctx.prog_fd, &topts);
if (err) {
fprintf(stderr, " [%s] FAIL: test_run: %s\n", sc->name, strerror(errno));
return false;
}
if ((int)topts.retval != sc->expected_retval) {
fprintf(stderr, " [%s] FAIL: retval %s, expected %s\n", sc->name,
xdp_action_str(topts.retval), xdp_action_str(sc->expected_retval));
return false;
}
if (sc->expected_retval != XDP_DROP && !sc->cold_lru) {
if (topts.data_size_out != expected_len[idx] ||
memcmp(out, expected_buf[idx], expected_len[idx]) != 0) {
fprintf(stderr, " [%s] FAIL: output packet mismatch\n", sc->name);
print_hex_diff(sc->name, out, topts.data_size_out, expected_buf[idx],
expected_len[idx]);
return false;
}
}
if (!validate_counters(idx))
return false;
return true;
}
static int find_scenario(const char *name)
{
int i;
for (i = 0; i < NUM_SCENARIOS; i++) {
if (strcmp(scenarios[i].name, name) == 0)
return i;
}
return -1;
}
static void xdp_lb_validate(void)
{
if (env.consumer_cnt != 0) {
fprintf(stderr, "benchmark doesn't support consumers\n");
exit(1);
}
if (bpf_num_possible_cpus() > BENCH_NR_CPUS) {
fprintf(stderr, "too many CPUs (%d > %d), increase BENCH_NR_CPUS\n",
bpf_num_possible_cpus(), BENCH_NR_CPUS);
exit(1);
}
}
static void xdp_lb_run_once(void *unused __always_unused)
{
int idx = args.scenario;
LIBBPF_OPTS(bpf_test_run_opts, topts,
.data_in = pkt_buf[idx],
.data_size_in = pkt_len[idx],
.repeat = 1,
);
bpf_prog_test_run_opts(ctx.prog_fd, &topts);
}
static void xdp_lb_setup(void)
{
struct xdp_lb_bench *skel;
int err;
if (args.scenario < 0) {
fprintf(stderr, "--scenario is required. Use --list-scenarios to see options.\n");
exit(1);
}
setup_libbpf();
skel = xdp_lb_bench__open();
if (!skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
err = xdp_lb_bench__load(skel);
if (err) {
fprintf(stderr, "failed to load skeleton: %s\n", strerror(-err));
xdp_lb_bench__destroy(skel);
exit(1);
}
ctx.skel = skel;
ctx.prog_fd = bpf_program__fd(skel->progs.xdp_lb_bench);
build_packet(args.scenario);
build_expected_packet(args.scenario);
populate_maps(skel);
BENCH_TIMING_INIT(&ctx.timing, skel, 0);
ctx.timing.machine_readable = args.machine_readable;
if (scenarios[args.scenario].fixed_batch_iters) {
ctx.timing.batch_iters = scenarios[args.scenario].fixed_batch_iters;
skel->bss->batch_iters = ctx.timing.batch_iters;
} else {
bpf_bench_calibrate(&ctx.timing, xdp_lb_run_once, NULL);
}
env.duration_sec = 600;
if (scenarios[args.scenario].cold_lru) {
skel->bss->cold_lru = 1;
xdp_lb_run_once(NULL);
}
reset_stats(bpf_map__fd(skel->maps.stats));
if (!validate_scenario(args.scenario)) {
fprintf(stderr, "Validation FAILED - aborting benchmark\n");
exit(1);
}
if (scenarios[args.scenario].flow_mask)
skel->bss->flow_mask = scenarios[args.scenario].flow_mask;
}
static void *xdp_lb_producer(void *input)
{
while (true)
xdp_lb_run_once(NULL);
return NULL;
}
static void xdp_lb_measure(struct bench_res *res)
{
bpf_bench_timing_measure(&ctx.timing, res);
}
static void xdp_lb_report_final(struct bench_res res[], int res_cnt)
{
bpf_bench_timing_report(&ctx.timing, scenarios[args.scenario].name,
scenarios[args.scenario].description);
}
enum {
ARG_SCENARIO = 9001,
ARG_LIST_SCENARIOS = 9002,
ARG_MACHINE_READABLE = 9003,
};
static const struct argp_option opts[] = {
{ "scenario", ARG_SCENARIO, "NAME", 0,
"Scenario to benchmark (required)" },
{ "list-scenarios", ARG_LIST_SCENARIOS, NULL, 0,
"List available scenarios and exit" },
{ "machine-readable", ARG_MACHINE_READABLE, NULL, 0,
"Print only a machine-readable RESULT line" },
{},
};
static error_t parse_arg(int key, char *arg, struct argp_state *state)
{
int i;
switch (key) {
case ARG_SCENARIO:
args.scenario = find_scenario(arg);
if (args.scenario < 0) {
fprintf(stderr, "unknown scenario: '%s'\n", arg);
fprintf(stderr, "use --list-scenarios to see options\n");
argp_usage(state);
}
break;
case ARG_LIST_SCENARIOS:
printf("Available scenarios:\n");
for (i = 0; i < NUM_SCENARIOS; i++)
printf(" %-20s %s\n", scenarios[i].name, scenarios[i].description);
exit(0);
case ARG_MACHINE_READABLE:
args.machine_readable = true;
env.quiet = true;
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp bench_xdp_lb_argp = {
.options = opts,
.parser = parse_arg,
};
const struct bench bench_xdp_lb = {
.name = "xdp-lb",
.argp = &bench_xdp_lb_argp,
.validate = xdp_lb_validate,
.setup = xdp_lb_setup,
.producer_thread = xdp_lb_producer,
.measure = xdp_lb_measure,
.report_final = xdp_lb_report_final,
};
