net_hotdata
READ_ONCE(net_hotdata.max_backlog)) {
if (gro->rx_count >= READ_ONCE(net_hotdata.gro_normal_batch))
#define inet_ehash_secret net_hotdata.tcp_protocol.secret
#define udp_ehash_secret net_hotdata.udp_protocol.secret
#define inet6_ehash_secret net_hotdata.tcpv6_protocol.secret
#define tcp_ipv6_hash_secret net_hotdata.tcpv6_offload.secret
#define udp6_ehash_secret net_hotdata.udpv6_protocol.secret
#define udp_ipv6_hash_secret net_hotdata.udpv6_offload.secret
extern struct net_hotdata net_hotdata;
if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
u32 val = hash & ~net_hotdata.rps_cpu_mask;
tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
if (unlikely(!kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp,
net_hotdata.skb_defer_nodes =
if (!net_hotdata.skb_defer_nodes)
if (unlikely(defer_count > READ_ONCE(net_hotdata.qdisc_max_burst))) {
global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
next_cpu = ident & net_hotdata.rps_cpu_mask;
max_backlog = READ_ONCE(net_hotdata.max_backlog);
net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node;
usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
int budget = READ_ONCE(net_hotdata.netdev_budget);
struct list_head *head = &net_hotdata.offload_base;
list_for_each_entry(elem, &net_hotdata.offload_base, list) {
struct list_head *head = &net_hotdata.offload_base;
struct list_head *head = &net_hotdata.offload_base;
struct list_head *offload_head = &net_hotdata.offload_base;
struct list_head *offload_head = &net_hotdata.offload_base;
if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(net_hotdata.max_backlog)) {
list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
struct net_hotdata net_hotdata __cacheline_aligned = {
.offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
EXPORT_SYMBOL(net_hotdata);
kmem_cache_free(net_hotdata.skbuff_cache, skb);
kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
kmem_cache_free_bulk(net_hotdata.skbuff_cache,
n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
skb = kmem_cache_alloc(net_hotdata.skbuff_cache,
skb = kmem_cache_alloc(net_hotdata.skbuff_cache,
net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
skbuff_cache_size = kmem_cache_size(net_hotdata.skbuff_cache);
net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
return kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
kmem_cache_free(net_hotdata.skbuff_cache, skb);
cache = net_hotdata.skbuff_fclone_cache;
cache = net_hotdata.skbuff_cache;
sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + numa_node_id();
defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags);
o_tag_ptr = tag_ptr = net_hotdata.rps_sock_flow_table;
net_hotdata.rps_cpu_mask =
smp_store_release(&net_hotdata.rps_sock_flow_table, tag_ptr);
WRITE_ONCE(net_hotdata.dev_rx_weight, weight * dev_weight_rx_bias);
WRITE_ONCE(net_hotdata.dev_tx_weight, weight * dev_weight_tx_bias);
oval = !net_hotdata.sysctl_skb_defer_max;
nval = !net_hotdata.sysctl_skb_defer_max;
.data = &net_hotdata.sysctl_mem_pcpu_rsv,
.data = &net_hotdata.max_backlog,
.data = &net_hotdata.qdisc_max_burst,
.data = &net_hotdata.tstamp_prequeue,
.data = &net_hotdata.netdev_budget,
.data = &net_hotdata.sysctl_max_skb_frags,
.data = &net_hotdata.netdev_budget_usecs,
.data = &net_hotdata.gro_normal_batch,
.data = &net_hotdata.sysctl_skb_defer_max,
skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
net_hotdata.ip_packet_offload = (struct packet_offload) {
dev_add_offload(&net_hotdata.ip_packet_offload);
net_hotdata.udp_protocol = (struct net_protocol) {
if (inet_add_protocol(&net_hotdata.udp_protocol, IPPROTO_UDP) < 0)
net_hotdata.tcp_protocol = (struct net_protocol) {
if (inet_add_protocol(&net_hotdata.tcp_protocol, IPPROTO_TCP) < 0)
if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
net_hotdata.tcpv4_offload = (struct net_offload) {
return inet_add_offload(&net_hotdata.tcpv4_offload, IPPROTO_TCP);
net_hotdata.udpv4_offload = (struct net_offload) {
return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP);
if (likely(ops == &net_hotdata.tcpv6_offload))
if (ops == &net_hotdata.udpv6_offload)
net_hotdata.ipv6_packet_offload = (struct packet_offload) {
dev_add_offload(&net_hotdata.ipv6_packet_offload);
net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
net_hotdata.tcpv6_offload = (struct net_offload) {
return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
net_hotdata.udpv6_protocol = (struct inet6_protocol) {
ret = inet6_add_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
net_hotdata.udpv6_offload = (struct net_offload) {
return inet6_add_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
return inet6_del_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
if (!can_coalesce && i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
int quota = READ_ONCE(net_hotdata.dev_tx_weight);
READ_ONCE(net_hotdata.max_backlog)) {
if (skb_queue_len(&trans->queue) >= READ_ONCE(net_hotdata.max_backlog))