// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *      Generic address resolution entity
 *
 *      Authors:
 *      Pedro Roque             <roque@di.fc.ul.pt>
 *      Alexey Kuznetsov        <kuznet@ms2.inr.ac.ru>
 *
 *      Fixes:
 *      Vitaly E. Lavrov        releasing NULL neighbor in neigh_add.
 *      Harald Welte            Add neighbour cache statistics like rtstat
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/socket.h>
#include <linux/netdevice.h>
#include <linux/proc_fs.h>
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif
#include <linux/times.h>
#include <net/net_namespace.h>
#include <net/neighbour.h>
#include <net/arp.h>
#include <net/dst.h>
#include <net/ip.h>
#include <net/sock.h>
#include <net/netevent.h>
#include <net/netlink.h>
#include <linux/rtnetlink.h>
#include <linux/random.h>
#include <linux/string.h>
#include <linux/log2.h>
#include <linux/inetdevice.h>
#include <net/addrconf.h>

#include <trace/events/neigh.h>

#define NEIGH_DEBUG 1
#define neigh_dbg(level, fmt, ...)              \
do {                                            \
        if (level <= NEIGH_DEBUG)               \
                pr_debug(fmt, ##__VA_ARGS__);   \
} while (0)

#define PNEIGH_HASHMASK         0xF

static void neigh_timer_handler(struct timer_list *t);
static void neigh_notify(struct neighbour *n, int type, int flags, u32 pid);
static void __neigh_notify(struct neighbour *n, int type, int flags, u32 pid);
static void pneigh_ifdown(struct neigh_table *tbl, struct net_device *dev,
                          bool skip_perm);

#ifdef CONFIG_PROC_FS
static const struct seq_operations neigh_stat_seq_ops;
#endif

static struct hlist_head *neigh_get_dev_table(struct net_device *dev, int family)
{
        int i;

        switch (family) {
        default:
                DEBUG_NET_WARN_ON_ONCE(1);
                fallthrough; /* to avoid panic by null-ptr-deref */
        case AF_INET:
                i = NEIGH_ARP_TABLE;
                break;
        case AF_INET6:
                i = NEIGH_ND_TABLE;
                break;
        }

        return &dev->neighbours[i];
}

/*
   Neighbour hash table buckets are protected with tbl->lock.

   - All the scans/updates to hash buckets MUST be made under this lock.
   - NOTHING clever should be made under this lock: no callbacks
     to protocol backends, no attempts to send something to network.
     It will result in deadlocks, if backend/driver wants to use neighbour
     cache.
   - If the entry requires some non-trivial actions, increase
     its reference count and release table lock.

   Neighbour entries are protected:
   - with reference count.
   - with rwlock neigh->lock

   Reference count prevents destruction.

   neigh->lock mainly serializes ll address data and its validity state.
   However, the same lock is used to protect another entry fields:
    - timer
    - resolution queue

   Again, nothing clever shall be made under neigh->lock,
   the most complicated procedure, which we allow is dev->hard_header.
   It is supposed, that dev->hard_header is simplistic and does
   not make callbacks to neighbour tables.
 */

static int neigh_blackhole(struct neighbour *neigh, struct sk_buff *skb)
{
        kfree_skb(skb);
        return -ENETDOWN;
}

static void neigh_cleanup_and_release(struct neighbour *neigh)
{
        trace_neigh_cleanup_and_release(neigh, 0);
        neigh_notify(neigh, RTM_DELNEIGH, 0, 0);
        call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);
        neigh_release(neigh);
}

/*
 * It is random distribution in the interval (1/2)*base...(3/2)*base.
 * It corresponds to default IPv6 settings and is not overridable,
 * because it is really reasonable choice.
 */

unsigned long neigh_rand_reach_time(unsigned long base)
{
        return base ? get_random_u32_below(base) + (base >> 1) : 0;
}
EXPORT_SYMBOL(neigh_rand_reach_time);

static void neigh_mark_dead(struct neighbour *n)
{
        n->dead = 1;
        if (!list_empty(&n->gc_list)) {
                list_del_init(&n->gc_list);
                atomic_dec(&n->tbl->gc_entries);
        }
        if (!list_empty(&n->managed_list))
                list_del_init(&n->managed_list);
}

static void neigh_update_gc_list(struct neighbour *n)
{
        bool on_gc_list, exempt_from_gc;

        spin_lock_bh(&n->tbl->lock);
        write_lock(&n->lock);
        if (n->dead)
                goto out;

        /* remove from the gc list if new state is permanent or if neighbor is
         * externally learned / validated; otherwise entry should be on the gc
         * list
         */
        exempt_from_gc = n->nud_state & NUD_PERMANENT ||
                         n->flags & (NTF_EXT_LEARNED | NTF_EXT_VALIDATED);
        on_gc_list = !list_empty(&n->gc_list);

        if (exempt_from_gc && on_gc_list) {
                list_del_init(&n->gc_list);
                atomic_dec(&n->tbl->gc_entries);
        } else if (!exempt_from_gc && !on_gc_list) {
                /* add entries to the tail; cleaning removes from the front */
                list_add_tail(&n->gc_list, &n->tbl->gc_list);
                atomic_inc(&n->tbl->gc_entries);
        }
out:
        write_unlock(&n->lock);
        spin_unlock_bh(&n->tbl->lock);
}

static void neigh_update_managed_list(struct neighbour *n)
{
        bool on_managed_list, add_to_managed;

        spin_lock_bh(&n->tbl->lock);
        write_lock(&n->lock);
        if (n->dead)
                goto out;

        add_to_managed = n->flags & NTF_MANAGED;
        on_managed_list = !list_empty(&n->managed_list);

        if (!add_to_managed && on_managed_list)
                list_del_init(&n->managed_list);
        else if (add_to_managed && !on_managed_list)
                list_add_tail(&n->managed_list, &n->tbl->managed_list);
out:
        write_unlock(&n->lock);
        spin_unlock_bh(&n->tbl->lock);
}

static void neigh_update_flags(struct neighbour *neigh, u32 flags, int *notify,
                               bool *gc_update, bool *managed_update)
{
        u32 ndm_flags, old_flags = neigh->flags;

        if (!(flags & NEIGH_UPDATE_F_ADMIN))
                return;

        ndm_flags  = (flags & NEIGH_UPDATE_F_EXT_LEARNED) ? NTF_EXT_LEARNED : 0;
        ndm_flags |= (flags & NEIGH_UPDATE_F_MANAGED) ? NTF_MANAGED : 0;
        ndm_flags |= (flags & NEIGH_UPDATE_F_EXT_VALIDATED) ? NTF_EXT_VALIDATED : 0;

        if ((old_flags ^ ndm_flags) & NTF_EXT_LEARNED) {
                if (ndm_flags & NTF_EXT_LEARNED)
                        neigh->flags |= NTF_EXT_LEARNED;
                else
                        neigh->flags &= ~NTF_EXT_LEARNED;
                *notify = 1;
                *gc_update = true;
        }
        if ((old_flags ^ ndm_flags) & NTF_MANAGED) {
                if (ndm_flags & NTF_MANAGED)
                        neigh->flags |= NTF_MANAGED;
                else
                        neigh->flags &= ~NTF_MANAGED;
                *notify = 1;
                *managed_update = true;
        }
        if ((old_flags ^ ndm_flags) & NTF_EXT_VALIDATED) {
                if (ndm_flags & NTF_EXT_VALIDATED)
                        neigh->flags |= NTF_EXT_VALIDATED;
                else
                        neigh->flags &= ~NTF_EXT_VALIDATED;
                *notify = 1;
                *gc_update = true;
        }
}

bool neigh_remove_one(struct neighbour *n)
{
        bool retval = false;

        write_lock(&n->lock);
        if (refcount_read(&n->refcnt) == 1) {
                hlist_del_rcu(&n->hash);
                hlist_del_rcu(&n->dev_list);
                neigh_mark_dead(n);
                retval = true;
        }
        write_unlock(&n->lock);
        if (retval)
                neigh_cleanup_and_release(n);
        return retval;
}

static int neigh_forced_gc(struct neigh_table *tbl)
{
        int max_clean = atomic_read(&tbl->gc_entries) -
                        READ_ONCE(tbl->gc_thresh2);
        u64 tmax = ktime_get_ns() + NSEC_PER_MSEC;
        unsigned long tref = jiffies - 5 * HZ;
        struct neighbour *n, *tmp;
        int shrunk = 0;
        int loop = 0;

        NEIGH_CACHE_STAT_INC(tbl, forced_gc_runs);

        spin_lock_bh(&tbl->lock);

        list_for_each_entry_safe(n, tmp, &tbl->gc_list, gc_list) {
                if (refcount_read(&n->refcnt) == 1) {
                        bool remove = false;

                        write_lock(&n->lock);
                        if ((n->nud_state == NUD_FAILED) ||
                            (n->nud_state == NUD_NOARP) ||
                            (tbl->is_multicast &&
                             tbl->is_multicast(n->primary_key)) ||
                            !time_in_range(n->updated, tref, jiffies))
                                remove = true;
                        write_unlock(&n->lock);

                        if (remove && neigh_remove_one(n))
                                shrunk++;
                        if (shrunk >= max_clean)
                                break;
                        if (++loop == 16) {
                                if (ktime_get_ns() > tmax)
                                        goto unlock;
                                loop = 0;
                        }
                }
        }

        WRITE_ONCE(tbl->last_flush, jiffies);
unlock:
        spin_unlock_bh(&tbl->lock);

        return shrunk;
}

static void neigh_add_timer(struct neighbour *n, unsigned long when)
{
        /* Use safe distance from the jiffies - LONG_MAX point while timer
         * is running in DELAY/PROBE state but still show to user space
         * large times in the past.
         */
        unsigned long mint = jiffies - (LONG_MAX - 86400 * HZ);

        neigh_hold(n);
        if (!time_in_range(n->confirmed, mint, jiffies))
                n->confirmed = mint;
        if (time_before(n->used, n->confirmed))
                n->used = n->confirmed;
        if (unlikely(mod_timer(&n->timer, when))) {
                printk("NEIGH: BUG, double timer add, state is %x\n",
                       n->nud_state);
                dump_stack();
        }
}

static int neigh_del_timer(struct neighbour *n)
{
        if ((n->nud_state & NUD_IN_TIMER) &&
            timer_delete(&n->timer)) {
                neigh_release(n);
                return 1;
        }
        return 0;
}

static struct neigh_parms *neigh_get_dev_parms_rcu(struct net_device *dev,
                                                   int family)
{
        switch (family) {
        case AF_INET:
                return __in_dev_arp_parms_get_rcu(dev);
        case AF_INET6:
                return __in6_dev_nd_parms_get_rcu(dev);
        }
        return NULL;
}

static void neigh_parms_qlen_dec(struct net_device *dev, int family)
{
        struct neigh_parms *p;

        rcu_read_lock();
        p = neigh_get_dev_parms_rcu(dev, family);
        if (p)
                p->qlen--;
        rcu_read_unlock();
}

static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net,
                               int family)
{
        struct sk_buff_head tmp;
        unsigned long flags;
        struct sk_buff *skb;

        skb_queue_head_init(&tmp);
        spin_lock_irqsave(&list->lock, flags);
        skb = skb_peek(list);
        while (skb != NULL) {
                struct sk_buff *skb_next = skb_peek_next(skb, list);
                struct net_device *dev = skb->dev;

                if (net == NULL || net_eq(dev_net(dev), net)) {
                        neigh_parms_qlen_dec(dev, family);
                        __skb_unlink(skb, list);
                        __skb_queue_tail(&tmp, skb);
                }
                skb = skb_next;
        }
        spin_unlock_irqrestore(&list->lock, flags);

        while ((skb = __skb_dequeue(&tmp))) {
                dev_put(skb->dev);
                kfree_skb(skb);
        }
}

static void neigh_flush_one(struct neighbour *n)
{
        hlist_del_rcu(&n->hash);
        hlist_del_rcu(&n->dev_list);

        write_lock(&n->lock);

        neigh_del_timer(n);
        neigh_mark_dead(n);

        if (refcount_read(&n->refcnt) != 1) {
                /* The most unpleasant situation.
                 * We must destroy neighbour entry,
                 * but someone still uses it.
                 *
                 * The destroy will be delayed until
                 * the last user releases us, but
                 * we must kill timers etc. and move
                 * it to safe state.
                 */
                __skb_queue_purge(&n->arp_queue);
                n->arp_queue_len_bytes = 0;
                WRITE_ONCE(n->output, neigh_blackhole);

                if (n->nud_state & NUD_VALID)
                        n->nud_state = NUD_NOARP;
                else
                        n->nud_state = NUD_NONE;

                neigh_dbg(2, "neigh %p is stray\n", n);
        }

        write_unlock(&n->lock);

        neigh_cleanup_and_release(n);
}

static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev,
                            bool skip_perm)
{
        struct hlist_head *dev_head;
        struct hlist_node *tmp;
        struct neighbour *n;

        dev_head = neigh_get_dev_table(dev, tbl->family);

        hlist_for_each_entry_safe(n, tmp, dev_head, dev_list) {
                if (skip_perm &&
                    (n->nud_state & NUD_PERMANENT ||
                     n->flags & NTF_EXT_VALIDATED))
                        continue;

                neigh_flush_one(n);
        }
}

static void neigh_flush_table(struct neigh_table *tbl)
{
        struct neigh_hash_table *nht;
        int i;

        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));

        for (i = 0; i < (1 << nht->hash_shift); i++) {
                struct hlist_node *tmp;
                struct neighbour *n;

                neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[i])
                        neigh_flush_one(n);
        }
}

void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev)
{
        spin_lock_bh(&tbl->lock);
        neigh_flush_dev(tbl, dev, false);
        spin_unlock_bh(&tbl->lock);
}

static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev,
                          bool skip_perm)
{
        spin_lock_bh(&tbl->lock);
        if (likely(dev)) {
                neigh_flush_dev(tbl, dev, skip_perm);
        } else {
                DEBUG_NET_WARN_ON_ONCE(skip_perm);
                neigh_flush_table(tbl);
        }
        spin_unlock_bh(&tbl->lock);

        pneigh_ifdown(tbl, dev, skip_perm);
        pneigh_queue_purge(&tbl->proxy_queue, dev ? dev_net(dev) : NULL,
                           tbl->family);
        if (skb_queue_empty_lockless(&tbl->proxy_queue))
                timer_delete_sync(&tbl->proxy_timer);
        return 0;
}

int neigh_carrier_down(struct neigh_table *tbl, struct net_device *dev)
{
        __neigh_ifdown(tbl, dev, true);
        return 0;
}

int neigh_ifdown(struct neigh_table *tbl, struct net_device *dev)
{
        __neigh_ifdown(tbl, dev, false);
        return 0;
}

static struct neighbour *neigh_alloc(struct neigh_table *tbl,
                                     struct net_device *dev,
                                     u32 flags, bool exempt_from_gc)
{
        struct neighbour *n = NULL;
        unsigned long now = jiffies;
        int entries, gc_thresh3;

        if (exempt_from_gc)
                goto do_alloc;

        entries = atomic_inc_return(&tbl->gc_entries) - 1;
        gc_thresh3 = READ_ONCE(tbl->gc_thresh3);
        if (entries >= gc_thresh3 ||
            (entries >= READ_ONCE(tbl->gc_thresh2) &&
             time_after(now, READ_ONCE(tbl->last_flush) + 5 * HZ))) {
                if (!neigh_forced_gc(tbl) && entries >= gc_thresh3) {
                        net_info_ratelimited("%s: neighbor table overflow!\n",
                                             tbl->id);
                        NEIGH_CACHE_STAT_INC(tbl, table_fulls);
                        goto out_entries;
                }
        }

do_alloc:
        n = kzalloc(tbl->entry_size + dev->neigh_priv_len, GFP_ATOMIC);
        if (!n)
                goto out_entries;

        __skb_queue_head_init(&n->arp_queue);
        rwlock_init(&n->lock);
        seqlock_init(&n->ha_lock);
        n->updated        = n->used = now;
        n->nud_state      = NUD_NONE;
        n->output         = neigh_blackhole;
        n->flags          = flags;
        seqlock_init(&n->hh.hh_lock);
        n->parms          = neigh_parms_clone(&tbl->parms);
        timer_setup(&n->timer, neigh_timer_handler, 0);

        NEIGH_CACHE_STAT_INC(tbl, allocs);
        n->tbl            = tbl;
        refcount_set(&n->refcnt, 1);
        n->dead           = 1;
        INIT_LIST_HEAD(&n->gc_list);
        INIT_LIST_HEAD(&n->managed_list);

        atomic_inc(&tbl->entries);
out:
        return n;

out_entries:
        if (!exempt_from_gc)
                atomic_dec(&tbl->gc_entries);
        goto out;
}

static void neigh_get_hash_rnd(u32 *x)
{
        *x = get_random_u32() | 1;
}

static struct neigh_hash_table *neigh_hash_alloc(unsigned int shift)
{
        size_t size = (1 << shift) * sizeof(struct hlist_head);
        struct hlist_head *hash_heads;
        struct neigh_hash_table *ret;
        int i;

        ret = kmalloc_obj(*ret, GFP_ATOMIC);
        if (!ret)
                return NULL;

        hash_heads = kzalloc(size, GFP_ATOMIC);
        if (!hash_heads) {
                kfree(ret);
                return NULL;
        }
        ret->hash_heads = hash_heads;
        ret->hash_shift = shift;
        for (i = 0; i < NEIGH_NUM_HASH_RND; i++)
                neigh_get_hash_rnd(&ret->hash_rnd[i]);
        return ret;
}

static void neigh_hash_free_rcu(struct rcu_head *head)
{
        struct neigh_hash_table *nht = container_of(head,
                                                    struct neigh_hash_table,
                                                    rcu);

        kfree(nht->hash_heads);
        kfree(nht);
}

static struct neigh_hash_table *neigh_hash_grow(struct neigh_table *tbl,
                                                unsigned long new_shift)
{
        unsigned int i, hash;
        struct neigh_hash_table *new_nht, *old_nht;

        NEIGH_CACHE_STAT_INC(tbl, hash_grows);

        old_nht = rcu_dereference_protected(tbl->nht,
                                            lockdep_is_held(&tbl->lock));
        new_nht = neigh_hash_alloc(new_shift);
        if (!new_nht)
                return old_nht;

        for (i = 0; i < (1 << old_nht->hash_shift); i++) {
                struct hlist_node *tmp;
                struct neighbour *n;

                neigh_for_each_in_bucket_safe(n, tmp, &old_nht->hash_heads[i]) {
                        hash = tbl->hash(n->primary_key, n->dev,
                                         new_nht->hash_rnd);

                        hash >>= (32 - new_nht->hash_shift);

                        hlist_del_rcu(&n->hash);
                        hlist_add_head_rcu(&n->hash, &new_nht->hash_heads[hash]);
                }
        }

        rcu_assign_pointer(tbl->nht, new_nht);
        call_rcu(&old_nht->rcu, neigh_hash_free_rcu);
        return new_nht;
}

struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,
                               struct net_device *dev)
{
        struct neighbour *n;

        NEIGH_CACHE_STAT_INC(tbl, lookups);

        rcu_read_lock();
        n = __neigh_lookup_noref(tbl, pkey, dev);
        if (n) {
                if (!refcount_inc_not_zero(&n->refcnt))
                        n = NULL;
                NEIGH_CACHE_STAT_INC(tbl, hits);
        }

        rcu_read_unlock();
        return n;
}
EXPORT_SYMBOL(neigh_lookup);

static struct neighbour *
___neigh_create(struct neigh_table *tbl, const void *pkey,
                struct net_device *dev, u32 flags,
                bool exempt_from_gc, bool want_ref)
{
        u32 hash_val, key_len = tbl->key_len;
        struct neighbour *n1, *rc, *n;
        struct neigh_hash_table *nht;
        int error;

        n = neigh_alloc(tbl, dev, flags, exempt_from_gc);
        trace_neigh_create(tbl, dev, pkey, n, exempt_from_gc);
        if (!n) {
                rc = ERR_PTR(-ENOBUFS);
                goto out;
        }

        memcpy(n->primary_key, pkey, key_len);
        n->dev = dev;
        netdev_hold(dev, &n->dev_tracker, GFP_ATOMIC);

        /* Protocol specific setup. */
        if (tbl->constructor && (error = tbl->constructor(n)) < 0) {
                rc = ERR_PTR(error);
                goto out_neigh_release;
        }

        if (dev->netdev_ops->ndo_neigh_construct) {
                error = dev->netdev_ops->ndo_neigh_construct(dev, n);
                if (error < 0) {
                        rc = ERR_PTR(error);
                        goto out_neigh_release;
                }
        }

        /* Device specific setup. */
        if (n->parms->neigh_setup &&
            (error = n->parms->neigh_setup(n)) < 0) {
                rc = ERR_PTR(error);
                goto out_neigh_release;
        }

        n->confirmed = jiffies - (NEIGH_VAR(n->parms, BASE_REACHABLE_TIME) << 1);

        spin_lock_bh(&tbl->lock);
        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));

        if (atomic_read(&tbl->entries) > (1 << nht->hash_shift))
                nht = neigh_hash_grow(tbl, nht->hash_shift + 1);

        hash_val = tbl->hash(n->primary_key, dev, nht->hash_rnd) >> (32 - nht->hash_shift);

        if (n->parms->dead) {
                rc = ERR_PTR(-EINVAL);
                goto out_tbl_unlock;
        }

        neigh_for_each_in_bucket(n1, &nht->hash_heads[hash_val]) {
                if (dev == n1->dev && !memcmp(n1->primary_key, n->primary_key, key_len)) {
                        if (want_ref)
                                neigh_hold(n1);
                        rc = n1;
                        goto out_tbl_unlock;
                }
        }

        n->dead = 0;
        if (!exempt_from_gc)
                list_add_tail(&n->gc_list, &n->tbl->gc_list);
        if (n->flags & NTF_MANAGED)
                list_add_tail(&n->managed_list, &n->tbl->managed_list);
        if (want_ref)
                neigh_hold(n);
        hlist_add_head_rcu(&n->hash, &nht->hash_heads[hash_val]);

        hlist_add_head_rcu(&n->dev_list,
                           neigh_get_dev_table(dev, tbl->family));

        spin_unlock_bh(&tbl->lock);
        neigh_dbg(2, "neigh %p is created\n", n);
        rc = n;
out:
        return rc;
out_tbl_unlock:
        spin_unlock_bh(&tbl->lock);
out_neigh_release:
        if (!exempt_from_gc)
                atomic_dec(&tbl->gc_entries);
        neigh_release(n);
        goto out;
}

struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,
                                 struct net_device *dev, bool want_ref)
{
        bool exempt_from_gc = !!(dev->flags & IFF_LOOPBACK);

        return ___neigh_create(tbl, pkey, dev, 0, exempt_from_gc, want_ref);
}
EXPORT_SYMBOL(__neigh_create);

static u32 pneigh_hash(const void *pkey, unsigned int key_len)
{
        u32 hash_val = *(u32 *)(pkey + key_len - 4);
        hash_val ^= (hash_val >> 16);
        hash_val ^= hash_val >> 8;
        hash_val ^= hash_val >> 4;
        hash_val &= PNEIGH_HASHMASK;
        return hash_val;
}

struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl,
                                   struct net *net, const void *pkey,
                                   struct net_device *dev)
{
        struct pneigh_entry *n;
        unsigned int key_len;
        u32 hash_val;

        key_len = tbl->key_len;
        hash_val = pneigh_hash(pkey, key_len);
        n = rcu_dereference_check(tbl->phash_buckets[hash_val],
                                  lockdep_is_held(&tbl->phash_lock));

        while (n) {
                if (!memcmp(n->key, pkey, key_len) &&
                    net_eq(pneigh_net(n), net) &&
                    (n->dev == dev || !n->dev))
                        return n;

                n = rcu_dereference_check(n->next, lockdep_is_held(&tbl->phash_lock));
        }

        return NULL;
}

int pneigh_create(struct neigh_table *tbl, struct net *net,
                  const void *pkey, struct net_device *dev,
                  u32 flags, u8 protocol, bool permanent)
{
        struct pneigh_entry *n;
        unsigned int key_len;
        u32 hash_val;
        int err = 0;

        mutex_lock(&tbl->phash_lock);

        n = pneigh_lookup(tbl, net, pkey, dev);
        if (n)
                goto update;

        key_len = tbl->key_len;
        n = kzalloc(sizeof(*n) + key_len, GFP_KERNEL);
        if (!n) {
                err = -ENOBUFS;
                goto out;
        }

        write_pnet(&n->net, net);
        memcpy(n->key, pkey, key_len);
        n->dev = dev;
        netdev_hold(dev, &n->dev_tracker, GFP_KERNEL);

        if (tbl->pconstructor && tbl->pconstructor(n)) {
                netdev_put(dev, &n->dev_tracker);
                kfree(n);
                err = -ENOBUFS;
                goto out;
        }

        hash_val = pneigh_hash(pkey, key_len);
        n->next = tbl->phash_buckets[hash_val];
        rcu_assign_pointer(tbl->phash_buckets[hash_val], n);
update:
        WRITE_ONCE(n->flags, flags);
        n->permanent = permanent;
        if (protocol)
                WRITE_ONCE(n->protocol, protocol);
out:
        mutex_unlock(&tbl->phash_lock);
        return err;
}

static void pneigh_destroy(struct rcu_head *rcu)
{
        struct pneigh_entry *n = container_of(rcu, struct pneigh_entry, rcu);

        netdev_put(n->dev, &n->dev_tracker);
        kfree(n);
}

int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey,
                  struct net_device *dev)
{
        struct pneigh_entry *n, __rcu **np;
        unsigned int key_len;
        u32 hash_val;

        key_len = tbl->key_len;
        hash_val = pneigh_hash(pkey, key_len);

        mutex_lock(&tbl->phash_lock);

        for (np = &tbl->phash_buckets[hash_val];
             (n = rcu_dereference_protected(*np, 1)) != NULL;
             np = &n->next) {
                if (!memcmp(n->key, pkey, key_len) && n->dev == dev &&
                    net_eq(pneigh_net(n), net)) {
                        rcu_assign_pointer(*np, n->next);

                        mutex_unlock(&tbl->phash_lock);

                        if (tbl->pdestructor)
                                tbl->pdestructor(n);

                        call_rcu(&n->rcu, pneigh_destroy);
                        return 0;
                }
        }

        mutex_unlock(&tbl->phash_lock);
        return -ENOENT;
}

static void pneigh_ifdown(struct neigh_table *tbl, struct net_device *dev,
                          bool skip_perm)
{
        struct pneigh_entry *n, __rcu **np;
        LIST_HEAD(head);
        u32 h;

        mutex_lock(&tbl->phash_lock);

        for (h = 0; h <= PNEIGH_HASHMASK; h++) {
                np = &tbl->phash_buckets[h];
                while ((n = rcu_dereference_protected(*np, 1)) != NULL) {
                        if (skip_perm && n->permanent)
                                goto skip;
                        if (!dev || n->dev == dev) {
                                rcu_assign_pointer(*np, n->next);
                                list_add(&n->free_node, &head);
                                continue;
                        }
skip:
                        np = &n->next;
                }
        }

        mutex_unlock(&tbl->phash_lock);

        while (!list_empty(&head)) {
                n = list_first_entry(&head, typeof(*n), free_node);
                list_del(&n->free_node);

                if (tbl->pdestructor)
                        tbl->pdestructor(n);

                call_rcu(&n->rcu, pneigh_destroy);
        }
}

static inline void neigh_parms_put(struct neigh_parms *parms)
{
        if (refcount_dec_and_test(&parms->refcnt))
                kfree(parms);
}

/*
 *      neighbour must already be out of the table;
 *
 */
void neigh_destroy(struct neighbour *neigh)
{
        struct net_device *dev = neigh->dev;

        NEIGH_CACHE_STAT_INC(neigh->tbl, destroys);

        if (!neigh->dead) {
                pr_warn("Destroying alive neighbour %p\n", neigh);
                dump_stack();
                return;
        }

        if (neigh_del_timer(neigh))
                pr_warn("Impossible event\n");

        write_lock_bh(&neigh->lock);
        __skb_queue_purge(&neigh->arp_queue);
        write_unlock_bh(&neigh->lock);
        neigh->arp_queue_len_bytes = 0;

        if (dev->netdev_ops->ndo_neigh_destroy)
                dev->netdev_ops->ndo_neigh_destroy(dev, neigh);

        netdev_put(dev, &neigh->dev_tracker);
        neigh_parms_put(neigh->parms);

        neigh_dbg(2, "neigh %p is destroyed\n", neigh);

        atomic_dec(&neigh->tbl->entries);
        kfree_rcu(neigh, rcu);
}
EXPORT_SYMBOL(neigh_destroy);

/* Neighbour state is suspicious;
   disable fast path.

   Called with write_locked neigh.
 */
static void neigh_suspect(struct neighbour *neigh)
{
        neigh_dbg(2, "neigh %p is suspected\n", neigh);

        WRITE_ONCE(neigh->output, neigh->ops->output);
}

/* Neighbour state is OK;
   enable fast path.

   Called with write_locked neigh.
 */
static void neigh_connect(struct neighbour *neigh)
{
        neigh_dbg(2, "neigh %p is connected\n", neigh);

        WRITE_ONCE(neigh->output, neigh->ops->connected_output);
}

static void neigh_periodic_work(struct work_struct *work)
{
        struct neigh_table *tbl = container_of(work, struct neigh_table, gc_work.work);
        struct neigh_hash_table *nht;
        struct hlist_node *tmp;
        struct neighbour *n;
        unsigned int i;

        NEIGH_CACHE_STAT_INC(tbl, periodic_gc_runs);

        spin_lock_bh(&tbl->lock);
        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));

        /*
         *      periodically recompute ReachableTime from random function
         */

        if (time_after(jiffies, tbl->last_rand + 300 * HZ)) {
                struct neigh_parms *p;

                WRITE_ONCE(tbl->last_rand, jiffies);
                list_for_each_entry(p, &tbl->parms_list, list)
                        neigh_set_reach_time(p);
        }

        if (atomic_read(&tbl->entries) < READ_ONCE(tbl->gc_thresh1))
                goto out;

        for (i = 0 ; i < (1 << nht->hash_shift); i++) {
                neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[i]) {
                        unsigned int state;

                        write_lock(&n->lock);

                        state = n->nud_state;
                        if ((state & (NUD_PERMANENT | NUD_IN_TIMER)) ||
                            (n->flags &
                             (NTF_EXT_LEARNED | NTF_EXT_VALIDATED))) {
                                write_unlock(&n->lock);
                                continue;
                        }

                        if (time_before(n->used, n->confirmed) &&
                            time_is_before_eq_jiffies(n->confirmed))
                                n->used = n->confirmed;

                        if (refcount_read(&n->refcnt) == 1 &&
                            (state == NUD_FAILED ||
                             !time_in_range_open(jiffies, n->used,
                                                 n->used + NEIGH_VAR(n->parms, GC_STALETIME)))) {
                                hlist_del_rcu(&n->hash);
                                hlist_del_rcu(&n->dev_list);
                                neigh_mark_dead(n);
                                write_unlock(&n->lock);
                                neigh_cleanup_and_release(n);
                                continue;
                        }
                        write_unlock(&n->lock);
                }
                /*
                 * It's fine to release lock here, even if hash table
                 * grows while we are preempted.
                 */
                spin_unlock_bh(&tbl->lock);
                cond_resched();
                spin_lock_bh(&tbl->lock);
                nht = rcu_dereference_protected(tbl->nht,
                                                lockdep_is_held(&tbl->lock));
        }
out:
        /* Cycle through all hash buckets every BASE_REACHABLE_TIME/2 ticks.
         * ARP entry timeouts range from 1/2 BASE_REACHABLE_TIME to 3/2
         * BASE_REACHABLE_TIME.
         */
        queue_delayed_work(system_power_efficient_wq, &tbl->gc_work,
                              NEIGH_VAR(&tbl->parms, BASE_REACHABLE_TIME) >> 1);
        spin_unlock_bh(&tbl->lock);
}

static __inline__ int neigh_max_probes(struct neighbour *n)
{
        struct neigh_parms *p = n->parms;
        return NEIGH_VAR(p, UCAST_PROBES) + NEIGH_VAR(p, APP_PROBES) +
               (n->nud_state & NUD_PROBE ? NEIGH_VAR(p, MCAST_REPROBES) :
                NEIGH_VAR(p, MCAST_PROBES));
}

static void neigh_invalidate(struct neighbour *neigh)
        __releases(neigh->lock)
        __acquires(neigh->lock)
{
        struct sk_buff *skb;

        NEIGH_CACHE_STAT_INC(neigh->tbl, res_failed);
        neigh_dbg(2, "neigh %p is failed\n", neigh);
        neigh->updated = jiffies;

        /* It is very thin place. report_unreachable is very complicated
           routine. Particularly, it can hit the same neighbour entry!

           So that, we try to be accurate and avoid dead loop. --ANK
         */
        while (neigh->nud_state == NUD_FAILED &&
               (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
                write_unlock(&neigh->lock);
                neigh->ops->error_report(neigh, skb);
                write_lock(&neigh->lock);
        }
        __skb_queue_purge(&neigh->arp_queue);
        neigh->arp_queue_len_bytes = 0;
}

static void neigh_probe(struct neighbour *neigh)
        __releases(neigh->lock)
{
        struct sk_buff *skb = skb_peek_tail(&neigh->arp_queue);
        /* keep skb alive even if arp_queue overflows */
        if (skb)
                skb = skb_clone(skb, GFP_ATOMIC);
        write_unlock(&neigh->lock);
        if (neigh->ops->solicit)
                neigh->ops->solicit(neigh, skb);
        atomic_inc(&neigh->probes);
        consume_skb(skb);
}

/* Called when a timer expires for a neighbour entry. */

static void neigh_timer_handler(struct timer_list *t)
{
        unsigned long now, next;
        struct neighbour *neigh = timer_container_of(neigh, t, timer);
        bool skip_probe = false;
        unsigned int state;
        int notify = 0;

        write_lock(&neigh->lock);

        state = neigh->nud_state;
        now = jiffies;
        next = now + HZ;

        if (!(state & NUD_IN_TIMER))
                goto out;

        if (state & NUD_REACHABLE) {
                if (time_before_eq(now,
                                   neigh->confirmed + neigh->parms->reachable_time)) {
                        neigh_dbg(2, "neigh %p is still alive\n", neigh);
                        next = neigh->confirmed + neigh->parms->reachable_time;
                } else if (time_before_eq(now,
                                          neigh->used +
                                          NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
                        neigh_dbg(2, "neigh %p is delayed\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_DELAY);
                        neigh->updated = jiffies;
                        neigh_suspect(neigh);
                        next = now + NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME);
                } else {
                        neigh_dbg(2, "neigh %p is suspected\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_STALE);
                        neigh->updated = jiffies;
                        neigh_suspect(neigh);
                        notify = 1;
                }
        } else if (state & NUD_DELAY) {
                if (time_before_eq(now,
                                   neigh->confirmed +
                                   NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
                        neigh_dbg(2, "neigh %p is now reachable\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_REACHABLE);
                        neigh->updated = jiffies;
                        neigh_connect(neigh);
                        notify = 1;
                        next = neigh->confirmed + neigh->parms->reachable_time;
                } else {
                        neigh_dbg(2, "neigh %p is probed\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_PROBE);
                        neigh->updated = jiffies;
                        atomic_set(&neigh->probes, 0);
                        notify = 1;
                        next = now + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
                                         HZ/100);
                }
        } else {
                /* NUD_PROBE|NUD_INCOMPLETE */
                next = now + max(NEIGH_VAR(neigh->parms, RETRANS_TIME), HZ/100);
        }

        if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) &&
            atomic_read(&neigh->probes) >= neigh_max_probes(neigh)) {
                if (neigh->nud_state == NUD_PROBE &&
                    neigh->flags & NTF_EXT_VALIDATED) {
                        WRITE_ONCE(neigh->nud_state, NUD_STALE);
                        neigh->updated = jiffies;
                } else {
                        WRITE_ONCE(neigh->nud_state, NUD_FAILED);
                        neigh_invalidate(neigh);
                }
                notify = 1;
                skip_probe = true;
        }

        if (notify)
                __neigh_notify(neigh, RTM_NEWNEIGH, 0, 0);

        if (skip_probe)
                goto out;

        if (neigh->nud_state & NUD_IN_TIMER) {
                if (time_before(next, jiffies + HZ/100))
                        next = jiffies + HZ/100;
                if (!mod_timer(&neigh->timer, next))
                        neigh_hold(neigh);
        }
        if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) {
                neigh_probe(neigh);
        } else {
out:
                write_unlock(&neigh->lock);
        }

        if (notify)
                call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);

        trace_neigh_timer_handler(neigh, 0);

        neigh_release(neigh);
}

int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb,
                       const bool immediate_ok)
{
        int rc;
        bool immediate_probe = false;

        write_lock_bh(&neigh->lock);

        rc = 0;
        if (neigh->nud_state & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE))
                goto out_unlock_bh;
        if (neigh->dead)
                goto out_dead;

        if (!(neigh->nud_state & (NUD_STALE | NUD_INCOMPLETE))) {
                if (NEIGH_VAR(neigh->parms, MCAST_PROBES) +
                    NEIGH_VAR(neigh->parms, APP_PROBES)) {
                        unsigned long next, now = jiffies;

                        atomic_set(&neigh->probes,
                                   NEIGH_VAR(neigh->parms, UCAST_PROBES));
                        neigh_del_timer(neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
                        neigh->updated = now;
                        if (!immediate_ok) {
                                next = now + 1;
                        } else {
                                immediate_probe = true;
                                next = now + max(NEIGH_VAR(neigh->parms,
                                                           RETRANS_TIME),
                                                 HZ / 100);
                        }
                        neigh_add_timer(neigh, next);
                } else {
                        WRITE_ONCE(neigh->nud_state, NUD_FAILED);
                        neigh->updated = jiffies;
                        write_unlock_bh(&neigh->lock);

                        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_FAILED);
                        return 1;
                }
        } else if (neigh->nud_state & NUD_STALE) {
                neigh_dbg(2, "neigh %p is delayed\n", neigh);
                neigh_del_timer(neigh);
                WRITE_ONCE(neigh->nud_state, NUD_DELAY);
                neigh->updated = jiffies;
                neigh_add_timer(neigh, jiffies +
                                NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME));
        }

        if (neigh->nud_state == NUD_INCOMPLETE) {
                if (skb) {
                        while (neigh->arp_queue_len_bytes + skb->truesize >
                               NEIGH_VAR(neigh->parms, QUEUE_LEN_BYTES)) {
                                struct sk_buff *buff;

                                buff = __skb_dequeue(&neigh->arp_queue);
                                if (!buff)
                                        break;
                                neigh->arp_queue_len_bytes -= buff->truesize;
                                kfree_skb_reason(buff, SKB_DROP_REASON_NEIGH_QUEUEFULL);
                                NEIGH_CACHE_STAT_INC(neigh->tbl, unres_discards);
                        }
                        skb_dst_force(skb);
                        __skb_queue_tail(&neigh->arp_queue, skb);
                        neigh->arp_queue_len_bytes += skb->truesize;
                }
                rc = 1;
        }
out_unlock_bh:
        if (immediate_probe)
                neigh_probe(neigh);
        else
                write_unlock(&neigh->lock);
        local_bh_enable();
        trace_neigh_event_send_done(neigh, rc);
        return rc;

out_dead:
        if (neigh->nud_state & NUD_STALE)
                goto out_unlock_bh;
        write_unlock_bh(&neigh->lock);
        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_DEAD);
        trace_neigh_event_send_dead(neigh, 1);
        return 1;
}
EXPORT_SYMBOL(__neigh_event_send);

static void neigh_update_hhs(struct neighbour *neigh)
{
        struct hh_cache *hh;
        void (*update)(struct hh_cache*, const struct net_device*, const unsigned char *)
                = NULL;

        if (neigh->dev->header_ops)
                update = neigh->dev->header_ops->cache_update;

        if (update) {
                hh = &neigh->hh;
                if (READ_ONCE(hh->hh_len)) {
                        write_seqlock_bh(&hh->hh_lock);
                        update(hh, neigh->dev, neigh->ha);
                        write_sequnlock_bh(&hh->hh_lock);
                }
        }
}

static void neigh_update_process_arp_queue(struct neighbour *neigh)
        __releases(neigh->lock)
        __acquires(neigh->lock)
{
        struct sk_buff *skb;

        /* Again: avoid deadlock if something went wrong. */
        while (neigh->nud_state & NUD_VALID &&
               (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
                struct dst_entry *dst = skb_dst(skb);
                struct neighbour *n2, *n1 = neigh;

                write_unlock_bh(&neigh->lock);

                rcu_read_lock();

                /* Why not just use 'neigh' as-is?  The problem is that
                 * things such as shaper, eql, and sch_teql can end up
                 * using alternative, different, neigh objects to output
                 * the packet in the output path.  So what we need to do
                 * here is re-lookup the top-level neigh in the path so
                 * we can reinject the packet there.
                 */
                n2 = NULL;
                if (dst &&
                    READ_ONCE(dst->obsolete) != DST_OBSOLETE_DEAD) {
                        n2 = dst_neigh_lookup_skb(dst, skb);
                        if (n2)
                                n1 = n2;
                }
                READ_ONCE(n1->output)(n1, skb);
                if (n2)
                        neigh_release(n2);
                rcu_read_unlock();

                write_lock_bh(&neigh->lock);
        }
        __skb_queue_purge(&neigh->arp_queue);
        neigh->arp_queue_len_bytes = 0;
}

/* Generic update routine.
   -- lladdr is new lladdr or NULL, if it is not supplied.
   -- new    is new state.
   -- flags
        NEIGH_UPDATE_F_OVERRIDE allows to override existing lladdr,
                                if it is different.
        NEIGH_UPDATE_F_WEAK_OVERRIDE will suspect existing "connected"
                                lladdr instead of overriding it
                                if it is different.
        NEIGH_UPDATE_F_ADMIN    means that the change is administrative.
        NEIGH_UPDATE_F_USE      means that the entry is user triggered.
        NEIGH_UPDATE_F_MANAGED  means that the entry will be auto-refreshed.
        NEIGH_UPDATE_F_OVERRIDE_ISROUTER allows to override existing
                                NTF_ROUTER flag.
        NEIGH_UPDATE_F_ISROUTER indicates if the neighbour is known as
                                a router.
        NEIGH_UPDATE_F_EXT_VALIDATED means that the entry will not be removed
                                or invalidated.

   Caller MUST hold reference count on the entry.
 */
static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
                          u8 new, u32 flags, u32 nlmsg_pid,
                          struct netlink_ext_ack *extack)
{
        bool gc_update = false, managed_update = false;
        bool process_arp_queue = false;
        int update_isrouter = 0;
        struct net_device *dev;
        int err, notify = 0;
        u8 old;

        trace_neigh_update(neigh, lladdr, new, flags, nlmsg_pid);

        write_lock_bh(&neigh->lock);

        dev    = neigh->dev;
        old    = neigh->nud_state;
        err    = -EPERM;

        if (neigh->dead) {
                NL_SET_ERR_MSG(extack, "Neighbor entry is now dead");
                new = old;
                goto out;
        }
        if (!(flags & NEIGH_UPDATE_F_ADMIN) &&
            (old & (NUD_NOARP | NUD_PERMANENT)))
                goto out;

        neigh_update_flags(neigh, flags, &notify, &gc_update, &managed_update);
        if (flags & (NEIGH_UPDATE_F_USE | NEIGH_UPDATE_F_MANAGED)) {
                new = old & ~NUD_PERMANENT;
                WRITE_ONCE(neigh->nud_state, new);
                err = 0;
                goto out;
        }

        if (!(new & NUD_VALID)) {
                neigh_del_timer(neigh);
                if (old & NUD_CONNECTED)
                        neigh_suspect(neigh);
                WRITE_ONCE(neigh->nud_state, new);
                err = 0;
                notify = old & NUD_VALID;
                if ((old & (NUD_INCOMPLETE | NUD_PROBE)) &&
                    (new & NUD_FAILED)) {
                        neigh_invalidate(neigh);
                        notify = 1;
                }
                goto out;
        }

        /* Compare new lladdr with cached one */
        if (!dev->addr_len) {
                /* First case: device needs no address. */
                lladdr = neigh->ha;
        } else if (lladdr) {
                /* The second case: if something is already cached
                   and a new address is proposed:
                   - compare new & old
                   - if they are different, check override flag
                 */
                if ((old & NUD_VALID) &&
                    !memcmp(lladdr, neigh->ha, dev->addr_len))
                        lladdr = neigh->ha;
        } else {
                /* No address is supplied; if we know something,
                   use it, otherwise discard the request.
                 */
                err = -EINVAL;
                if (!(old & NUD_VALID)) {
                        NL_SET_ERR_MSG(extack, "No link layer address given");
                        goto out;
                }
                lladdr = neigh->ha;
        }

        /* Update confirmed timestamp for neighbour entry after we
         * received ARP packet even if it doesn't change IP to MAC binding.
         */
        if (new & NUD_CONNECTED)
                neigh->confirmed = jiffies;

        /* If entry was valid and address is not changed,
           do not change entry state, if new one is STALE.
         */
        err = 0;
        update_isrouter = flags & NEIGH_UPDATE_F_OVERRIDE_ISROUTER;
        if (old & NUD_VALID) {
                if (lladdr != neigh->ha && !(flags & NEIGH_UPDATE_F_OVERRIDE)) {
                        update_isrouter = 0;
                        if ((flags & NEIGH_UPDATE_F_WEAK_OVERRIDE) &&
                            (old & NUD_CONNECTED)) {
                                lladdr = neigh->ha;
                                new = NUD_STALE;
                        } else
                                goto out;
                } else {
                        if (lladdr == neigh->ha && new == NUD_STALE &&
                            !(flags & NEIGH_UPDATE_F_ADMIN))
                                new = old;
                }
        }

        /* Update timestamp only once we know we will make a change to the
         * neighbour entry. Otherwise we risk to move the locktime window with
         * noop updates and ignore relevant ARP updates.
         */
        if (new != old || lladdr != neigh->ha)
                neigh->updated = jiffies;

        if (new != old) {
                neigh_del_timer(neigh);
                if (new & NUD_PROBE)
                        atomic_set(&neigh->probes, 0);
                if (new & NUD_IN_TIMER)
                        neigh_add_timer(neigh, (jiffies +
                                                ((new & NUD_REACHABLE) ?
                                                 neigh->parms->reachable_time :
                                                 0)));
                WRITE_ONCE(neigh->nud_state, new);
                notify = 1;
        }

        if (lladdr != neigh->ha) {
                write_seqlock(&neigh->ha_lock);
                memcpy(&neigh->ha, lladdr, dev->addr_len);
                write_sequnlock(&neigh->ha_lock);
                neigh_update_hhs(neigh);
                if (!(new & NUD_CONNECTED))
                        neigh->confirmed = jiffies -
                                      (NEIGH_VAR(neigh->parms, BASE_REACHABLE_TIME) << 1);
                notify = 1;
        }
        if (new == old)
                goto out;
        if (new & NUD_CONNECTED)
                neigh_connect(neigh);
        else
                neigh_suspect(neigh);

        if (!(old & NUD_VALID))
                process_arp_queue = true;

out:
        if (update_isrouter)
                neigh_update_is_router(neigh, flags, &notify);

        if (notify)
                __neigh_notify(neigh, RTM_NEWNEIGH, 0, nlmsg_pid);

        if (process_arp_queue)
                neigh_update_process_arp_queue(neigh);

        write_unlock_bh(&neigh->lock);

        if (((new ^ old) & NUD_PERMANENT) || gc_update)
                neigh_update_gc_list(neigh);
        if (managed_update)
                neigh_update_managed_list(neigh);

        if (notify)
                call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);

        trace_neigh_update_done(neigh, err);
        return err;
}

int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,
                 u32 flags, u32 nlmsg_pid)
{
        return __neigh_update(neigh, lladdr, new, flags, nlmsg_pid, NULL);
}
EXPORT_SYMBOL(neigh_update);

/* Update the neigh to listen temporarily for probe responses, even if it is
 * in a NUD_FAILED state. The caller has to hold neigh->lock for writing.
 */
void __neigh_set_probe_once(struct neighbour *neigh)
{
        if (neigh->dead)
                return;
        neigh->updated = jiffies;
        if (!(neigh->nud_state & NUD_FAILED))
                return;
        WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
        atomic_set(&neigh->probes, neigh_max_probes(neigh));
        neigh_add_timer(neigh,
                        jiffies + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
                                      HZ/100));
}
EXPORT_SYMBOL(__neigh_set_probe_once);

struct neighbour *neigh_event_ns(struct neigh_table *tbl,
                                 u8 *lladdr, void *saddr,
                                 struct net_device *dev)
{
        struct neighbour *neigh = __neigh_lookup(tbl, saddr, dev,
                                                 lladdr || !dev->addr_len);
        if (neigh)
                neigh_update(neigh, lladdr, NUD_STALE,
                             NEIGH_UPDATE_F_OVERRIDE, 0);
        return neigh;
}
EXPORT_SYMBOL(neigh_event_ns);

/* called with read_lock_bh(&n->lock); */
static void neigh_hh_init(struct neighbour *n)
{
        struct net_device *dev = n->dev;
        __be16 prot = n->tbl->protocol;
        struct hh_cache *hh = &n->hh;

        write_lock_bh(&n->lock);

        /* Only one thread can come in here and initialize the
         * hh_cache entry.
         */
        if (!hh->hh_len)
                dev->header_ops->cache(n, hh, prot);

        write_unlock_bh(&n->lock);
}

/* Slow and careful. */

int neigh_resolve_output(struct neighbour *neigh, struct sk_buff *skb)
{
        int rc = 0;

        if (!neigh_event_send(neigh, skb)) {
                int err;
                struct net_device *dev = neigh->dev;
                unsigned int seq;

                if (dev->header_ops->cache && !READ_ONCE(neigh->hh.hh_len))
                        neigh_hh_init(neigh);

                do {
                        __skb_pull(skb, skb_network_offset(skb));
                        seq = read_seqbegin(&neigh->ha_lock);
                        err = dev_hard_header(skb, dev, ntohs(skb->protocol),
                                              neigh->ha, NULL, skb->len);
                } while (read_seqretry(&neigh->ha_lock, seq));

                if (err >= 0)
                        rc = dev_queue_xmit(skb);
                else
                        goto out_kfree_skb;
        }
out:
        return rc;
out_kfree_skb:
        rc = -EINVAL;
        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_HH_FILLFAIL);
        goto out;
}
EXPORT_SYMBOL(neigh_resolve_output);

/* As fast as possible without hh cache */

int neigh_connected_output(struct neighbour *neigh, struct sk_buff *skb)
{
        struct net_device *dev = neigh->dev;
        unsigned int seq;
        int err;

        do {
                __skb_pull(skb, skb_network_offset(skb));
                seq = read_seqbegin(&neigh->ha_lock);
                err = dev_hard_header(skb, dev, ntohs(skb->protocol),
                                      neigh->ha, NULL, skb->len);
        } while (read_seqretry(&neigh->ha_lock, seq));

        if (err >= 0)
                err = dev_queue_xmit(skb);
        else {
                err = -EINVAL;
                kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_HH_FILLFAIL);
        }
        return err;
}

int neigh_direct_output(struct neighbour *neigh, struct sk_buff *skb)
{
        return dev_queue_xmit(skb);
}

static void neigh_managed_work(struct work_struct *work)
{
        struct neigh_table *tbl = container_of(work, struct neigh_table,
                                               managed_work.work);
        struct neighbour *neigh;

        spin_lock_bh(&tbl->lock);
        list_for_each_entry(neigh, &tbl->managed_list, managed_list)
                neigh_event_send_probe(neigh, NULL, false);
        queue_delayed_work(system_power_efficient_wq, &tbl->managed_work,
                           NEIGH_VAR(&tbl->parms, INTERVAL_PROBE_TIME_MS));
        spin_unlock_bh(&tbl->lock);
}

static void neigh_proxy_process(struct timer_list *t)
{
        struct neigh_table *tbl = timer_container_of(tbl, t, proxy_timer);
        long sched_next = 0;
        unsigned long now = jiffies;
        struct sk_buff *skb, *n;

        spin_lock(&tbl->proxy_queue.lock);

        skb_queue_walk_safe(&tbl->proxy_queue, skb, n) {
                long tdif = NEIGH_CB(skb)->sched_next - now;

                if (tdif <= 0) {
                        struct net_device *dev = skb->dev;

                        neigh_parms_qlen_dec(dev, tbl->family);
                        __skb_unlink(skb, &tbl->proxy_queue);

                        if (tbl->proxy_redo && netif_running(dev)) {
                                rcu_read_lock();
                                tbl->proxy_redo(skb);
                                rcu_read_unlock();
                        } else {
                                kfree_skb(skb);
                        }

                        dev_put(dev);
                } else if (!sched_next || tdif < sched_next)
                        sched_next = tdif;
        }
        timer_delete(&tbl->proxy_timer);
        if (sched_next)
                mod_timer(&tbl->proxy_timer, jiffies + sched_next);
        spin_unlock(&tbl->proxy_queue.lock);
}

static unsigned long neigh_proxy_delay(struct neigh_parms *p)
{
        /* If proxy_delay is zero, do not call get_random_u32_below()
         * as it is undefined behavior.
         */
        unsigned long proxy_delay = NEIGH_VAR(p, PROXY_DELAY);

        return proxy_delay ?
               jiffies + get_random_u32_below(proxy_delay) : jiffies;
}

void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p,
                    struct sk_buff *skb)
{
        unsigned long sched_next = neigh_proxy_delay(p);

        if (p->qlen > NEIGH_VAR(p, PROXY_QLEN)) {
                kfree_skb(skb);
                return;
        }

        NEIGH_CB(skb)->sched_next = sched_next;
        NEIGH_CB(skb)->flags |= LOCALLY_ENQUEUED;

        spin_lock(&tbl->proxy_queue.lock);
        if (timer_delete(&tbl->proxy_timer)) {
                if (time_before(tbl->proxy_timer.expires, sched_next))
                        sched_next = tbl->proxy_timer.expires;
        }
        skb_dst_drop(skb);
        dev_hold(skb->dev);
        __skb_queue_tail(&tbl->proxy_queue, skb);
        p->qlen++;
        mod_timer(&tbl->proxy_timer, sched_next);
        spin_unlock(&tbl->proxy_queue.lock);
}
EXPORT_SYMBOL(pneigh_enqueue);

static inline struct neigh_parms *lookup_neigh_parms(struct neigh_table *tbl,
                                                      struct net *net, int ifindex)
{
        struct neigh_parms *p;

        list_for_each_entry(p, &tbl->parms_list, list) {
                if ((p->dev && p->dev->ifindex == ifindex && net_eq(neigh_parms_net(p), net)) ||
                    (!p->dev && !ifindex && net_eq(net, &init_net)))
                        return p;
        }

        return NULL;
}

struct neigh_parms *neigh_parms_alloc(struct net_device *dev,
                                      struct neigh_table *tbl)
{
        struct neigh_parms *p;
        struct net *net = dev_net(dev);
        const struct net_device_ops *ops = dev->netdev_ops;

        p = kmemdup(&tbl->parms, sizeof(*p), GFP_KERNEL);
        if (p) {
                p->tbl            = tbl;
                refcount_set(&p->refcnt, 1);
                neigh_set_reach_time(p);
                p->qlen = 0;
                netdev_hold(dev, &p->dev_tracker, GFP_KERNEL);
                p->dev = dev;
                write_pnet(&p->net, net);
                p->sysctl_table = NULL;

                if (ops->ndo_neigh_setup && ops->ndo_neigh_setup(dev, p)) {
                        netdev_put(dev, &p->dev_tracker);
                        kfree(p);
                        return NULL;
                }

                spin_lock_bh(&tbl->lock);
                list_add_rcu(&p->list, &tbl->parms.list);
                spin_unlock_bh(&tbl->lock);

                neigh_parms_data_state_cleanall(p);
        }
        return p;
}
EXPORT_SYMBOL(neigh_parms_alloc);

static void neigh_rcu_free_parms(struct rcu_head *head)
{
        struct neigh_parms *parms =
                container_of(head, struct neigh_parms, rcu_head);

        neigh_parms_put(parms);
}

void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms)
{
        if (!parms || parms == &tbl->parms)
                return;

        spin_lock_bh(&tbl->lock);
        list_del_rcu(&parms->list);
        parms->dead = 1;
        spin_unlock_bh(&tbl->lock);

        netdev_put(parms->dev, &parms->dev_tracker);
        call_rcu(&parms->rcu_head, neigh_rcu_free_parms);
}
EXPORT_SYMBOL(neigh_parms_release);

static struct lock_class_key neigh_table_proxy_queue_class;

static struct neigh_table __rcu *neigh_tables[NEIGH_NR_TABLES] __read_mostly;

void neigh_table_init(int index, struct neigh_table *tbl)
{
        unsigned long now = jiffies;
        unsigned long phsize;

        INIT_LIST_HEAD(&tbl->parms_list);
        INIT_LIST_HEAD(&tbl->gc_list);
        INIT_LIST_HEAD(&tbl->managed_list);

        list_add(&tbl->parms.list, &tbl->parms_list);
        write_pnet(&tbl->parms.net, &init_net);
        refcount_set(&tbl->parms.refcnt, 1);
        neigh_set_reach_time(&tbl->parms);
        tbl->parms.qlen = 0;

        tbl->stats = alloc_percpu(struct neigh_statistics);
        if (!tbl->stats)
                panic("cannot create neighbour cache statistics");

#ifdef CONFIG_PROC_FS
        if (!proc_create_seq_data(tbl->id, 0, init_net.proc_net_stat,
                              &neigh_stat_seq_ops, tbl))
                panic("cannot create neighbour proc dir entry");
#endif

        RCU_INIT_POINTER(tbl->nht, neigh_hash_alloc(3));

        phsize = (PNEIGH_HASHMASK + 1) * sizeof(struct pneigh_entry *);
        tbl->phash_buckets = kzalloc(phsize, GFP_KERNEL);

        if (!tbl->nht || !tbl->phash_buckets)
                panic("cannot allocate neighbour cache hashes");

        if (!tbl->entry_size)
                tbl->entry_size = ALIGN(offsetof(struct neighbour, primary_key) +
                                        tbl->key_len, NEIGH_PRIV_ALIGN);
        else
                WARN_ON(tbl->entry_size % NEIGH_PRIV_ALIGN);

        spin_lock_init(&tbl->lock);
        mutex_init(&tbl->phash_lock);

        INIT_DEFERRABLE_WORK(&tbl->gc_work, neigh_periodic_work);
        queue_delayed_work(system_power_efficient_wq, &tbl->gc_work,
                        tbl->parms.reachable_time);
        INIT_DEFERRABLE_WORK(&tbl->managed_work, neigh_managed_work);
        queue_delayed_work(system_power_efficient_wq, &tbl->managed_work, 0);

        timer_setup(&tbl->proxy_timer, neigh_proxy_process, 0);
        skb_queue_head_init_class(&tbl->proxy_queue,
                        &neigh_table_proxy_queue_class);

        tbl->last_flush = now;
        tbl->last_rand  = now + tbl->parms.reachable_time * 20;

        rcu_assign_pointer(neigh_tables[index], tbl);
}

/*
 * Only called from ndisc_cleanup(), which means this is dead code
 * because we no longer can unload IPv6 module.
 */
int neigh_table_clear(int index, struct neigh_table *tbl)
{
        RCU_INIT_POINTER(neigh_tables[index], NULL);
        synchronize_rcu();

        /* It is not clean... Fix it to unload IPv6 module safely */
        cancel_delayed_work_sync(&tbl->managed_work);
        cancel_delayed_work_sync(&tbl->gc_work);
        timer_delete_sync(&tbl->proxy_timer);
        pneigh_queue_purge(&tbl->proxy_queue, NULL, tbl->family);
        neigh_ifdown(tbl, NULL);
        if (atomic_read(&tbl->entries))
                pr_crit("neighbour leakage\n");

        call_rcu(&rcu_dereference_protected(tbl->nht, 1)->rcu,
                 neigh_hash_free_rcu);
        tbl->nht = NULL;

        kfree(tbl->phash_buckets);
        tbl->phash_buckets = NULL;

        remove_proc_entry(tbl->id, init_net.proc_net_stat);

        free_percpu(tbl->stats);
        tbl->stats = NULL;

        return 0;
}

static struct neigh_table *neigh_find_table(int family)
{
        struct neigh_table *tbl = NULL;

        switch (family) {
        case AF_INET:
                tbl = rcu_dereference_rtnl(neigh_tables[NEIGH_ARP_TABLE]);
                break;
        case AF_INET6:
                tbl = rcu_dereference_rtnl(neigh_tables[NEIGH_ND_TABLE]);
                break;
        }

        return tbl;
}

const struct nla_policy nda_policy[NDA_MAX+1] = {
        [NDA_UNSPEC]            = { .strict_start_type = NDA_NH_ID },
        [NDA_DST]               = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
        [NDA_LLADDR]            = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
        [NDA_CACHEINFO]         = { .len = sizeof(struct nda_cacheinfo) },
        [NDA_PROBES]            = { .type = NLA_U32 },
        [NDA_VLAN]              = { .type = NLA_U16 },
        [NDA_PORT]              = { .type = NLA_U16 },
        [NDA_VNI]               = { .type = NLA_U32 },
        [NDA_IFINDEX]           = { .type = NLA_U32 },
        [NDA_MASTER]            = { .type = NLA_U32 },
        [NDA_PROTOCOL]          = { .type = NLA_U8 },
        [NDA_NH_ID]             = { .type = NLA_U32 },
        [NDA_FLAGS_EXT]         = NLA_POLICY_MASK(NLA_U32, NTF_EXT_MASK),
        [NDA_FDB_EXT_ATTRS]     = { .type = NLA_NESTED },
};

static int neigh_delete(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct ndmsg *ndm;
        struct nlattr *dst_attr;
        struct neigh_table *tbl;
        struct neighbour *neigh;
        struct net_device *dev = NULL;
        int err = -EINVAL;

        ASSERT_RTNL();
        if (nlmsg_len(nlh) < sizeof(*ndm))
                goto out;

        dst_attr = nlmsg_find_attr(nlh, sizeof(*ndm), NDA_DST);
        if (!dst_attr) {
                NL_SET_ERR_MSG(extack, "Network address not specified");
                goto out;
        }

        ndm = nlmsg_data(nlh);
        if (ndm->ndm_ifindex) {
                dev = __dev_get_by_index(net, ndm->ndm_ifindex);
                if (dev == NULL) {
                        err = -ENODEV;
                        goto out;
                }
        }

        tbl = neigh_find_table(ndm->ndm_family);
        if (tbl == NULL)
                return -EAFNOSUPPORT;

        if (nla_len(dst_attr) < (int)tbl->key_len) {
                NL_SET_ERR_MSG(extack, "Invalid network address");
                goto out;
        }

        if (ndm->ndm_flags & NTF_PROXY) {
                err = pneigh_delete(tbl, net, nla_data(dst_attr), dev);
                goto out;
        }

        if (dev == NULL)
                goto out;

        neigh = neigh_lookup(tbl, nla_data(dst_attr), dev);
        if (neigh == NULL) {
                err = -ENOENT;
                goto out;
        }

        err = __neigh_update(neigh, NULL, NUD_FAILED,
                             NEIGH_UPDATE_F_OVERRIDE | NEIGH_UPDATE_F_ADMIN,
                             NETLINK_CB(skb).portid, extack);
        spin_lock_bh(&tbl->lock);
        neigh_release(neigh);
        neigh_remove_one(neigh);
        spin_unlock_bh(&tbl->lock);

out:
        return err;
}

static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh,
                     struct netlink_ext_ack *extack)
{
        int flags = NEIGH_UPDATE_F_ADMIN | NEIGH_UPDATE_F_OVERRIDE |
                    NEIGH_UPDATE_F_OVERRIDE_ISROUTER;
        struct net *net = sock_net(skb->sk);
        struct ndmsg *ndm;
        struct nlattr *tb[NDA_MAX+1];
        struct neigh_table *tbl;
        struct net_device *dev = NULL;
        struct neighbour *neigh;
        void *dst, *lladdr;
        u8 protocol = 0;
        u32 ndm_flags;
        int err;

        ASSERT_RTNL();
        err = nlmsg_parse_deprecated(nlh, sizeof(*ndm), tb, NDA_MAX,
                                     nda_policy, extack);
        if (err < 0)
                goto out;

        err = -EINVAL;
        if (!tb[NDA_DST]) {
                NL_SET_ERR_MSG(extack, "Network address not specified");
                goto out;
        }

        ndm = nlmsg_data(nlh);
        ndm_flags = ndm->ndm_flags;
        if (tb[NDA_FLAGS_EXT]) {
                u32 ext = nla_get_u32(tb[NDA_FLAGS_EXT]);

                BUILD_BUG_ON(sizeof(neigh->flags) * BITS_PER_BYTE <
                             (sizeof(ndm->ndm_flags) * BITS_PER_BYTE +
                              hweight32(NTF_EXT_MASK)));
                ndm_flags |= (ext << NTF_EXT_SHIFT);
        }
        if (ndm->ndm_ifindex) {
                dev = __dev_get_by_index(net, ndm->ndm_ifindex);
                if (dev == NULL) {
                        err = -ENODEV;
                        goto out;
                }

                if (tb[NDA_LLADDR] && nla_len(tb[NDA_LLADDR]) < dev->addr_len) {
                        NL_SET_ERR_MSG(extack, "Invalid link address");
                        goto out;
                }
        }

        tbl = neigh_find_table(ndm->ndm_family);
        if (tbl == NULL)
                return -EAFNOSUPPORT;

        if (nla_len(tb[NDA_DST]) < (int)tbl->key_len) {
                NL_SET_ERR_MSG(extack, "Invalid network address");
                goto out;
        }

        dst = nla_data(tb[NDA_DST]);
        lladdr = tb[NDA_LLADDR] ? nla_data(tb[NDA_LLADDR]) : NULL;

        if (tb[NDA_PROTOCOL])
                protocol = nla_get_u8(tb[NDA_PROTOCOL]);
        if (ndm_flags & NTF_PROXY) {
                if (ndm_flags & (NTF_MANAGED | NTF_EXT_VALIDATED)) {
                        NL_SET_ERR_MSG(extack, "Invalid NTF_* flag combination");
                        goto out;
                }

                err = pneigh_create(tbl, net, dst, dev, ndm_flags, protocol,
                                    !!(ndm->ndm_state & NUD_PERMANENT));
                goto out;
        }

        if (!dev) {
                NL_SET_ERR_MSG(extack, "Device not specified");
                goto out;
        }

        if (tbl->allow_add && !tbl->allow_add(dev, extack)) {
                err = -EINVAL;
                goto out;
        }

        neigh = neigh_lookup(tbl, dst, dev);
        if (neigh == NULL) {
                bool ndm_permanent  = ndm->ndm_state & NUD_PERMANENT;
                bool exempt_from_gc = ndm_permanent ||
                                      ndm_flags & (NTF_EXT_LEARNED |
                                                   NTF_EXT_VALIDATED);

                if (!(nlh->nlmsg_flags & NLM_F_CREATE)) {
                        err = -ENOENT;
                        goto out;
                }
                if (ndm_permanent && (ndm_flags & NTF_MANAGED)) {
                        NL_SET_ERR_MSG(extack, "Invalid NTF_* flag for permanent entry");
                        err = -EINVAL;
                        goto out;
                }
                if (ndm_flags & NTF_EXT_VALIDATED) {
                        u8 state = ndm->ndm_state;

                        /* NTF_USE and NTF_MANAGED will result in the neighbor
                         * being created with an invalid state (NUD_NONE).
                         */
                        if (ndm_flags & (NTF_USE | NTF_MANAGED))
                                state = NUD_NONE;

                        if (!(state & NUD_VALID)) {
                                NL_SET_ERR_MSG(extack,
                                               "Cannot create externally validated neighbor with an invalid state");
                                err = -EINVAL;
                                goto out;
                        }
                }

                neigh = ___neigh_create(tbl, dst, dev,
                                        ndm_flags &
                                        (NTF_EXT_LEARNED | NTF_MANAGED |
                                         NTF_EXT_VALIDATED),
                                        exempt_from_gc, true);
                if (IS_ERR(neigh)) {
                        err = PTR_ERR(neigh);
                        goto out;
                }
        } else {
                if (nlh->nlmsg_flags & NLM_F_EXCL) {
                        err = -EEXIST;
                        neigh_release(neigh);
                        goto out;
                }
                if (ndm_flags & NTF_EXT_VALIDATED) {
                        u8 state = ndm->ndm_state;

                        /* NTF_USE and NTF_MANAGED do not update the existing
                         * state other than clearing it if it was
                         * NUD_PERMANENT.
                         */
                        if (ndm_flags & (NTF_USE | NTF_MANAGED))
                                state = READ_ONCE(neigh->nud_state) & ~NUD_PERMANENT;

                        if (!(state & NUD_VALID)) {
                                NL_SET_ERR_MSG(extack,
                                               "Cannot mark neighbor as externally validated with an invalid state");
                                err = -EINVAL;
                                neigh_release(neigh);
                                goto out;
                        }
                }

                if (!(nlh->nlmsg_flags & NLM_F_REPLACE))
                        flags &= ~(NEIGH_UPDATE_F_OVERRIDE |
                                   NEIGH_UPDATE_F_OVERRIDE_ISROUTER);
        }

        if (protocol)
                neigh->protocol = protocol;
        if (ndm_flags & NTF_EXT_LEARNED)
                flags |= NEIGH_UPDATE_F_EXT_LEARNED;
        if (ndm_flags & NTF_ROUTER)
                flags |= NEIGH_UPDATE_F_ISROUTER;
        if (ndm_flags & NTF_MANAGED)
                flags |= NEIGH_UPDATE_F_MANAGED;
        if (ndm_flags & NTF_USE)
                flags |= NEIGH_UPDATE_F_USE;
        if (ndm_flags & NTF_EXT_VALIDATED)
                flags |= NEIGH_UPDATE_F_EXT_VALIDATED;

        err = __neigh_update(neigh, lladdr, ndm->ndm_state, flags,
                             NETLINK_CB(skb).portid, extack);
        if (!err && ndm_flags & (NTF_USE | NTF_MANAGED))
                neigh_event_send(neigh, NULL);
        neigh_release(neigh);
out:
        return err;
}

static int neightbl_fill_parms(struct sk_buff *skb, struct neigh_parms *parms)
{
        struct nlattr *nest;

        nest = nla_nest_start_noflag(skb, NDTA_PARMS);
        if (nest == NULL)
                return -ENOBUFS;

        if ((parms->dev &&
             nla_put_u32(skb, NDTPA_IFINDEX, READ_ONCE(parms->dev->ifindex))) ||
            nla_put_u32(skb, NDTPA_REFCNT, refcount_read(&parms->refcnt)) ||
            nla_put_u32(skb, NDTPA_QUEUE_LENBYTES,
                        NEIGH_VAR(parms, QUEUE_LEN_BYTES)) ||
            /* approximative value for deprecated QUEUE_LEN (in packets) */
            nla_put_u32(skb, NDTPA_QUEUE_LEN,
                        NEIGH_VAR(parms, QUEUE_LEN_BYTES) / SKB_TRUESIZE(ETH_FRAME_LEN)) ||
            nla_put_u32(skb, NDTPA_PROXY_QLEN, NEIGH_VAR(parms, PROXY_QLEN)) ||
            nla_put_u32(skb, NDTPA_APP_PROBES, NEIGH_VAR(parms, APP_PROBES)) ||
            nla_put_u32(skb, NDTPA_UCAST_PROBES,
                        NEIGH_VAR(parms, UCAST_PROBES)) ||
            nla_put_u32(skb, NDTPA_MCAST_PROBES,
                        NEIGH_VAR(parms, MCAST_PROBES)) ||
            nla_put_u32(skb, NDTPA_MCAST_REPROBES,
                        NEIGH_VAR(parms, MCAST_REPROBES)) ||
            nla_put_msecs(skb, NDTPA_REACHABLE_TIME, READ_ONCE(parms->reachable_time),
                          NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_BASE_REACHABLE_TIME,
                          NEIGH_VAR(parms, BASE_REACHABLE_TIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_GC_STALETIME,
                          NEIGH_VAR(parms, GC_STALETIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_DELAY_PROBE_TIME,
                          NEIGH_VAR(parms, DELAY_PROBE_TIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_RETRANS_TIME,
                          NEIGH_VAR(parms, RETRANS_TIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_ANYCAST_DELAY,
                          NEIGH_VAR(parms, ANYCAST_DELAY), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_PROXY_DELAY,
                          NEIGH_VAR(parms, PROXY_DELAY), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_LOCKTIME,
                          NEIGH_VAR(parms, LOCKTIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_INTERVAL_PROBE_TIME_MS,
                          NEIGH_VAR(parms, INTERVAL_PROBE_TIME_MS), NDTPA_PAD))
                goto nla_put_failure;
        return nla_nest_end(skb, nest);

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int neightbl_fill_info(struct sk_buff *skb, struct neigh_table *tbl,
                              u32 pid, u32 seq, int type, int flags)
{
        struct nlmsghdr *nlh;
        struct ndtmsg *ndtmsg;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndtmsg), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        ndtmsg = nlmsg_data(nlh);
        ndtmsg->ndtm_family = tbl->family;
        ndtmsg->ndtm_pad1   = 0;
        ndtmsg->ndtm_pad2   = 0;

        if (nla_put_string(skb, NDTA_NAME, tbl->id) ||
            nla_put_msecs(skb, NDTA_GC_INTERVAL, READ_ONCE(tbl->gc_interval),
                          NDTA_PAD) ||
            nla_put_u32(skb, NDTA_THRESH1, READ_ONCE(tbl->gc_thresh1)) ||
            nla_put_u32(skb, NDTA_THRESH2, READ_ONCE(tbl->gc_thresh2)) ||
            nla_put_u32(skb, NDTA_THRESH3, READ_ONCE(tbl->gc_thresh3)))
                goto nla_put_failure;
        {
                unsigned long now = jiffies;
                long flush_delta = now - READ_ONCE(tbl->last_flush);
                long rand_delta = now - READ_ONCE(tbl->last_rand);
                struct neigh_hash_table *nht;
                struct ndt_config ndc = {
                        .ndtc_key_len           = tbl->key_len,
                        .ndtc_entry_size        = tbl->entry_size,
                        .ndtc_entries           = atomic_read(&tbl->entries),
                        .ndtc_last_flush        = jiffies_to_msecs(flush_delta),
                        .ndtc_last_rand         = jiffies_to_msecs(rand_delta),
                        .ndtc_proxy_qlen        = READ_ONCE(tbl->proxy_queue.qlen),
                };

                nht = rcu_dereference(tbl->nht);
                ndc.ndtc_hash_rnd = nht->hash_rnd[0];
                ndc.ndtc_hash_mask = ((1 << nht->hash_shift) - 1);

                if (nla_put(skb, NDTA_CONFIG, sizeof(ndc), &ndc))
                        goto nla_put_failure;
        }

        {
                int cpu;
                struct ndt_stats ndst;

                memset(&ndst, 0, sizeof(ndst));

                for_each_possible_cpu(cpu) {
                        struct neigh_statistics *st;

                        st = per_cpu_ptr(tbl->stats, cpu);
                        ndst.ndts_allocs                += READ_ONCE(st->allocs);
                        ndst.ndts_destroys              += READ_ONCE(st->destroys);
                        ndst.ndts_hash_grows            += READ_ONCE(st->hash_grows);
                        ndst.ndts_res_failed            += READ_ONCE(st->res_failed);
                        ndst.ndts_lookups               += READ_ONCE(st->lookups);
                        ndst.ndts_hits                  += READ_ONCE(st->hits);
                        ndst.ndts_rcv_probes_mcast      += READ_ONCE(st->rcv_probes_mcast);
                        ndst.ndts_rcv_probes_ucast      += READ_ONCE(st->rcv_probes_ucast);
                        ndst.ndts_periodic_gc_runs      += READ_ONCE(st->periodic_gc_runs);
                        ndst.ndts_forced_gc_runs        += READ_ONCE(st->forced_gc_runs);
                        ndst.ndts_table_fulls           += READ_ONCE(st->table_fulls);
                }

                if (nla_put_64bit(skb, NDTA_STATS, sizeof(ndst), &ndst,
                                  NDTA_PAD))
                        goto nla_put_failure;
        }

        BUG_ON(tbl->parms.dev);
        if (neightbl_fill_parms(skb, &tbl->parms) < 0)
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int neightbl_fill_param_info(struct sk_buff *skb,
                                    struct neigh_table *tbl,
                                    struct neigh_parms *parms,
                                    u32 pid, u32 seq, int type,
                                    unsigned int flags)
{
        struct ndtmsg *ndtmsg;
        struct nlmsghdr *nlh;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndtmsg), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        ndtmsg = nlmsg_data(nlh);
        ndtmsg->ndtm_family = tbl->family;
        ndtmsg->ndtm_pad1   = 0;
        ndtmsg->ndtm_pad2   = 0;

        if (nla_put_string(skb, NDTA_NAME, tbl->id) < 0 ||
            neightbl_fill_parms(skb, parms) < 0)
                goto errout;

        nlmsg_end(skb, nlh);
        return 0;
errout:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static const struct nla_policy nl_neightbl_policy[NDTA_MAX+1] = {
        [NDTA_NAME]             = { .type = NLA_STRING },
        [NDTA_THRESH1]          = { .type = NLA_U32 },
        [NDTA_THRESH2]          = { .type = NLA_U32 },
        [NDTA_THRESH3]          = { .type = NLA_U32 },
        [NDTA_GC_INTERVAL]      = { .type = NLA_U64 },
        [NDTA_PARMS]            = { .type = NLA_NESTED },
};

static const struct nla_policy nl_ntbl_parm_policy[NDTPA_MAX+1] = {
        [NDTPA_IFINDEX]                 = { .type = NLA_U32 },
        [NDTPA_QUEUE_LEN]               = { .type = NLA_U32 },
        [NDTPA_QUEUE_LENBYTES]          = { .type = NLA_U32 },
        [NDTPA_PROXY_QLEN]              = { .type = NLA_U32 },
        [NDTPA_APP_PROBES]              = { .type = NLA_U32 },
        [NDTPA_UCAST_PROBES]            = { .type = NLA_U32 },
        [NDTPA_MCAST_PROBES]            = { .type = NLA_U32 },
        [NDTPA_MCAST_REPROBES]          = { .type = NLA_U32 },
        [NDTPA_BASE_REACHABLE_TIME]     = { .type = NLA_U64 },
        [NDTPA_GC_STALETIME]            = { .type = NLA_U64 },
        [NDTPA_DELAY_PROBE_TIME]        = { .type = NLA_U64 },
        [NDTPA_RETRANS_TIME]            = { .type = NLA_U64 },
        [NDTPA_ANYCAST_DELAY]           = { .type = NLA_U64 },
        [NDTPA_PROXY_DELAY]             = { .type = NLA_U64 },
        [NDTPA_LOCKTIME]                = { .type = NLA_U64 },
        [NDTPA_INTERVAL_PROBE_TIME_MS]  = { .type = NLA_U64, .min = 1 },
};

static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tb[NDTA_MAX + 1];
        struct neigh_table *tbl;
        struct ndtmsg *ndtmsg;
        bool found = false;
        int err, tidx;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ndtmsg), tb, NDTA_MAX,
                                     nl_neightbl_policy, extack);
        if (err < 0)
                goto errout;

        if (tb[NDTA_NAME] == NULL) {
                err = -EINVAL;
                goto errout;
        }

        ndtmsg = nlmsg_data(nlh);

        rcu_read_lock();

        for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
                tbl = rcu_dereference(neigh_tables[tidx]);
                if (!tbl)
                        continue;

                if (ndtmsg->ndtm_family && tbl->family != ndtmsg->ndtm_family)
                        continue;

                if (nla_strcmp(tb[NDTA_NAME], tbl->id) == 0) {
                        found = true;
                        break;
                }
        }

        if (!found) {
                rcu_read_unlock();
                err = -ENOENT;
                goto errout;
        }

        /*
         * We acquire tbl->lock to be nice to the periodic timers and
         * make sure they always see a consistent set of values.
         */
        spin_lock_bh(&tbl->lock);

        if (tb[NDTA_PARMS]) {
                struct nlattr *tbp[NDTPA_MAX+1];
                struct neigh_parms *p;
                int i, ifindex = 0;

                err = nla_parse_nested_deprecated(tbp, NDTPA_MAX,
                                                  tb[NDTA_PARMS],
                                                  nl_ntbl_parm_policy, extack);
                if (err < 0)
                        goto errout_tbl_lock;

                if (tbp[NDTPA_IFINDEX])
                        ifindex = nla_get_u32(tbp[NDTPA_IFINDEX]);

                p = lookup_neigh_parms(tbl, net, ifindex);
                if (p == NULL) {
                        err = -ENOENT;
                        goto errout_tbl_lock;
                }

                for (i = 1; i <= NDTPA_MAX; i++) {
                        if (tbp[i] == NULL)
                                continue;

                        switch (i) {
                        case NDTPA_QUEUE_LEN:
                                NEIGH_VAR_SET(p, QUEUE_LEN_BYTES,
                                              nla_get_u32(tbp[i]) *
                                              SKB_TRUESIZE(ETH_FRAME_LEN));
                                break;
                        case NDTPA_QUEUE_LENBYTES:
                                NEIGH_VAR_SET(p, QUEUE_LEN_BYTES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_PROXY_QLEN:
                                NEIGH_VAR_SET(p, PROXY_QLEN,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_APP_PROBES:
                                NEIGH_VAR_SET(p, APP_PROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_UCAST_PROBES:
                                NEIGH_VAR_SET(p, UCAST_PROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_MCAST_PROBES:
                                NEIGH_VAR_SET(p, MCAST_PROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_MCAST_REPROBES:
                                NEIGH_VAR_SET(p, MCAST_REPROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_BASE_REACHABLE_TIME:
                                NEIGH_VAR_SET(p, BASE_REACHABLE_TIME,
                                              nla_get_msecs(tbp[i]));
                                /* update reachable_time as well, otherwise, the change will
                                 * only be effective after the next time neigh_periodic_work
                                 * decides to recompute it (can be multiple minutes)
                                 */
                                neigh_set_reach_time(p);
                                break;
                        case NDTPA_GC_STALETIME:
                                NEIGH_VAR_SET(p, GC_STALETIME,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_DELAY_PROBE_TIME:
                                NEIGH_VAR_SET(p, DELAY_PROBE_TIME,
                                              nla_get_msecs(tbp[i]));
                                call_netevent_notifiers(NETEVENT_DELAY_PROBE_TIME_UPDATE, p);
                                break;
                        case NDTPA_INTERVAL_PROBE_TIME_MS:
                                NEIGH_VAR_SET(p, INTERVAL_PROBE_TIME_MS,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_RETRANS_TIME:
                                NEIGH_VAR_SET(p, RETRANS_TIME,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_ANYCAST_DELAY:
                                NEIGH_VAR_SET(p, ANYCAST_DELAY,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_PROXY_DELAY:
                                NEIGH_VAR_SET(p, PROXY_DELAY,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_LOCKTIME:
                                NEIGH_VAR_SET(p, LOCKTIME,
                                              nla_get_msecs(tbp[i]));
                                break;
                        }
                }
        }

        err = -ENOENT;
        if ((tb[NDTA_THRESH1] || tb[NDTA_THRESH2] ||
             tb[NDTA_THRESH3] || tb[NDTA_GC_INTERVAL]) &&
            !net_eq(net, &init_net))
                goto errout_tbl_lock;

        if (tb[NDTA_THRESH1])
                WRITE_ONCE(tbl->gc_thresh1, nla_get_u32(tb[NDTA_THRESH1]));

        if (tb[NDTA_THRESH2])
                WRITE_ONCE(tbl->gc_thresh2, nla_get_u32(tb[NDTA_THRESH2]));

        if (tb[NDTA_THRESH3])
                WRITE_ONCE(tbl->gc_thresh3, nla_get_u32(tb[NDTA_THRESH3]));

        if (tb[NDTA_GC_INTERVAL])
                WRITE_ONCE(tbl->gc_interval, nla_get_msecs(tb[NDTA_GC_INTERVAL]));

        err = 0;

errout_tbl_lock:
        spin_unlock_bh(&tbl->lock);
        rcu_read_unlock();
errout:
        return err;
}

static int neightbl_valid_dump_info(const struct nlmsghdr *nlh,
                                    struct netlink_ext_ack *extack)
{
        struct ndtmsg *ndtm;

        ndtm = nlmsg_payload(nlh, sizeof(*ndtm));
        if (!ndtm) {
                NL_SET_ERR_MSG(extack, "Invalid header for neighbor table dump request");
                return -EINVAL;
        }

        if (ndtm->ndtm_pad1  || ndtm->ndtm_pad2) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor table dump request");
                return -EINVAL;
        }

        if (nlmsg_attrlen(nlh, sizeof(*ndtm))) {
                NL_SET_ERR_MSG(extack, "Invalid data after header in neighbor table dump request");
                return -EINVAL;
        }

        return 0;
}

static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        int family, tidx, nidx = 0;
        int tbl_skip = cb->args[0];
        int neigh_skip = cb->args[1];
        struct neigh_table *tbl;

        if (cb->strict_check) {
                int err = neightbl_valid_dump_info(nlh, cb->extack);

                if (err < 0)
                        return err;
        }

        family = ((struct rtgenmsg *)nlmsg_data(nlh))->rtgen_family;

        rcu_read_lock();

        for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
                struct neigh_parms *p;

                tbl = rcu_dereference(neigh_tables[tidx]);
                if (!tbl)
                        continue;

                if (tidx < tbl_skip || (family && tbl->family != family))
                        continue;

                if (neightbl_fill_info(skb, tbl, NETLINK_CB(cb->skb).portid,
                                       nlh->nlmsg_seq, RTM_NEWNEIGHTBL,
                                       NLM_F_MULTI) < 0)
                        break;

                nidx = 0;
                p = list_next_entry(&tbl->parms, list);
                list_for_each_entry_from_rcu(p, &tbl->parms_list, list) {
                        if (!net_eq(neigh_parms_net(p), net))
                                continue;

                        if (nidx < neigh_skip)
                                goto next;

                        if (neightbl_fill_param_info(skb, tbl, p,
                                                     NETLINK_CB(cb->skb).portid,
                                                     nlh->nlmsg_seq,
                                                     RTM_NEWNEIGHTBL,
                                                     NLM_F_MULTI) < 0)
                                goto out;
                next:
                        nidx++;
                }

                neigh_skip = 0;
        }
out:
        rcu_read_unlock();

        cb->args[0] = tidx;
        cb->args[1] = nidx;

        return skb->len;
}

static int __neigh_fill_info(struct sk_buff *skb, struct neighbour *neigh,
                             u32 pid, u32 seq, int type, unsigned int flags)
{
        u32 neigh_flags, neigh_flags_ext;
        unsigned long now = jiffies;
        struct nda_cacheinfo ci;
        struct nlmsghdr *nlh;
        struct ndmsg *ndm;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        neigh_flags_ext = neigh->flags >> NTF_EXT_SHIFT;
        neigh_flags     = neigh->flags & NTF_OLD_MASK;

        ndm = nlmsg_data(nlh);
        ndm->ndm_family  = neigh->ops->family;
        ndm->ndm_pad1    = 0;
        ndm->ndm_pad2    = 0;
        ndm->ndm_flags   = neigh_flags;
        ndm->ndm_type    = neigh->type;
        ndm->ndm_ifindex = neigh->dev->ifindex;

        if (nla_put(skb, NDA_DST, neigh->tbl->key_len, neigh->primary_key))
                goto nla_put_failure;

        ndm->ndm_state   = neigh->nud_state;
        if (neigh->nud_state & NUD_VALID) {
                char haddr[MAX_ADDR_LEN];

                neigh_ha_snapshot(haddr, neigh, neigh->dev);
                if (nla_put(skb, NDA_LLADDR, neigh->dev->addr_len, haddr) < 0)
                        goto nla_put_failure;
        }

        ci.ndm_used      = jiffies_to_clock_t(now - neigh->used);
        ci.ndm_confirmed = jiffies_to_clock_t(now - neigh->confirmed);
        ci.ndm_updated   = jiffies_to_clock_t(now - neigh->updated);
        ci.ndm_refcnt    = refcount_read(&neigh->refcnt) - 1;

        if (nla_put_u32(skb, NDA_PROBES, atomic_read(&neigh->probes)) ||
            nla_put(skb, NDA_CACHEINFO, sizeof(ci), &ci))
                goto nla_put_failure;

        if (neigh->protocol && nla_put_u8(skb, NDA_PROTOCOL, neigh->protocol))
                goto nla_put_failure;
        if (neigh_flags_ext && nla_put_u32(skb, NDA_FLAGS_EXT, neigh_flags_ext))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int neigh_fill_info(struct sk_buff *skb, struct neighbour *neigh,
                           u32 pid, u32 seq, int type, unsigned int flags)
        __releases(neigh->lock)
        __acquires(neigh->lock)
{
        int err;

        read_lock_bh(&neigh->lock);
        err = __neigh_fill_info(skb, neigh, pid, seq, type, flags);
        read_unlock_bh(&neigh->lock);

        return err;
}

static int pneigh_fill_info(struct sk_buff *skb, struct pneigh_entry *pn,
                            u32 pid, u32 seq, int type, unsigned int flags,
                            struct neigh_table *tbl)
{
        u32 neigh_flags, neigh_flags_ext;
        struct nlmsghdr *nlh;
        struct ndmsg *ndm;
        u8 protocol;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        neigh_flags = READ_ONCE(pn->flags);
        neigh_flags_ext = neigh_flags >> NTF_EXT_SHIFT;
        neigh_flags &= NTF_OLD_MASK;

        ndm = nlmsg_data(nlh);
        ndm->ndm_family  = tbl->family;
        ndm->ndm_pad1    = 0;
        ndm->ndm_pad2    = 0;
        ndm->ndm_flags   = neigh_flags | NTF_PROXY;
        ndm->ndm_type    = RTN_UNICAST;
        ndm->ndm_ifindex = pn->dev ? pn->dev->ifindex : 0;
        ndm->ndm_state   = NUD_NONE;

        if (nla_put(skb, NDA_DST, tbl->key_len, pn->key))
                goto nla_put_failure;

        protocol = READ_ONCE(pn->protocol);
        if (protocol && nla_put_u8(skb, NDA_PROTOCOL, protocol))
                goto nla_put_failure;
        if (neigh_flags_ext && nla_put_u32(skb, NDA_FLAGS_EXT, neigh_flags_ext))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static bool neigh_master_filtered(struct net_device *dev, int master_idx)
{
        struct net_device *master;

        if (!master_idx)
                return false;

        master = dev ? netdev_master_upper_dev_get_rcu(dev) : NULL;

        /* 0 is already used to denote NDA_MASTER wasn't passed, therefore need another
         * invalid value for ifindex to denote "no master".
         */
        if (master_idx == -1)
                return !!master;

        if (!master || master->ifindex != master_idx)
                return true;

        return false;
}

static bool neigh_ifindex_filtered(struct net_device *dev, int filter_idx)
{
        if (filter_idx && (!dev || dev->ifindex != filter_idx))
                return true;

        return false;
}

struct neigh_dump_filter {
        int master_idx;
        int dev_idx;
};

static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
                            struct netlink_callback *cb,
                            struct neigh_dump_filter *filter)
{
        struct net *net = sock_net(skb->sk);
        struct neighbour *n;
        int err = 0, h, s_h = cb->args[1];
        int idx, s_idx = idx = cb->args[2];
        struct neigh_hash_table *nht;
        unsigned int flags = NLM_F_MULTI;

        if (filter->dev_idx || filter->master_idx)
                flags |= NLM_F_DUMP_FILTERED;

        nht = rcu_dereference(tbl->nht);

        for (h = s_h; h < (1 << nht->hash_shift); h++) {
                if (h > s_h)
                        s_idx = 0;
                idx = 0;
                neigh_for_each_in_bucket_rcu(n, &nht->hash_heads[h]) {
                        if (idx < s_idx || !net_eq(dev_net(n->dev), net))
                                goto next;
                        if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
                            neigh_master_filtered(n->dev, filter->master_idx))
                                goto next;
                        err = neigh_fill_info(skb, n, NETLINK_CB(cb->skb).portid,
                                              cb->nlh->nlmsg_seq,
                                              RTM_NEWNEIGH, flags);
                        if (err < 0)
                                goto out;
next:
                        idx++;
                }
        }
out:
        cb->args[1] = h;
        cb->args[2] = idx;
        return err;
}

static int pneigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
                             struct netlink_callback *cb,
                             struct neigh_dump_filter *filter)
{
        struct pneigh_entry *n;
        struct net *net = sock_net(skb->sk);
        int err = 0, h, s_h = cb->args[3];
        int idx, s_idx = idx = cb->args[4];
        unsigned int flags = NLM_F_MULTI;

        if (filter->dev_idx || filter->master_idx)
                flags |= NLM_F_DUMP_FILTERED;

        for (h = s_h; h <= PNEIGH_HASHMASK; h++) {
                if (h > s_h)
                        s_idx = 0;
                for (n = rcu_dereference(tbl->phash_buckets[h]), idx = 0;
                     n;
                     n = rcu_dereference(n->next)) {
                        if (idx < s_idx || pneigh_net(n) != net)
                                goto next;
                        if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
                            neigh_master_filtered(n->dev, filter->master_idx))
                                goto next;
                        err = pneigh_fill_info(skb, n, NETLINK_CB(cb->skb).portid,
                                               cb->nlh->nlmsg_seq,
                                               RTM_NEWNEIGH, flags, tbl);
                        if (err < 0)
                                goto out;
                next:
                        idx++;
                }
        }

out:
        cb->args[3] = h;
        cb->args[4] = idx;
        return err;
}

static int neigh_valid_dump_req(const struct nlmsghdr *nlh,
                                bool strict_check,
                                struct neigh_dump_filter *filter,
                                struct netlink_ext_ack *extack)
{
        struct nlattr *tb[NDA_MAX + 1];
        int err, i;

        if (strict_check) {
                struct ndmsg *ndm;

                ndm = nlmsg_payload(nlh, sizeof(*ndm));
                if (!ndm) {
                        NL_SET_ERR_MSG(extack, "Invalid header for neighbor dump request");
                        return -EINVAL;
                }

                if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_ifindex ||
                    ndm->ndm_state || ndm->ndm_type) {
                        NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor dump request");
                        return -EINVAL;
                }

                if (ndm->ndm_flags & ~NTF_PROXY) {
                        NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor dump request");
                        return -EINVAL;
                }

                err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg),
                                                    tb, NDA_MAX, nda_policy,
                                                    extack);
        } else {
                err = nlmsg_parse_deprecated(nlh, sizeof(struct ndmsg), tb,
                                             NDA_MAX, nda_policy, extack);
        }
        if (err < 0)
                return err;

        for (i = 0; i <= NDA_MAX; ++i) {
                if (!tb[i])
                        continue;

                /* all new attributes should require strict_check */
                switch (i) {
                case NDA_IFINDEX:
                        filter->dev_idx = nla_get_u32(tb[i]);
                        break;
                case NDA_MASTER:
                        filter->master_idx = nla_get_u32(tb[i]);
                        break;
                default:
                        if (strict_check) {
                                NL_SET_ERR_MSG(extack, "Unsupported attribute in neighbor dump request");
                                return -EINVAL;
                        }
                }
        }

        return 0;
}

static int neigh_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct neigh_dump_filter filter = {};
        struct neigh_table *tbl;
        int t, family, s_t;
        int proxy = 0;
        int err;

        family = ((struct rtgenmsg *)nlmsg_data(nlh))->rtgen_family;

        /* check for full ndmsg structure presence, family member is
         * the same for both structures
         */
        if (nlmsg_len(nlh) >= sizeof(struct ndmsg) &&
            ((struct ndmsg *)nlmsg_data(nlh))->ndm_flags == NTF_PROXY)
                proxy = 1;

        err = neigh_valid_dump_req(nlh, cb->strict_check, &filter, cb->extack);
        if (err < 0 && cb->strict_check)
                return err;
        err = 0;

        s_t = cb->args[0];

        rcu_read_lock();
        for (t = 0; t < NEIGH_NR_TABLES; t++) {
                tbl = rcu_dereference(neigh_tables[t]);

                if (!tbl)
                        continue;
                if (t < s_t || (family && tbl->family != family))
                        continue;
                if (t > s_t)
                        memset(&cb->args[1], 0, sizeof(cb->args) -
                                                sizeof(cb->args[0]));
                if (proxy)
                        err = pneigh_dump_table(tbl, skb, cb, &filter);
                else
                        err = neigh_dump_table(tbl, skb, cb, &filter);
                if (err < 0)
                        break;
        }
        rcu_read_unlock();

        cb->args[0] = t;
        return err;
}

static struct ndmsg *neigh_valid_get_req(const struct nlmsghdr *nlh,
                                         struct nlattr **tb,
                                         struct netlink_ext_ack *extack)
{
        struct ndmsg *ndm;
        int err, i;

        ndm = nlmsg_payload(nlh, sizeof(*ndm));
        if (!ndm) {
                NL_SET_ERR_MSG(extack, "Invalid header for neighbor get request");
                return ERR_PTR(-EINVAL);
        }

        if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_state ||
            ndm->ndm_type) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor get request");
                return ERR_PTR(-EINVAL);
        }

        if (ndm->ndm_flags & ~NTF_PROXY) {
                NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor get request");
                return ERR_PTR(-EINVAL);
        }

        if (!(ndm->ndm_flags & NTF_PROXY) && !ndm->ndm_ifindex) {
                NL_SET_ERR_MSG(extack, "No device specified");
                return ERR_PTR(-EINVAL);
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg), tb,
                                            NDA_MAX, nda_policy, extack);
        if (err < 0)
                return ERR_PTR(err);

        for (i = 0; i <= NDA_MAX; ++i) {
                switch (i) {
                case NDA_DST:
                        if (!tb[i]) {
                                NL_SET_ERR_ATTR_MISS(extack, NULL, NDA_DST);
                                return ERR_PTR(-EINVAL);
                        }
                        break;
                default:
                        if (!tb[i])
                                continue;

                        NL_SET_ERR_MSG(extack, "Unsupported attribute in neighbor get request");
                        return ERR_PTR(-EINVAL);
                }
        }

        return ndm;
}

static inline size_t neigh_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ndmsg))
               + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */
               + nla_total_size(MAX_ADDR_LEN) /* NDA_LLADDR */
               + nla_total_size(sizeof(struct nda_cacheinfo))
               + nla_total_size(4)  /* NDA_PROBES */
               + nla_total_size(4)  /* NDA_FLAGS_EXT */
               + nla_total_size(1); /* NDA_PROTOCOL */
}

static inline size_t pneigh_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ndmsg))
               + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */
               + nla_total_size(4)  /* NDA_FLAGS_EXT */
               + nla_total_size(1); /* NDA_PROTOCOL */
}

static int neigh_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                     struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        u32 pid = NETLINK_CB(in_skb).portid;
        struct nlattr *tb[NDA_MAX + 1];
        struct net_device *dev = NULL;
        u32 seq = nlh->nlmsg_seq;
        struct neigh_table *tbl;
        struct neighbour *neigh;
        struct sk_buff *skb;
        struct ndmsg *ndm;
        void *dst;
        int err;

        ndm = neigh_valid_get_req(nlh, tb, extack);
        if (IS_ERR(ndm))
                return PTR_ERR(ndm);

        if (ndm->ndm_flags & NTF_PROXY)
                skb = nlmsg_new(neigh_nlmsg_size(), GFP_KERNEL);
        else
                skb = nlmsg_new(pneigh_nlmsg_size(), GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        rcu_read_lock();

        tbl = neigh_find_table(ndm->ndm_family);
        if (!tbl) {
                NL_SET_ERR_MSG(extack, "Unsupported family in header for neighbor get request");
                err = -EAFNOSUPPORT;
                goto err_unlock;
        }

        if (nla_len(tb[NDA_DST]) != (int)tbl->key_len) {
                NL_SET_ERR_MSG(extack, "Invalid network address in neighbor get request");
                err = -EINVAL;
                goto err_unlock;
        }

        dst = nla_data(tb[NDA_DST]);

        if (ndm->ndm_ifindex) {
                dev = dev_get_by_index_rcu(net, ndm->ndm_ifindex);
                if (!dev) {
                        NL_SET_ERR_MSG(extack, "Unknown device ifindex");
                        err = -ENODEV;
                        goto err_unlock;
                }
        }

        if (ndm->ndm_flags & NTF_PROXY) {
                struct pneigh_entry *pn;

                pn = pneigh_lookup(tbl, net, dst, dev);
                if (!pn) {
                        NL_SET_ERR_MSG(extack, "Proxy neighbour entry not found");
                        err = -ENOENT;
                        goto err_unlock;
                }

                err = pneigh_fill_info(skb, pn, pid, seq, RTM_NEWNEIGH, 0, tbl);
                if (err)
                        goto err_unlock;
        } else {
                neigh = neigh_lookup(tbl, dst, dev);
                if (!neigh) {
                        NL_SET_ERR_MSG(extack, "Neighbour entry not found");
                        err = -ENOENT;
                        goto err_unlock;
                }

                err = neigh_fill_info(skb, neigh, pid, seq, RTM_NEWNEIGH, 0);
                neigh_release(neigh);
                if (err)
                        goto err_unlock;
        }

        rcu_read_unlock();

        return rtnl_unicast(skb, net, pid);
err_unlock:
        rcu_read_unlock();
        kfree_skb(skb);
        return err;
}

void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void *), void *cookie)
{
        int chain;
        struct neigh_hash_table *nht;

        rcu_read_lock();
        nht = rcu_dereference(tbl->nht);

        spin_lock_bh(&tbl->lock); /* avoid resizes */
        for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
                struct neighbour *n;

                neigh_for_each_in_bucket(n, &nht->hash_heads[chain])
                        cb(n, cookie);
        }
        spin_unlock_bh(&tbl->lock);
        rcu_read_unlock();
}
EXPORT_SYMBOL(neigh_for_each);

/* The tbl->lock must be held as a writer and BH disabled. */
void __neigh_for_each_release(struct neigh_table *tbl,
                              int (*cb)(struct neighbour *))
{
        struct neigh_hash_table *nht;
        int chain;

        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));
        for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
                struct hlist_node *tmp;
                struct neighbour *n;

                neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[chain]) {
                        int release;

                        write_lock(&n->lock);
                        release = cb(n);
                        if (release) {
                                hlist_del_rcu(&n->hash);
                                hlist_del_rcu(&n->dev_list);
                                neigh_mark_dead(n);
                        }
                        write_unlock(&n->lock);
                        if (release)
                                neigh_cleanup_and_release(n);
                }
        }
}
EXPORT_SYMBOL(__neigh_for_each_release);

int neigh_xmit(int index, struct net_device *dev,
               const void *addr, struct sk_buff *skb)
{
        int err = -EAFNOSUPPORT;

        if (likely(index < NEIGH_NR_TABLES)) {
                struct neigh_table *tbl;
                struct neighbour *neigh;

                rcu_read_lock();
                tbl = rcu_dereference(neigh_tables[index]);
                if (!tbl) {
                        rcu_read_unlock();
                        goto out_kfree_skb;
                }
                if (index == NEIGH_ARP_TABLE) {
                        u32 key = *((u32 *)addr);

                        neigh = __ipv4_neigh_lookup_noref(dev, key);
                } else {
                        neigh = __neigh_lookup_noref(tbl, addr, dev);
                }
                if (!neigh)
                        neigh = __neigh_create(tbl, addr, dev, false);
                err = PTR_ERR(neigh);
                if (IS_ERR(neigh)) {
                        rcu_read_unlock();
                        goto out_kfree_skb;
                }
                err = READ_ONCE(neigh->output)(neigh, skb);
                rcu_read_unlock();
        }
        else if (index == NEIGH_LINK_TABLE) {
                err = dev_hard_header(skb, dev, ntohs(skb->protocol),
                                      addr, NULL, skb->len);
                if (err < 0)
                        goto out_kfree_skb;
                err = dev_queue_xmit(skb);
        }
        return err;
out_kfree_skb:
        kfree_skb(skb);
        return err;
}
EXPORT_SYMBOL(neigh_xmit);

#ifdef CONFIG_PROC_FS

static struct neighbour *neigh_get_valid(struct seq_file *seq,
                                         struct neighbour *n,
                                         loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        struct net *net = seq_file_net(seq);

        if (!net_eq(dev_net(n->dev), net))
                return NULL;

        if (state->neigh_sub_iter) {
                loff_t fakep = 0;
                void *v;

                v = state->neigh_sub_iter(state, n, pos ? pos : &fakep);
                if (!v)
                        return NULL;
                if (pos)
                        return v;
        }

        if (!(state->flags & NEIGH_SEQ_SKIP_NOARP))
                return n;

        if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
                return n;

        return NULL;
}

static struct neighbour *neigh_get_first(struct seq_file *seq)
{
        struct neigh_seq_state *state = seq->private;
        struct neigh_hash_table *nht = state->nht;
        struct neighbour *n, *tmp;

        state->flags &= ~NEIGH_SEQ_IS_PNEIGH;

        while (++state->bucket < (1 << nht->hash_shift)) {
                neigh_for_each_in_bucket(n, &nht->hash_heads[state->bucket]) {
                        tmp = neigh_get_valid(seq, n, NULL);
                        if (tmp)
                                return tmp;
                }
        }

        return NULL;
}

static struct neighbour *neigh_get_next(struct seq_file *seq,
                                        struct neighbour *n,
                                        loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        struct neighbour *tmp;

        if (state->neigh_sub_iter) {
                void *v = state->neigh_sub_iter(state, n, pos);

                if (v)
                        return n;
        }

        hlist_for_each_entry_continue(n, hash) {
                tmp = neigh_get_valid(seq, n, pos);
                if (tmp) {
                        n = tmp;
                        goto out;
                }
        }

        n = neigh_get_first(seq);
out:
        if (n && pos)
                --(*pos);

        return n;
}

static struct neighbour *neigh_get_idx(struct seq_file *seq, loff_t *pos)
{
        struct neighbour *n = neigh_get_first(seq);

        if (n) {
                --(*pos);
                while (*pos) {
                        n = neigh_get_next(seq, n, pos);
                        if (!n)
                                break;
                }
        }
        return *pos ? NULL : n;
}

static struct pneigh_entry *pneigh_get_first(struct seq_file *seq)
{
        struct neigh_seq_state *state = seq->private;
        struct net *net = seq_file_net(seq);
        struct neigh_table *tbl = state->tbl;
        struct pneigh_entry *pn = NULL;
        int bucket;

        state->flags |= NEIGH_SEQ_IS_PNEIGH;
        for (bucket = 0; bucket <= PNEIGH_HASHMASK; bucket++) {
                pn = rcu_dereference(tbl->phash_buckets[bucket]);

                while (pn && !net_eq(pneigh_net(pn), net))
                        pn = rcu_dereference(pn->next);
                if (pn)
                        break;
        }
        state->bucket = bucket;

        return pn;
}

static struct pneigh_entry *pneigh_get_next(struct seq_file *seq,
                                            struct pneigh_entry *pn,
                                            loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        struct net *net = seq_file_net(seq);
        struct neigh_table *tbl = state->tbl;

        do {
                pn = rcu_dereference(pn->next);
        } while (pn && !net_eq(pneigh_net(pn), net));

        while (!pn) {
                if (++state->bucket > PNEIGH_HASHMASK)
                        break;

                pn = rcu_dereference(tbl->phash_buckets[state->bucket]);

                while (pn && !net_eq(pneigh_net(pn), net))
                        pn = rcu_dereference(pn->next);
                if (pn)
                        break;
        }

        if (pn && pos)
                --(*pos);

        return pn;
}

static struct pneigh_entry *pneigh_get_idx(struct seq_file *seq, loff_t *pos)
{
        struct pneigh_entry *pn = pneigh_get_first(seq);

        if (pn) {
                --(*pos);
                while (*pos) {
                        pn = pneigh_get_next(seq, pn, pos);
                        if (!pn)
                                break;
                }
        }
        return *pos ? NULL : pn;
}

static void *neigh_get_idx_any(struct seq_file *seq, loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        void *rc;
        loff_t idxpos = *pos;

        rc = neigh_get_idx(seq, &idxpos);
        if (!rc && !(state->flags & NEIGH_SEQ_NEIGH_ONLY))
                rc = pneigh_get_idx(seq, &idxpos);

        return rc;
}

void *neigh_seq_start(struct seq_file *seq, loff_t *pos, struct neigh_table *tbl, unsigned int neigh_seq_flags)
        __acquires(tbl->lock)
        __acquires(rcu)
{
        struct neigh_seq_state *state = seq->private;

        state->tbl = tbl;
        state->bucket = -1;
        state->flags = (neigh_seq_flags & ~NEIGH_SEQ_IS_PNEIGH);

        rcu_read_lock();
        state->nht = rcu_dereference(tbl->nht);
        spin_lock_bh(&tbl->lock);

        return *pos ? neigh_get_idx_any(seq, pos) : SEQ_START_TOKEN;
}
EXPORT_SYMBOL(neigh_seq_start);

void *neigh_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct neigh_seq_state *state;
        void *rc;

        if (v == SEQ_START_TOKEN) {
                rc = neigh_get_first(seq);
                goto out;
        }

        state = seq->private;
        if (!(state->flags & NEIGH_SEQ_IS_PNEIGH)) {
                rc = neigh_get_next(seq, v, NULL);
                if (rc)
                        goto out;
                if (!(state->flags & NEIGH_SEQ_NEIGH_ONLY))
                        rc = pneigh_get_first(seq);
        } else {
                BUG_ON(state->flags & NEIGH_SEQ_NEIGH_ONLY);
                rc = pneigh_get_next(seq, v, NULL);
        }
out:
        ++(*pos);
        return rc;
}
EXPORT_SYMBOL(neigh_seq_next);

void neigh_seq_stop(struct seq_file *seq, void *v)
        __releases(tbl->lock)
        __releases(rcu)
{
        struct neigh_seq_state *state = seq->private;
        struct neigh_table *tbl = state->tbl;

        spin_unlock_bh(&tbl->lock);
        rcu_read_unlock();
}
EXPORT_SYMBOL(neigh_seq_stop);

/* statistics via seq_file */

static void *neigh_stat_seq_start(struct seq_file *seq, loff_t *pos)
{
        struct neigh_table *tbl = pde_data(file_inode(seq->file));
        int cpu;

        if (*pos == 0)
                return SEQ_START_TOKEN;

        for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
                if (!cpu_possible(cpu))
                        continue;
                *pos = cpu+1;
                return per_cpu_ptr(tbl->stats, cpu);
        }
        return NULL;
}

static void *neigh_stat_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct neigh_table *tbl = pde_data(file_inode(seq->file));
        int cpu;

        for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
                if (!cpu_possible(cpu))
                        continue;
                *pos = cpu+1;
                return per_cpu_ptr(tbl->stats, cpu);
        }
        (*pos)++;
        return NULL;
}

static void neigh_stat_seq_stop(struct seq_file *seq, void *v)
{

}

static int neigh_stat_seq_show(struct seq_file *seq, void *v)
{
        struct neigh_table *tbl = pde_data(file_inode(seq->file));
        struct neigh_statistics *st = v;

        if (v == SEQ_START_TOKEN) {
                seq_puts(seq, "entries  allocs   destroys hash_grows lookups  hits     res_failed rcv_probes_mcast rcv_probes_ucast periodic_gc_runs forced_gc_runs unresolved_discards table_fulls\n");
                return 0;
        }

        seq_printf(seq, "%08x %08lx %08lx %08lx   %08lx %08lx %08lx   "
                        "%08lx         %08lx         %08lx         "
                        "%08lx       %08lx            %08lx\n",
                   atomic_read(&tbl->entries),

                   st->allocs,
                   st->destroys,
                   st->hash_grows,

                   st->lookups,
                   st->hits,

                   st->res_failed,

                   st->rcv_probes_mcast,
                   st->rcv_probes_ucast,

                   st->periodic_gc_runs,
                   st->forced_gc_runs,
                   st->unres_discards,
                   st->table_fulls
                   );

        return 0;
}

static const struct seq_operations neigh_stat_seq_ops = {
        .start  = neigh_stat_seq_start,
        .next   = neigh_stat_seq_next,
        .stop   = neigh_stat_seq_stop,
        .show   = neigh_stat_seq_show,
};
#endif /* CONFIG_PROC_FS */

static void __neigh_notify(struct neighbour *n, int type, int flags,
                           u32 pid)
{
        struct sk_buff *skb;
        int err = -ENOBUFS;
        struct net *net;

        rcu_read_lock();
        net = dev_net_rcu(n->dev);
        skb = nlmsg_new(neigh_nlmsg_size(), GFP_ATOMIC);
        if (skb == NULL)
                goto errout;

        err = __neigh_fill_info(skb, n, pid, 0, type, flags);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in neigh_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC);
        goto out;
errout:
        rtnl_set_sk_err(net, RTNLGRP_NEIGH, err);
out:
        rcu_read_unlock();
}

static void neigh_notify(struct neighbour *neigh, int type, int flags, u32 pid)
{
        read_lock_bh(&neigh->lock);
        __neigh_notify(neigh, type, flags, pid);
        read_unlock_bh(&neigh->lock);
}

void neigh_app_ns(struct neighbour *n)
{
        neigh_notify(n, RTM_GETNEIGH, NLM_F_REQUEST, 0);
}
EXPORT_SYMBOL(neigh_app_ns);

#ifdef CONFIG_SYSCTL
static int unres_qlen_max = INT_MAX / SKB_TRUESIZE(ETH_FRAME_LEN);

static int proc_unres_qlen(const struct ctl_table *ctl, int write,
                           void *buffer, size_t *lenp, loff_t *ppos)
{
        int size, ret;
        struct ctl_table tmp = *ctl;

        tmp.extra1 = SYSCTL_ZERO;
        tmp.extra2 = &unres_qlen_max;
        tmp.data = &size;

        size = *(int *)ctl->data / SKB_TRUESIZE(ETH_FRAME_LEN);
        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && !ret)
                *(int *)ctl->data = size * SKB_TRUESIZE(ETH_FRAME_LEN);
        return ret;
}

static void neigh_copy_dflt_parms(struct net *net, struct neigh_parms *p,
                                  int index)
{
        struct net_device *dev;
        int family = neigh_parms_family(p);

        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                struct neigh_parms *dst_p =
                                neigh_get_dev_parms_rcu(dev, family);

                if (dst_p && !test_bit(index, dst_p->data_state))
                        dst_p->data[index] = p->data[index];
        }
        rcu_read_unlock();
}

static void neigh_proc_update(const struct ctl_table *ctl, int write)
{
        struct net_device *dev = ctl->extra1;
        struct neigh_parms *p = ctl->extra2;
        struct net *net = neigh_parms_net(p);
        int index = (int *) ctl->data - p->data;

        if (!write)
                return;

        set_bit(index, p->data_state);
        if (index == NEIGH_VAR_DELAY_PROBE_TIME)
                call_netevent_notifiers(NETEVENT_DELAY_PROBE_TIME_UPDATE, p);
        if (!dev) /* NULL dev means this is default value */
                neigh_copy_dflt_parms(net, p, index);
}

static int neigh_proc_dointvec_zero_intmax(const struct ctl_table *ctl, int write,
                                           void *buffer, size_t *lenp,
                                           loff_t *ppos)
{
        struct ctl_table tmp = *ctl;
        int ret;

        tmp.extra1 = SYSCTL_ZERO;
        tmp.extra2 = SYSCTL_INT_MAX;

        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
        neigh_proc_update(ctl, write);
        return ret;
}

static int neigh_proc_dointvec_ms_jiffies_positive(const struct ctl_table *ctl, int write,
                                                   void *buffer, size_t *lenp, loff_t *ppos)
{
        struct ctl_table tmp = *ctl;
        int ret;

        int min = msecs_to_jiffies(1);

        tmp.extra1 = &min;
        tmp.extra2 = NULL;

        ret = proc_dointvec_ms_jiffies_minmax(&tmp, write, buffer, lenp, ppos);
        neigh_proc_update(ctl, write);
        return ret;
}

int neigh_proc_dointvec(const struct ctl_table *ctl, int write, void *buffer,
                        size_t *lenp, loff_t *ppos)
{
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}
EXPORT_SYMBOL(neigh_proc_dointvec);

int neigh_proc_dointvec_jiffies(const struct ctl_table *ctl, int write, void *buffer,
                                size_t *lenp, loff_t *ppos)
{
        int ret = proc_dointvec_jiffies(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}
EXPORT_SYMBOL(neigh_proc_dointvec_jiffies);

static int neigh_proc_dointvec_userhz_jiffies(const struct ctl_table *ctl, int write,
                                              void *buffer, size_t *lenp,
                                              loff_t *ppos)
{
        int ret = proc_dointvec_userhz_jiffies(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}

int neigh_proc_dointvec_ms_jiffies(const struct ctl_table *ctl, int write,
                                   void *buffer, size_t *lenp, loff_t *ppos)
{
        int ret = proc_dointvec_ms_jiffies(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}
EXPORT_SYMBOL(neigh_proc_dointvec_ms_jiffies);

static int neigh_proc_dointvec_unres_qlen(const struct ctl_table *ctl, int write,
                                          void *buffer, size_t *lenp,
                                          loff_t *ppos)
{
        int ret = proc_unres_qlen(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}

static int neigh_proc_base_reachable_time(const struct ctl_table *ctl, int write,
                                          void *buffer, size_t *lenp,
                                          loff_t *ppos)
{
        struct neigh_parms *p = ctl->extra2;
        int ret;

        if (strcmp(ctl->procname, "base_reachable_time") == 0)
                ret = neigh_proc_dointvec_jiffies(ctl, write, buffer, lenp, ppos);
        else if (strcmp(ctl->procname, "base_reachable_time_ms") == 0)
                ret = neigh_proc_dointvec_ms_jiffies(ctl, write, buffer, lenp, ppos);
        else
                ret = -1;

        if (write && ret == 0) {
                /* update reachable_time as well, otherwise, the change will
                 * only be effective after the next time neigh_periodic_work
                 * decides to recompute it
                 */
                neigh_set_reach_time(p);
        }
        return ret;
}

#define NEIGH_PARMS_DATA_OFFSET(index)  \
        (&((struct neigh_parms *) 0)->data[index])

#define NEIGH_SYSCTL_ENTRY(attr, data_attr, name, mval, proc) \
        [NEIGH_VAR_ ## attr] = { \
                .procname       = name, \
                .data           = NEIGH_PARMS_DATA_OFFSET(NEIGH_VAR_ ## data_attr), \
                .maxlen         = sizeof(int), \
                .mode           = mval, \
                .proc_handler   = proc, \
        }

#define NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_zero_intmax)

#define NEIGH_SYSCTL_JIFFIES_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_jiffies)

#define NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_userhz_jiffies)

#define NEIGH_SYSCTL_MS_JIFFIES_POSITIVE_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_ms_jiffies_positive)

#define NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(attr, data_attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, data_attr, name, 0644, neigh_proc_dointvec_ms_jiffies)

#define NEIGH_SYSCTL_UNRES_QLEN_REUSED_ENTRY(attr, data_attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, data_attr, name, 0644, neigh_proc_dointvec_unres_qlen)

static struct neigh_sysctl_table {
        struct ctl_table_header *sysctl_header;
        struct ctl_table neigh_vars[NEIGH_VAR_MAX];
} neigh_sysctl_template __read_mostly = {
        .neigh_vars = {
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(MCAST_PROBES, "mcast_solicit"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(UCAST_PROBES, "ucast_solicit"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(APP_PROBES, "app_solicit"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(MCAST_REPROBES, "mcast_resolicit"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(RETRANS_TIME, "retrans_time"),
                NEIGH_SYSCTL_JIFFIES_ENTRY(BASE_REACHABLE_TIME, "base_reachable_time"),
                NEIGH_SYSCTL_JIFFIES_ENTRY(DELAY_PROBE_TIME, "delay_first_probe_time"),
                NEIGH_SYSCTL_MS_JIFFIES_POSITIVE_ENTRY(INTERVAL_PROBE_TIME_MS,
                                                       "interval_probe_time_ms"),
                NEIGH_SYSCTL_JIFFIES_ENTRY(GC_STALETIME, "gc_stale_time"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(QUEUE_LEN_BYTES, "unres_qlen_bytes"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(PROXY_QLEN, "proxy_qlen"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(ANYCAST_DELAY, "anycast_delay"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(PROXY_DELAY, "proxy_delay"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(LOCKTIME, "locktime"),
                NEIGH_SYSCTL_UNRES_QLEN_REUSED_ENTRY(QUEUE_LEN, QUEUE_LEN_BYTES, "unres_qlen"),
                NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(RETRANS_TIME_MS, RETRANS_TIME, "retrans_time_ms"),
                NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(BASE_REACHABLE_TIME_MS, BASE_REACHABLE_TIME, "base_reachable_time_ms"),
                [NEIGH_VAR_GC_INTERVAL] = {
                        .procname       = "gc_interval",
                        .maxlen         = sizeof(int),
                        .mode           = 0644,
                        .proc_handler   = proc_dointvec_jiffies,
                },
                [NEIGH_VAR_GC_THRESH1] = {
                        .procname       = "gc_thresh1",
                        .maxlen         = sizeof(int),
                        .mode           = 0644,
                        .extra1         = SYSCTL_ZERO,
                        .extra2         = SYSCTL_INT_MAX,
                        .proc_handler   = proc_dointvec_minmax,
                },
                [NEIGH_VAR_GC_THRESH2] = {
                        .procname       = "gc_thresh2",
                        .maxlen         = sizeof(int),
                        .mode           = 0644,
                        .extra1         = SYSCTL_ZERO,
                        .extra2         = SYSCTL_INT_MAX,
                        .proc_handler   = proc_dointvec_minmax,
                },
                [NEIGH_VAR_GC_THRESH3] = {
                        .procname       = "gc_thresh3",
                        .maxlen         = sizeof(int),
                        .mode           = 0644,
                        .extra1         = SYSCTL_ZERO,
                        .extra2         = SYSCTL_INT_MAX,
                        .proc_handler   = proc_dointvec_minmax,
                },
        },
};

int neigh_sysctl_register(struct net_device *dev, struct neigh_parms *p,
                          proc_handler *handler)
{
        int i;
        struct neigh_sysctl_table *t;
        const char *dev_name_source;
        char neigh_path[ sizeof("net//neigh/") + IFNAMSIZ + IFNAMSIZ ];
        char *p_name;
        size_t neigh_vars_size;

        t = kmemdup(&neigh_sysctl_template, sizeof(*t), GFP_KERNEL_ACCOUNT);
        if (!t)
                goto err;

        for (i = 0; i < NEIGH_VAR_GC_INTERVAL; i++) {
                t->neigh_vars[i].data += (long) p;
                t->neigh_vars[i].extra1 = dev;
                t->neigh_vars[i].extra2 = p;
        }

        neigh_vars_size = ARRAY_SIZE(t->neigh_vars);
        if (dev) {
                dev_name_source = dev->name;
                /* Terminate the table early */
                neigh_vars_size = NEIGH_VAR_BASE_REACHABLE_TIME_MS + 1;
        } else {
                struct neigh_table *tbl = p->tbl;
                dev_name_source = "default";
                t->neigh_vars[NEIGH_VAR_GC_INTERVAL].data = &tbl->gc_interval;
                t->neigh_vars[NEIGH_VAR_GC_THRESH1].data = &tbl->gc_thresh1;
                t->neigh_vars[NEIGH_VAR_GC_THRESH2].data = &tbl->gc_thresh2;
                t->neigh_vars[NEIGH_VAR_GC_THRESH3].data = &tbl->gc_thresh3;
        }

        if (handler) {
                /* RetransTime */
                t->neigh_vars[NEIGH_VAR_RETRANS_TIME].proc_handler = handler;
                /* ReachableTime */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].proc_handler = handler;
                /* RetransTime (in milliseconds)*/
                t->neigh_vars[NEIGH_VAR_RETRANS_TIME_MS].proc_handler = handler;
                /* ReachableTime (in milliseconds) */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].proc_handler = handler;
        } else {
                /* Those handlers will update p->reachable_time after
                 * base_reachable_time(_ms) is set to ensure the new timer starts being
                 * applied after the next neighbour update instead of waiting for
                 * neigh_periodic_work to update its value (can be multiple minutes)
                 * So any handler that replaces them should do this as well
                 */
                /* ReachableTime */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].proc_handler =
                        neigh_proc_base_reachable_time;
                /* ReachableTime (in milliseconds) */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].proc_handler =
                        neigh_proc_base_reachable_time;
        }

        switch (neigh_parms_family(p)) {
        case AF_INET:
              p_name = "ipv4";
              break;
        case AF_INET6:
              p_name = "ipv6";
              break;
        default:
              BUG();
        }

        snprintf(neigh_path, sizeof(neigh_path), "net/%s/neigh/%s",
                p_name, dev_name_source);
        t->sysctl_header = register_net_sysctl_sz(neigh_parms_net(p),
                                                  neigh_path, t->neigh_vars,
                                                  neigh_vars_size);
        if (!t->sysctl_header)
                goto free;

        p->sysctl_table = t;
        return 0;

free:
        kfree(t);
err:
        return -ENOBUFS;
}
EXPORT_SYMBOL(neigh_sysctl_register);

void neigh_sysctl_unregister(struct neigh_parms *p)
{
        if (p->sysctl_table) {
                struct neigh_sysctl_table *t = p->sysctl_table;
                p->sysctl_table = NULL;
                unregister_net_sysctl_table(t->sysctl_header);
                kfree(t);
        }
}
EXPORT_SYMBOL(neigh_sysctl_unregister);

#endif  /* CONFIG_SYSCTL */

static const struct rtnl_msg_handler neigh_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWNEIGH, .doit = neigh_add},
        {.msgtype = RTM_DELNEIGH, .doit = neigh_delete},
        {.msgtype = RTM_GETNEIGH, .doit = neigh_get, .dumpit = neigh_dump_info,
         .flags = RTNL_FLAG_DOIT_UNLOCKED | RTNL_FLAG_DUMP_UNLOCKED},
        {.msgtype = RTM_GETNEIGHTBL, .dumpit = neightbl_dump_info,
         .flags = RTNL_FLAG_DUMP_UNLOCKED},
        {.msgtype = RTM_SETNEIGHTBL, .doit = neightbl_set,
         .flags = RTNL_FLAG_DOIT_UNLOCKED},
};

static int __init neigh_init(void)
{
        rtnl_register_many(neigh_rtnl_msg_handlers);
        return 0;
}

subsys_initcall(neigh_init);