/*
 * Copyright (C) 2017 Netronome Systems, Inc.
 *
 * This software is licensed under the GNU General License Version 2,
 * June 1991 as shown in the file COPYING in the top-level directory of this
 * source tree.
 *
 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
 */

#include <linux/debugfs.h>
#include <linux/etherdevice.h>
#include <linux/ethtool_netlink.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/slab.h>
#include <net/netdev_queues.h>
#include <net/netdev_rx_queue.h>
#include <net/page_pool/helpers.h>
#include <net/netlink.h>
#include <net/net_shaper.h>
#include <net/netdev_lock.h>
#include <net/pkt_cls.h>
#include <net/rtnetlink.h>
#include <net/udp_tunnel.h>
#include <net/busy_poll.h>

#include "netdevsim.h"

MODULE_IMPORT_NS("NETDEV_INTERNAL");

#define NSIM_RING_SIZE          256

static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct net_device *peer_dev;
        struct netdevsim *peer_ns;
        struct netdev_queue *txq;
        u16 idx;

        idx = rq->napi.index;
        rcu_read_lock();
        peer_ns = rcu_dereference(ns->peer);
        if (!peer_ns)
                goto out;

        /* TX device */
        peer_dev = peer_ns->netdev;
        if (dev->real_num_tx_queues != peer_dev->num_rx_queues)
                goto out;

        txq = netdev_get_tx_queue(peer_dev, idx);
        if (!netif_tx_queue_stopped(txq))
                goto out;

        netif_tx_wake_queue(txq);
out:
        rcu_read_unlock();
}

static void nsim_stop_tx_queue(struct net_device *tx_dev,
                               struct net_device *rx_dev,
                               struct nsim_rq *rq,
                               u16 idx)
{
        /* If different queues size, do not stop, since it is not
         * easy to find which TX queue is mapped here
         */
        if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues)
                return;

        /* rq is the queue on the receive side */
        netif_subqueue_try_stop(tx_dev, idx,
                                NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue),
                                NSIM_RING_SIZE / 2);
}

static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev,
                        struct nsim_rq *rq, struct sk_buff *skb)
{
        if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) {
                dev_kfree_skb_any(skb);
                return NET_RX_DROP;
        }

        skb_queue_tail(&rq->skb_queue, skb);

        /* Stop the peer TX queue avoiding dropping packets later */
        if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE)
                nsim_stop_tx_queue(tx_dev, rx_dev, rq,
                                   skb_get_queue_mapping(skb));

        return NET_RX_SUCCESS;
}

static int nsim_forward_skb(struct net_device *tx_dev,
                            struct net_device *rx_dev,
                            struct sk_buff *skb,
                            struct nsim_rq *rq,
                            struct skb_ext *psp_ext)
{
        int ret;

        ret = __dev_forward_skb(rx_dev, skb);
        if (ret) {
                if (psp_ext)
                        __skb_ext_put(psp_ext);
                return ret;
        }

        nsim_psp_handle_ext(skb, psp_ext);

        return nsim_napi_rx(tx_dev, rx_dev, rq, skb);
}

static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct skb_ext *psp_ext = NULL;
        struct net_device *peer_dev;
        unsigned int len = skb->len;
        struct netdevsim *peer_ns;
        struct netdev_config *cfg;
        struct nsim_rq *rq;
        int rxq;
        int dr;

        rcu_read_lock();
        if (!nsim_ipsec_tx(ns, skb))
                goto out_drop_any;

        /* Check if loopback mode is enabled */
        if (dev->features & NETIF_F_LOOPBACK) {
                peer_ns = ns;
                peer_dev = dev;
        } else {
                peer_ns = rcu_dereference(ns->peer);
                if (!peer_ns)
                        goto out_drop_any;
                peer_dev = peer_ns->netdev;
        }

        dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext);
        if (dr)
                goto out_drop_free;

        rxq = skb_get_queue_mapping(skb);
        if (rxq >= peer_dev->num_rx_queues)
                rxq = rxq % peer_dev->num_rx_queues;
        rq = peer_ns->rq[rxq];

        cfg = peer_dev->cfg;
        if (skb_is_nonlinear(skb) &&
            (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED ||
             (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
              cfg->hds_thresh > len)))
                skb_linearize(skb);

        skb_tx_timestamp(skb);
        if (unlikely(nsim_forward_skb(dev, peer_dev,
                                      skb, rq, psp_ext) == NET_RX_DROP))
                goto out_drop_cnt;

        if (!hrtimer_active(&rq->napi_timer))
                hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL);

        rcu_read_unlock();
        dev_dstats_tx_add(dev, len);
        return NETDEV_TX_OK;

out_drop_any:
        dr = SKB_DROP_REASON_NOT_SPECIFIED;
out_drop_free:
        kfree_skb_reason(skb, dr);
out_drop_cnt:
        rcu_read_unlock();
        dev_dstats_tx_dropped(dev);
        return NETDEV_TX_OK;
}

static void nsim_set_rx_mode(struct net_device *dev)
{
}

static int nsim_change_mtu(struct net_device *dev, int new_mtu)
{
        struct netdevsim *ns = netdev_priv(dev);

        if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags &&
            new_mtu > NSIM_XDP_MAX_MTU)
                return -EBUSY;

        WRITE_ONCE(dev->mtu, new_mtu);

        return 0;
}

static int
nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
{
        return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);
}

static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        /* Only refuse multicast addresses, zero address can mean unset/any. */
        if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac))
                return -EINVAL;
        memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN);

        return 0;
}

static int nsim_set_vf_vlan(struct net_device *dev, int vf,
                            u16 vlan, u8 qos, __be16 vlan_proto)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7)
                return -EINVAL;

        nsim_dev->vfconfigs[vf].vlan = vlan;
        nsim_dev->vfconfigs[vf].qos = qos;
        nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto;

        return 0;
}

static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) {
                pr_err("Not supported in switchdev mode. Please use devlink API.\n");
                return -EOPNOTSUPP;
        }

        if (vf >= nsim_dev_get_vfs(nsim_dev))
                return -EINVAL;

        nsim_dev->vfconfigs[vf].min_tx_rate = min;
        nsim_dev->vfconfigs[vf].max_tx_rate = max;

        return 0;
}

static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        if (vf >= nsim_dev_get_vfs(nsim_dev))
                return -EINVAL;
        nsim_dev->vfconfigs[vf].spoofchk_enabled = val;

        return 0;
}

static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        if (vf >= nsim_dev_get_vfs(nsim_dev))
                return -EINVAL;
        nsim_dev->vfconfigs[vf].rss_query_enabled = val;

        return 0;
}

static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        if (vf >= nsim_dev_get_vfs(nsim_dev))
                return -EINVAL;
        nsim_dev->vfconfigs[vf].trusted = val;

        return 0;
}

static int
nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        if (vf >= nsim_dev_get_vfs(nsim_dev))
                return -EINVAL;

        ivi->vf = vf;
        ivi->linkstate = nsim_dev->vfconfigs[vf].link_state;
        ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate;
        ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate;
        ivi->vlan = nsim_dev->vfconfigs[vf].vlan;
        ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto;
        ivi->qos = nsim_dev->vfconfigs[vf].qos;
        memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN);
        ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled;
        ivi->trusted = nsim_dev->vfconfigs[vf].trusted;
        ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled;

        return 0;
}

static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_dev *nsim_dev = ns->nsim_dev;

        if (vf >= nsim_dev_get_vfs(nsim_dev))
                return -EINVAL;

        switch (state) {
        case IFLA_VF_LINK_STATE_AUTO:
        case IFLA_VF_LINK_STATE_ENABLE:
        case IFLA_VF_LINK_STATE_DISABLE:
                break;
        default:
                return -EINVAL;
        }

        nsim_dev->vfconfigs[vf].link_state = state;

        return 0;
}

static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats)
{
        stats->window_drops = 0;
        stats->tx_overruns = 0;
}

static int nsim_setup_tc_taprio(struct net_device *dev,
                                struct tc_taprio_qopt_offload *offload)
{
        int err = 0;

        switch (offload->cmd) {
        case TAPRIO_CMD_REPLACE:
        case TAPRIO_CMD_DESTROY:
                break;
        case TAPRIO_CMD_STATS:
                nsim_taprio_stats(&offload->stats);
                break;
        default:
                err = -EOPNOTSUPP;
        }

        return err;
}

static LIST_HEAD(nsim_block_cb_list);

static int
nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
{
        struct netdevsim *ns = netdev_priv(dev);

        switch (type) {
        case TC_SETUP_QDISC_TAPRIO:
                return nsim_setup_tc_taprio(dev, type_data);
        case TC_SETUP_BLOCK:
                return flow_block_cb_setup_simple(type_data,
                                                  &nsim_block_cb_list,
                                                  nsim_setup_tc_block_cb,
                                                  ns, ns, true);
        default:
                return -EOPNOTSUPP;
        }
}

static int
nsim_set_features(struct net_device *dev, netdev_features_t features)
{
        struct netdevsim *ns = netdev_priv(dev);

        if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))
                return nsim_bpf_disable_tc(ns);

        return 0;
}

static int nsim_get_iflink(const struct net_device *dev)
{
        struct netdevsim *nsim, *peer;
        int iflink;

        nsim = netdev_priv(dev);

        rcu_read_lock();
        peer = rcu_dereference(nsim->peer);
        iflink = peer ? READ_ONCE(peer->netdev->ifindex) :
                        READ_ONCE(dev->ifindex);
        rcu_read_unlock();

        return iflink;
}

static int nsim_rcv(struct nsim_rq *rq, int budget)
{
        struct net_device *dev = rq->napi.dev;
        struct bpf_prog *xdp_prog;
        struct netdevsim *ns;
        struct sk_buff *skb;
        unsigned int skblen;
        int i, ret;

        ns = netdev_priv(dev);
        xdp_prog = READ_ONCE(ns->xdp.prog);

        for (i = 0; i < budget; i++) {
                if (skb_queue_empty(&rq->skb_queue))
                        break;

                skb = skb_dequeue(&rq->skb_queue);

                if (xdp_prog) {
                        /* skb might be freed directly by XDP, save the len */
                        skblen = skb->len;

                        if (skb->ip_summed == CHECKSUM_PARTIAL)
                                skb_checksum_help(skb);
                        ret = do_xdp_generic(xdp_prog, &skb);
                        if (ret != XDP_PASS) {
                                dev_dstats_rx_add(dev, skblen);
                                continue;
                        }
                }

                /* skb might be discard at netif_receive_skb, save the len */
                dev_dstats_rx_add(dev, skb->len);
                napi_gro_receive(&rq->napi, skb);
        }

        nsim_start_peer_tx_queue(dev, rq);
        return i;
}

static int nsim_poll(struct napi_struct *napi, int budget)
{
        struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi);
        int done;

        done = nsim_rcv(rq, budget);
        if (done < budget)
                napi_complete_done(napi, done);

        return done;
}

static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi)
{
        struct page_pool_params params = {
                .order = 0,
                .pool_size = NSIM_RING_SIZE,
                .nid = NUMA_NO_NODE,
                .dev = &napi->dev->dev,
                .napi = napi,
                .dma_dir = DMA_BIDIRECTIONAL,
                .netdev = napi->dev,
        };
        struct page_pool *pool;

        pool = page_pool_create(&params);
        if (IS_ERR(pool))
                return PTR_ERR(pool);

        *p = pool;
        return 0;
}

static int nsim_init_napi(struct netdevsim *ns)
{
        struct net_device *dev = ns->netdev;
        struct nsim_rq *rq;
        int err, i;

        for (i = 0; i < dev->num_rx_queues; i++) {
                rq = ns->rq[i];

                netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i);
        }

        for (i = 0; i < dev->num_rx_queues; i++) {
                rq = ns->rq[i];

                err = nsim_create_page_pool(&rq->page_pool, &rq->napi);
                if (err)
                        goto err_pp_destroy;
        }

        return 0;

err_pp_destroy:
        while (i--) {
                page_pool_destroy(ns->rq[i]->page_pool);
                ns->rq[i]->page_pool = NULL;
        }

        for (i = 0; i < dev->num_rx_queues; i++)
                __netif_napi_del_locked(&ns->rq[i]->napi);

        return err;
}

static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer)
{
        struct nsim_rq *rq;

        rq = container_of(timer, struct nsim_rq, napi_timer);
        napi_schedule(&rq->napi);

        return HRTIMER_NORESTART;
}

static void nsim_rq_timer_init(struct nsim_rq *rq)
{
        hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC,
                      HRTIMER_MODE_REL);
}

static void nsim_enable_napi(struct netdevsim *ns)
{
        struct net_device *dev = ns->netdev;
        int i;

        for (i = 0; i < dev->num_rx_queues; i++) {
                struct nsim_rq *rq = ns->rq[i];

                netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi);
                napi_enable_locked(&rq->napi);
        }
}

static int nsim_open(struct net_device *dev)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct netdevsim *peer;
        int err;

        netdev_assert_locked(dev);

        err = nsim_init_napi(ns);
        if (err)
                return err;

        nsim_enable_napi(ns);

        peer = rtnl_dereference(ns->peer);
        if (peer && netif_running(peer->netdev)) {
                netif_carrier_on(dev);
                netif_carrier_on(peer->netdev);
        }

        return 0;
}

static void nsim_del_napi(struct netdevsim *ns)
{
        struct net_device *dev = ns->netdev;
        int i;

        for (i = 0; i < dev->num_rx_queues; i++) {
                struct nsim_rq *rq = ns->rq[i];

                napi_disable_locked(&rq->napi);
                __netif_napi_del_locked(&rq->napi);
        }
        synchronize_net();

        for (i = 0; i < dev->num_rx_queues; i++) {
                page_pool_destroy(ns->rq[i]->page_pool);
                ns->rq[i]->page_pool = NULL;
        }
}

static int nsim_stop(struct net_device *dev)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct netdevsim *peer;

        netdev_assert_locked(dev);

        netif_carrier_off(dev);
        peer = rtnl_dereference(ns->peer);
        if (peer)
                netif_carrier_off(peer->netdev);

        nsim_del_napi(ns);

        return 0;
}

static int nsim_shaper_set(struct net_shaper_binding *binding,
                           const struct net_shaper *shaper,
                           struct netlink_ext_ack *extack)
{
        return 0;
}

static int nsim_shaper_del(struct net_shaper_binding *binding,
                           const struct net_shaper_handle *handle,
                           struct netlink_ext_ack *extack)
{
        return 0;
}

static int nsim_shaper_group(struct net_shaper_binding *binding,
                             int leaves_count,
                             const struct net_shaper *leaves,
                             const struct net_shaper *root,
                             struct netlink_ext_ack *extack)
{
        return 0;
}

static void nsim_shaper_cap(struct net_shaper_binding *binding,
                            enum net_shaper_scope scope,
                            unsigned long *flags)
{
        *flags = ULONG_MAX;
}

static const struct net_shaper_ops nsim_shaper_ops = {
        .set                    = nsim_shaper_set,
        .delete                 = nsim_shaper_del,
        .group                  = nsim_shaper_group,
        .capabilities           = nsim_shaper_cap,
};

static const struct net_device_ops nsim_netdev_ops = {
        .ndo_start_xmit         = nsim_start_xmit,
        .ndo_set_rx_mode        = nsim_set_rx_mode,
        .ndo_set_mac_address    = eth_mac_addr,
        .ndo_validate_addr      = eth_validate_addr,
        .ndo_change_mtu         = nsim_change_mtu,
        .ndo_set_vf_mac         = nsim_set_vf_mac,
        .ndo_set_vf_vlan        = nsim_set_vf_vlan,
        .ndo_set_vf_rate        = nsim_set_vf_rate,
        .ndo_set_vf_spoofchk    = nsim_set_vf_spoofchk,
        .ndo_set_vf_trust       = nsim_set_vf_trust,
        .ndo_get_vf_config      = nsim_get_vf_config,
        .ndo_set_vf_link_state  = nsim_set_vf_link_state,
        .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en,
        .ndo_setup_tc           = nsim_setup_tc,
        .ndo_set_features       = nsim_set_features,
        .ndo_get_iflink         = nsim_get_iflink,
        .ndo_bpf                = nsim_bpf,
        .ndo_open               = nsim_open,
        .ndo_stop               = nsim_stop,
        .net_shaper_ops         = &nsim_shaper_ops,
};

static const struct net_device_ops nsim_vf_netdev_ops = {
        .ndo_start_xmit         = nsim_start_xmit,
        .ndo_set_rx_mode        = nsim_set_rx_mode,
        .ndo_set_mac_address    = eth_mac_addr,
        .ndo_validate_addr      = eth_validate_addr,
        .ndo_change_mtu         = nsim_change_mtu,
        .ndo_setup_tc           = nsim_setup_tc,
        .ndo_set_features       = nsim_set_features,
};

/* We don't have true per-queue stats, yet, so do some random fakery here.
 * Only report stuff for queue 0.
 */
static void nsim_get_queue_stats_rx(struct net_device *dev, int idx,
                                    struct netdev_queue_stats_rx *stats)
{
        struct rtnl_link_stats64 rtstats = {};

        if (!idx)
                dev_get_stats(dev, &rtstats);

        stats->packets = rtstats.rx_packets - !!rtstats.rx_packets;
        stats->bytes = rtstats.rx_bytes;
}

static void nsim_get_queue_stats_tx(struct net_device *dev, int idx,
                                    struct netdev_queue_stats_tx *stats)
{
        struct rtnl_link_stats64 rtstats = {};

        if (!idx)
                dev_get_stats(dev, &rtstats);

        stats->packets = rtstats.tx_packets - !!rtstats.tx_packets;
        stats->bytes = rtstats.tx_bytes;
}

static void nsim_get_base_stats(struct net_device *dev,
                                struct netdev_queue_stats_rx *rx,
                                struct netdev_queue_stats_tx *tx)
{
        struct rtnl_link_stats64 rtstats = {};

        dev_get_stats(dev, &rtstats);

        rx->packets = !!rtstats.rx_packets;
        rx->bytes = 0;
        tx->packets = !!rtstats.tx_packets;
        tx->bytes = 0;
}

static const struct netdev_stat_ops nsim_stat_ops = {
        .get_queue_stats_tx     = nsim_get_queue_stats_tx,
        .get_queue_stats_rx     = nsim_get_queue_stats_rx,
        .get_base_stats         = nsim_get_base_stats,
};

static struct nsim_rq *nsim_queue_alloc(void)
{
        struct nsim_rq *rq;

        rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT);
        if (!rq)
                return NULL;

        skb_queue_head_init(&rq->skb_queue);
        nsim_rq_timer_init(rq);
        return rq;
}

static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq)
{
        hrtimer_cancel(&rq->napi_timer);

        if (rq->skb_queue.qlen) {
                local_bh_disable();
                dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen);
                local_bh_enable();
        }

        skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE);
        kfree(rq);
}

/* Queue reset mode is controlled by ns->rq_reset_mode.
 * - normal - new NAPI new pool (old NAPI enabled when new added)
 * - mode 1 - allocate new pool (NAPI is only disabled / enabled)
 * - mode 2 - new NAPI new pool (old NAPI removed before new added)
 * - mode 3 - new NAPI new pool (old NAPI disabled when new added)
 */
struct nsim_queue_mem {
        struct nsim_rq *rq;
        struct page_pool *pp;
};

static int
nsim_queue_mem_alloc(struct net_device *dev,
                     struct netdev_queue_config *qcfg,
                     void *per_queue_mem, int idx)
{
        struct nsim_queue_mem *qmem = per_queue_mem;
        struct netdevsim *ns = netdev_priv(dev);
        int err;

        if (ns->rq_reset_mode > 3)
                return -EINVAL;

        if (ns->rq_reset_mode == 1) {
                if (!netif_running(ns->netdev))
                        return -ENETDOWN;
                return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi);
        }

        qmem->rq = nsim_queue_alloc();
        if (!qmem->rq)
                return -ENOMEM;

        err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi);
        if (err)
                goto err_free;

        if (!ns->rq_reset_mode)
                netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
                                             idx);

        return 0;

err_free:
        nsim_queue_free(dev, qmem->rq);
        return err;
}

static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem)
{
        struct nsim_queue_mem *qmem = per_queue_mem;
        struct netdevsim *ns = netdev_priv(dev);

        page_pool_destroy(qmem->pp);
        if (qmem->rq) {
                if (!ns->rq_reset_mode)
                        netif_napi_del_locked(&qmem->rq->napi);
                page_pool_destroy(qmem->rq->page_pool);
                nsim_queue_free(dev, qmem->rq);
        }
}

static int
nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg,
                 void *per_queue_mem, int idx)
{
        struct nsim_queue_mem *qmem = per_queue_mem;
        struct netdevsim *ns = netdev_priv(dev);

        netdev_assert_locked(dev);

        if (ns->rq_reset_mode == 1) {
                ns->rq[idx]->page_pool = qmem->pp;
                napi_enable_locked(&ns->rq[idx]->napi);
                return 0;
        }

        /* netif_napi_add()/_del() should normally be called from alloc/free,
         * here we want to test various call orders.
         */
        if (ns->rq_reset_mode == 2) {
                netif_napi_del_locked(&ns->rq[idx]->napi);
                netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
                                             idx);
        } else if (ns->rq_reset_mode == 3) {
                netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll,
                                             idx);
                netif_napi_del_locked(&ns->rq[idx]->napi);
        }

        ns->rq[idx] = qmem->rq;
        napi_enable_locked(&ns->rq[idx]->napi);

        return 0;
}

static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx)
{
        struct nsim_queue_mem *qmem = per_queue_mem;
        struct netdevsim *ns = netdev_priv(dev);

        netdev_assert_locked(dev);

        napi_disable_locked(&ns->rq[idx]->napi);

        if (ns->rq_reset_mode == 1) {
                qmem->pp = ns->rq[idx]->page_pool;
                page_pool_disable_direct_recycling(qmem->pp);
        } else {
                qmem->rq = ns->rq[idx];
        }

        return 0;
}

static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = {
        .ndo_queue_mem_size     = sizeof(struct nsim_queue_mem),
        .ndo_queue_mem_alloc    = nsim_queue_mem_alloc,
        .ndo_queue_mem_free     = nsim_queue_mem_free,
        .ndo_queue_start        = nsim_queue_start,
        .ndo_queue_stop         = nsim_queue_stop,
};

static ssize_t
nsim_qreset_write(struct file *file, const char __user *data,
                  size_t count, loff_t *ppos)
{
        struct netdevsim *ns = file->private_data;
        unsigned int queue, mode;
        char buf[32];
        ssize_t ret;

        if (count >= sizeof(buf))
                return -EINVAL;
        if (copy_from_user(buf, data, count))
                return -EFAULT;
        buf[count] = '\0';

        ret = sscanf(buf, "%u %u", &queue, &mode);
        if (ret != 2)
                return -EINVAL;

        netdev_lock(ns->netdev);
        if (queue >= ns->netdev->real_num_rx_queues) {
                ret = -EINVAL;
                goto exit_unlock;
        }

        ns->rq_reset_mode = mode;
        ret = netdev_rx_queue_restart(ns->netdev, queue);
        ns->rq_reset_mode = 0;
        if (ret)
                goto exit_unlock;

        ret = count;
exit_unlock:
        netdev_unlock(ns->netdev);
        return ret;
}

static const struct file_operations nsim_qreset_fops = {
        .open = simple_open,
        .write = nsim_qreset_write,
        .owner = THIS_MODULE,
};

static ssize_t
nsim_pp_hold_read(struct file *file, char __user *data,
                  size_t count, loff_t *ppos)
{
        struct netdevsim *ns = file->private_data;
        char buf[3] = "n\n";

        if (ns->page)
                buf[0] = 'y';

        return simple_read_from_buffer(data, count, ppos, buf, 2);
}

static ssize_t
nsim_pp_hold_write(struct file *file, const char __user *data,
                   size_t count, loff_t *ppos)
{
        struct netdevsim *ns = file->private_data;
        ssize_t ret;
        bool val;

        ret = kstrtobool_from_user(data, count, &val);
        if (ret)
                return ret;

        rtnl_lock();
        ret = count;
        if (val == !!ns->page)
                goto exit;

        if (!netif_running(ns->netdev) && val) {
                ret = -ENETDOWN;
        } else if (val) {
                ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool);
                if (!ns->page)
                        ret = -ENOMEM;
        } else {
                page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp,
                                        ns->page, false);
                ns->page = NULL;
        }

exit:
        rtnl_unlock();
        return ret;
}

static const struct file_operations nsim_pp_hold_fops = {
        .open = simple_open,
        .read = nsim_pp_hold_read,
        .write = nsim_pp_hold_write,
        .llseek = generic_file_llseek,
        .owner = THIS_MODULE,
};

static void nsim_setup(struct net_device *dev)
{
        ether_setup(dev);
        eth_hw_addr_random(dev);

        dev->flags &= ~IFF_MULTICAST;
        dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
        dev->features |= NETIF_F_HIGHDMA |
                         NETIF_F_SG |
                         NETIF_F_FRAGLIST |
                         NETIF_F_HW_CSUM |
                         NETIF_F_LRO |
                         NETIF_F_TSO;
        dev->hw_features |= NETIF_F_HW_TC |
                            NETIF_F_SG |
                            NETIF_F_FRAGLIST |
                            NETIF_F_HW_CSUM |
                            NETIF_F_LRO |
                            NETIF_F_TSO |
                            NETIF_F_LOOPBACK;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
        dev->max_mtu = ETH_MAX_MTU;
        dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD;
}

static int nsim_queue_init(struct netdevsim *ns)
{
        struct net_device *dev = ns->netdev;
        int i;

        ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT);
        if (!ns->rq)
                return -ENOMEM;

        for (i = 0; i < dev->num_rx_queues; i++) {
                ns->rq[i] = nsim_queue_alloc();
                if (!ns->rq[i])
                        goto err_free_prev;
        }

        return 0;

err_free_prev:
        while (i--)
                kfree(ns->rq[i]);
        kfree(ns->rq);
        return -ENOMEM;
}

static void nsim_queue_uninit(struct netdevsim *ns)
{
        struct net_device *dev = ns->netdev;
        int i;

        for (i = 0; i < dev->num_rx_queues; i++)
                nsim_queue_free(dev, ns->rq[i]);

        kfree(ns->rq);
        ns->rq = NULL;
}

static int nsim_init_netdevsim(struct netdevsim *ns)
{
        struct netdevsim *peer;
        struct mock_phc *phc;
        int err;

        phc = mock_phc_create(&ns->nsim_bus_dev->dev);
        if (IS_ERR(phc))
                return PTR_ERR(phc);

        ns->phc = phc;
        ns->netdev->netdev_ops = &nsim_netdev_ops;
        ns->netdev->stat_ops = &nsim_stat_ops;
        ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops;
        netdev_lockdep_set_classes(ns->netdev);

        err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev);
        if (err)
                goto err_phc_destroy;

        rtnl_lock();
        err = nsim_queue_init(ns);
        if (err)
                goto err_utn_destroy;

        err = nsim_bpf_init(ns);
        if (err)
                goto err_rq_destroy;

        nsim_macsec_init(ns);
        nsim_ipsec_init(ns);

        err = register_netdevice(ns->netdev);
        if (err)
                goto err_ipsec_teardown;
        rtnl_unlock();

        err = nsim_psp_init(ns);
        if (err)
                goto err_unregister_netdev;

        if (IS_ENABLED(CONFIG_DEBUG_NET)) {
                ns->nb.notifier_call = netdev_debug_event;
                if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
                                                        &ns->nn))
                        ns->nb.notifier_call = NULL;
        }

        return 0;

err_unregister_netdev:
        rtnl_lock();
        peer = rtnl_dereference(ns->peer);
        if (peer)
                RCU_INIT_POINTER(peer->peer, NULL);
        RCU_INIT_POINTER(ns->peer, NULL);
        unregister_netdevice(ns->netdev);
err_ipsec_teardown:
        nsim_ipsec_teardown(ns);
        nsim_macsec_teardown(ns);
        nsim_bpf_uninit(ns);
err_rq_destroy:
        nsim_queue_uninit(ns);
err_utn_destroy:
        rtnl_unlock();
        nsim_udp_tunnels_info_destroy(ns->netdev);
err_phc_destroy:
        mock_phc_destroy(ns->phc);
        return err;
}

static int nsim_init_netdevsim_vf(struct netdevsim *ns)
{
        int err;

        ns->netdev->netdev_ops = &nsim_vf_netdev_ops;
        rtnl_lock();
        err = register_netdevice(ns->netdev);
        rtnl_unlock();
        return err;
}

static void nsim_exit_netdevsim(struct netdevsim *ns)
{
        nsim_udp_tunnels_info_destroy(ns->netdev);
        mock_phc_destroy(ns->phc);
}

struct netdevsim *nsim_create(struct nsim_dev *nsim_dev,
                              struct nsim_dev_port *nsim_dev_port,
                              u8 perm_addr[ETH_ALEN])
{
        struct net_device *dev;
        struct netdevsim *ns;
        int err;

        dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup,
                              nsim_dev->nsim_bus_dev->num_queues);
        if (!dev)
                return ERR_PTR(-ENOMEM);

        if (perm_addr)
                memcpy(dev->perm_addr, perm_addr, ETH_ALEN);

        dev_net_set(dev, nsim_dev_net(nsim_dev));
        ns = netdev_priv(dev);
        ns->netdev = dev;
        ns->nsim_dev = nsim_dev;
        ns->nsim_dev_port = nsim_dev_port;
        ns->nsim_bus_dev = nsim_dev->nsim_bus_dev;
        SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev);
        SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port);
        nsim_ethtool_init(ns);
        if (nsim_dev_port_is_pf(nsim_dev_port))
                err = nsim_init_netdevsim(ns);
        else
                err = nsim_init_netdevsim_vf(ns);
        if (err)
                goto err_free_netdev;

        ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir,
                                         ns, &nsim_pp_hold_fops);
        ns->qr_dfs = debugfs_create_file("queue_reset", 0200,
                                         nsim_dev_port->ddir, ns,
                                         &nsim_qreset_fops);
        return ns;

err_free_netdev:
        free_netdev(dev);
        return ERR_PTR(err);
}

void nsim_destroy(struct netdevsim *ns)
{
        struct net_device *dev = ns->netdev;
        struct netdevsim *peer;

        debugfs_remove(ns->qr_dfs);
        debugfs_remove(ns->pp_dfs);

        if (ns->nb.notifier_call)
                unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb,
                                                      &ns->nn);

        nsim_psp_uninit(ns);

        rtnl_lock();
        peer = rtnl_dereference(ns->peer);
        if (peer)
                RCU_INIT_POINTER(peer->peer, NULL);
        RCU_INIT_POINTER(ns->peer, NULL);
        unregister_netdevice(dev);
        if (nsim_dev_port_is_pf(ns->nsim_dev_port)) {
                nsim_macsec_teardown(ns);
                nsim_ipsec_teardown(ns);
                nsim_bpf_uninit(ns);
                nsim_queue_uninit(ns);
        }
        rtnl_unlock();
        if (nsim_dev_port_is_pf(ns->nsim_dev_port))
                nsim_exit_netdevsim(ns);

        /* Put this intentionally late to exercise the orphaning path */
        if (ns->page) {
                page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp,
                                        ns->page, false);
                ns->page = NULL;
        }

        free_netdev(dev);
}

bool netdev_is_nsim(struct net_device *dev)
{
        return dev->netdev_ops == &nsim_netdev_ops;
}

static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
                         struct netlink_ext_ack *extack)
{
        NL_SET_ERR_MSG_MOD(extack,
                           "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device");
        return -EOPNOTSUPP;
}

static struct rtnl_link_ops nsim_link_ops __read_mostly = {
        .kind           = DRV_NAME,
        .validate       = nsim_validate,
};

static int __init nsim_module_init(void)
{
        int err;

        err = nsim_dev_init();
        if (err)
                return err;

        err = nsim_bus_init();
        if (err)
                goto err_dev_exit;

        err = rtnl_link_register(&nsim_link_ops);
        if (err)
                goto err_bus_exit;

        return 0;

err_bus_exit:
        nsim_bus_exit();
err_dev_exit:
        nsim_dev_exit();
        return err;
}

static void __exit nsim_module_exit(void)
{
        rtnl_link_unregister(&nsim_link_ops);
        nsim_bus_exit();
        nsim_dev_exit();
}

module_init(nsim_module_init);
module_exit(nsim_module_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Simulated networking device for testing");
MODULE_ALIAS_RTNL_LINK(DRV_NAME);