// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 */

#include "queueing.h"
#include "socket.h"
#include "timers.h"
#include "device.h"
#include "ratelimiter.h"
#include "peer.h"
#include "messages.h"

#include <linux/module.h>
#include <linux/rtnetlink.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/if_arp.h>
#include <linux/icmp.h>
#include <linux/suspend.h>
#include <net/dst_metadata.h>
#include <net/gso.h>
#include <net/icmp.h>
#include <net/rtnetlink.h>
#include <net/ip_tunnels.h>
#include <net/addrconf.h>

static LIST_HEAD(device_list);

static int wg_open(struct net_device *dev)
{
        struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
        struct inet6_dev *dev_v6 = __in6_dev_get(dev);
        struct wg_device *wg = netdev_priv(dev);
        struct wg_peer *peer;
        int ret;

        if (dev_v4) {
                /* At some point we might put this check near the ip_rt_send_
                 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
                 * to the current secpath check.
                 */
                IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
                IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
        }
        if (dev_v6)
                dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;

        mutex_lock(&wg->device_update_lock);
        ret = wg_socket_init(wg, wg->incoming_port);
        if (ret < 0)
                goto out;
        list_for_each_entry(peer, &wg->peer_list, peer_list) {
                wg_packet_send_staged_packets(peer);
                if (peer->persistent_keepalive_interval)
                        wg_packet_send_keepalive(peer);
        }
out:
        mutex_unlock(&wg->device_update_lock);
        return ret;
}

static int wg_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        /* If the machine is constantly suspending and resuming, as part of
         * its normal operation rather than as a somewhat rare event, then we
         * don't actually want to clear keys.
         */
        if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) ||
            IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP))
                return 0;

        if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
                return 0;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                mutex_lock(&wg->device_update_lock);
                list_for_each_entry(peer, &wg->peer_list, peer_list) {
                        timer_delete(&peer->timer_zero_key_material);
                        wg_noise_handshake_clear(&peer->handshake);
                        wg_noise_keypairs_clear(&peer->keypairs);
                }
                mutex_unlock(&wg->device_update_lock);
        }
        rtnl_unlock();
        rcu_barrier();
        return 0;
}

static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };

static int wg_vm_notification(struct notifier_block *nb, unsigned long action, void *data)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                mutex_lock(&wg->device_update_lock);
                list_for_each_entry(peer, &wg->peer_list, peer_list)
                        wg_noise_expire_current_peer_keypairs(peer);
                mutex_unlock(&wg->device_update_lock);
        }
        rtnl_unlock();
        return 0;
}

static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification };

static int wg_stop(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        struct wg_peer *peer;
        struct sk_buff *skb;

        mutex_lock(&wg->device_update_lock);
        list_for_each_entry(peer, &wg->peer_list, peer_list) {
                wg_packet_purge_staged_packets(peer);
                wg_timers_stop(peer);
                wg_noise_handshake_clear(&peer->handshake);
                wg_noise_keypairs_clear(&peer->keypairs);
                wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
        }
        mutex_unlock(&wg->device_update_lock);
        while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL)
                kfree_skb(skb);
        atomic_set(&wg->handshake_queue_len, 0);
        wg_socket_reinit(wg, NULL, NULL);
        return 0;
}

static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        struct sk_buff_head packets;
        struct wg_peer *peer;
        struct sk_buff *next;
        sa_family_t family;
        u32 mtu;
        int ret;

        if (unlikely(!wg_check_packet_protocol(skb))) {
                ret = -EPROTONOSUPPORT;
                net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
                goto err;
        }

        peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
        if (unlikely(!peer)) {
                ret = -ENOKEY;
                if (skb->protocol == htons(ETH_P_IP))
                        net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
                                            dev->name, &ip_hdr(skb)->daddr);
                else if (skb->protocol == htons(ETH_P_IPV6))
                        net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
                                            dev->name, &ipv6_hdr(skb)->daddr);
                goto err_icmp;
        }

        family = READ_ONCE(peer->endpoint.addr.sa_family);
        if (unlikely(family != AF_INET && family != AF_INET6)) {
                ret = -EDESTADDRREQ;
                net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
                                    dev->name, peer->internal_id);
                goto err_peer;
        }

        mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;

        __skb_queue_head_init(&packets);
        if (!skb_is_gso(skb)) {
                skb_mark_not_on_list(skb);
        } else {
                struct sk_buff *segs = skb_gso_segment(skb, 0);

                if (IS_ERR(segs)) {
                        ret = PTR_ERR(segs);
                        goto err_peer;
                }
                dev_kfree_skb(skb);
                skb = segs;
        }

        skb_list_walk_safe(skb, skb, next) {
                skb_mark_not_on_list(skb);

                skb = skb_share_check(skb, GFP_ATOMIC);
                if (unlikely(!skb))
                        continue;

                /* We only need to keep the original dst around for icmp,
                 * so at this point we're in a position to drop it.
                 */
                skb_dst_drop(skb);

                PACKET_CB(skb)->mtu = mtu;

                __skb_queue_tail(&packets, skb);
        }

        spin_lock_bh(&peer->staged_packet_queue.lock);
        /* If the queue is getting too big, we start removing the oldest packets
         * until it's small again. We do this before adding the new packet, so
         * we don't remove GSO segments that are in excess.
         */
        while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
                dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
                DEV_STATS_INC(dev, tx_dropped);
        }
        skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
        spin_unlock_bh(&peer->staged_packet_queue.lock);

        wg_packet_send_staged_packets(peer);

        wg_peer_put(peer);
        return NETDEV_TX_OK;

err_peer:
        wg_peer_put(peer);
err_icmp:
        if (skb->protocol == htons(ETH_P_IP))
                icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
        else if (skb->protocol == htons(ETH_P_IPV6))
                icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
err:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
        return ret;
}

static const struct net_device_ops netdev_ops = {
        .ndo_open               = wg_open,
        .ndo_stop               = wg_stop,
        .ndo_start_xmit         = wg_xmit,
};

static void wg_destruct(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);

        rtnl_lock();
        list_del(&wg->device_list);
        rtnl_unlock();
        mutex_lock(&wg->device_update_lock);
        rcu_assign_pointer(wg->creating_net, NULL);
        wg->incoming_port = 0;
        wg_socket_reinit(wg, NULL, NULL);
        /* The final references are cleared in the below calls to destroy_workqueue. */
        wg_peer_remove_all(wg);
        destroy_workqueue(wg->handshake_receive_wq);
        destroy_workqueue(wg->handshake_send_wq);
        destroy_workqueue(wg->packet_crypt_wq);
        wg_packet_queue_free(&wg->handshake_queue, true);
        wg_packet_queue_free(&wg->decrypt_queue, false);
        wg_packet_queue_free(&wg->encrypt_queue, false);
        rcu_barrier(); /* Wait for all the peers to be actually freed. */
        wg_ratelimiter_uninit();
        memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
        kvfree(wg->index_hashtable);
        kvfree(wg->peer_hashtable);
        mutex_unlock(&wg->device_update_lock);

        pr_debug("%s: Interface destroyed\n", dev->name);
        free_netdev(dev);
}

static const struct device_type device_type = { .name = KBUILD_MODNAME };

static void wg_setup(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
                                    NETIF_F_SG | NETIF_F_GSO |
                                    NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
        const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
                             max(sizeof(struct ipv6hdr), sizeof(struct iphdr));

        dev->netdev_ops = &netdev_ops;
        dev->header_ops = &ip_tunnel_header_ops;
        dev->hard_header_len = 0;
        dev->addr_len = 0;
        dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
        dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
        dev->type = ARPHRD_NONE;
        dev->flags = IFF_POINTOPOINT | IFF_NOARP;
        dev->priv_flags |= IFF_NO_QUEUE;
        dev->lltx = true;
        dev->features |= WG_NETDEV_FEATURES;
        dev->hw_features |= WG_NETDEV_FEATURES;
        dev->hw_enc_features |= WG_NETDEV_FEATURES;
        dev->mtu = ETH_DATA_LEN - overhead;
        dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        SET_NETDEV_DEVTYPE(dev, &device_type);

        /* We need to keep the dst around in case of icmp replies. */
        netif_keep_dst(dev);

        netif_set_tso_max_size(dev, GSO_MAX_SIZE);

        wg->dev = dev;
}

static int wg_newlink(struct net_device *dev,
                      struct rtnl_newlink_params *params,
                      struct netlink_ext_ack *extack)
{
        struct net *link_net = rtnl_newlink_link_net(params);
        struct wg_device *wg = netdev_priv(dev);
        int ret = -ENOMEM;

        rcu_assign_pointer(wg->creating_net, link_net);
        init_rwsem(&wg->static_identity.lock);
        mutex_init(&wg->socket_update_lock);
        mutex_init(&wg->device_update_lock);
        wg_allowedips_init(&wg->peer_allowedips);
        wg_cookie_checker_init(&wg->cookie_checker, wg);
        INIT_LIST_HEAD(&wg->peer_list);
        wg->device_update_gen = 1;

        wg->peer_hashtable = wg_pubkey_hashtable_alloc();
        if (!wg->peer_hashtable)
                return ret;

        wg->index_hashtable = wg_index_hashtable_alloc();
        if (!wg->index_hashtable)
                goto err_free_peer_hashtable;

        wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
                        WQ_CPU_INTENSIVE | WQ_FREEZABLE | WQ_PERCPU, 0,
                        dev->name);
        if (!wg->handshake_receive_wq)
                goto err_free_index_hashtable;

        wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
                        WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
        if (!wg->handshake_send_wq)
                goto err_destroy_handshake_receive;

        wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
                        WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_PERCPU, 0,
                        dev->name);
        if (!wg->packet_crypt_wq)
                goto err_destroy_handshake_send;

        ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
                                   MAX_QUEUED_PACKETS);
        if (ret < 0)
                goto err_destroy_packet_crypt;

        ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
                                   MAX_QUEUED_PACKETS);
        if (ret < 0)
                goto err_free_encrypt_queue;

        ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker,
                                   MAX_QUEUED_INCOMING_HANDSHAKES);
        if (ret < 0)
                goto err_free_decrypt_queue;

        ret = wg_ratelimiter_init();
        if (ret < 0)
                goto err_free_handshake_queue;

        netif_threaded_enable(dev);
        ret = register_netdevice(dev);
        if (ret < 0)
                goto err_uninit_ratelimiter;

        list_add(&wg->device_list, &device_list);

        /* We wait until the end to assign priv_destructor, so that
         * register_netdevice doesn't call it for us if it fails.
         */
        dev->priv_destructor = wg_destruct;

        pr_debug("%s: Interface created\n", dev->name);
        return ret;

err_uninit_ratelimiter:
        wg_ratelimiter_uninit();
err_free_handshake_queue:
        wg_packet_queue_free(&wg->handshake_queue, false);
err_free_decrypt_queue:
        wg_packet_queue_free(&wg->decrypt_queue, false);
err_free_encrypt_queue:
        wg_packet_queue_free(&wg->encrypt_queue, false);
err_destroy_packet_crypt:
        destroy_workqueue(wg->packet_crypt_wq);
err_destroy_handshake_send:
        destroy_workqueue(wg->handshake_send_wq);
err_destroy_handshake_receive:
        destroy_workqueue(wg->handshake_receive_wq);
err_free_index_hashtable:
        kvfree(wg->index_hashtable);
err_free_peer_hashtable:
        kvfree(wg->peer_hashtable);
        return ret;
}

static struct rtnl_link_ops link_ops __read_mostly = {
        .kind                   = KBUILD_MODNAME,
        .priv_size              = sizeof(struct wg_device),
        .setup                  = wg_setup,
        .newlink                = wg_newlink,
};

static void wg_netns_pre_exit(struct net *net)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                if (rcu_access_pointer(wg->creating_net) == net) {
                        pr_debug("%s: Creating namespace exiting\n", wg->dev->name);
                        netif_carrier_off(wg->dev);
                        mutex_lock(&wg->device_update_lock);
                        rcu_assign_pointer(wg->creating_net, NULL);
                        wg_socket_reinit(wg, NULL, NULL);
                        list_for_each_entry(peer, &wg->peer_list, peer_list)
                                wg_socket_clear_peer_endpoint_src(peer);
                        mutex_unlock(&wg->device_update_lock);
                }
        }
        rtnl_unlock();
}

static struct pernet_operations pernet_ops = {
        .pre_exit = wg_netns_pre_exit
};

int __init wg_device_init(void)
{
        int ret;

        ret = register_pm_notifier(&pm_notifier);
        if (ret)
                return ret;

        ret = register_random_vmfork_notifier(&vm_notifier);
        if (ret)
                goto error_pm;

        ret = register_pernet_device(&pernet_ops);
        if (ret)
                goto error_vm;

        ret = rtnl_link_register(&link_ops);
        if (ret)
                goto error_pernet;

        return 0;

error_pernet:
        unregister_pernet_device(&pernet_ops);
error_vm:
        unregister_random_vmfork_notifier(&vm_notifier);
error_pm:
        unregister_pm_notifier(&pm_notifier);
        return ret;
}

void wg_device_uninit(void)
{
        rtnl_link_unregister(&link_ops);
        unregister_pernet_device(&pernet_ops);
        unregister_random_vmfork_notifier(&vm_notifier);
        unregister_pm_notifier(&pm_notifier);
        rcu_barrier();
}