// SPDX-License-Identifier: GPL-2.0
/*
 * NVMe PCI Endpoint Function target driver.
 *
 * Copyright (c) 2024, Western Digital Corporation or its affiliates.
 * Copyright (c) 2024, Rick Wertenbroek <rick.wertenbroek@gmail.com>
 *                     REDS Institute, HEIG-VD, HES-SO, Switzerland
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/delay.h>
#include <linux/dmaengine.h>
#include <linux/io.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/nvme.h>
#include <linux/pci_ids.h>
#include <linux/pci-epc.h>
#include <linux/pci-epf.h>
#include <linux/pci_regs.h>
#include <linux/slab.h>

#include "nvmet.h"

static LIST_HEAD(nvmet_pci_epf_ports);
static DEFINE_MUTEX(nvmet_pci_epf_ports_mutex);

/*
 * Default and maximum allowed data transfer size. For the default,
 * allow up to 128 page-sized segments. For the maximum allowed,
 * use 4 times the default (which is completely arbitrary).
 */
#define NVMET_PCI_EPF_MAX_SEGS          128
#define NVMET_PCI_EPF_MDTS_KB           \
        (NVMET_PCI_EPF_MAX_SEGS << (PAGE_SHIFT - 10))
#define NVMET_PCI_EPF_MAX_MDTS_KB       (NVMET_PCI_EPF_MDTS_KB * 4)

/*
 * IRQ vector coalescing threshold: by default, post 8 CQEs before raising an
 * interrupt vector to the host. This default 8 is completely arbitrary and can
 * be changed by the host with a nvme_set_features command.
 */
#define NVMET_PCI_EPF_IV_THRESHOLD      8

/*
 * BAR CC register and SQ polling intervals.
 */
#define NVMET_PCI_EPF_CC_POLL_INTERVAL  msecs_to_jiffies(10)
#define NVMET_PCI_EPF_SQ_POLL_INTERVAL  msecs_to_jiffies(5)
#define NVMET_PCI_EPF_SQ_POLL_IDLE      msecs_to_jiffies(5000)

/*
 * SQ arbitration burst default: fetch at most 8 commands at a time from an SQ.
 */
#define NVMET_PCI_EPF_SQ_AB             8

/*
 * Handling of CQs is normally immediate, unless we fail to map a CQ or the CQ
 * is full, in which case we retry the CQ processing after this interval.
 */
#define NVMET_PCI_EPF_CQ_RETRY_INTERVAL msecs_to_jiffies(1)

enum nvmet_pci_epf_queue_flags {
        NVMET_PCI_EPF_Q_LIVE = 0,       /* The queue is live */
        NVMET_PCI_EPF_Q_IRQ_ENABLED,    /* IRQ is enabled for this queue */
};

/*
 * IRQ vector descriptor.
 */
struct nvmet_pci_epf_irq_vector {
        unsigned int    vector;
        unsigned int    ref;
        bool            cd;
        int             nr_irqs;
};

struct nvmet_pci_epf_queue {
        union {
                struct nvmet_sq         nvme_sq;
                struct nvmet_cq         nvme_cq;
        };
        struct nvmet_pci_epf_ctrl       *ctrl;
        unsigned long                   flags;

        u64                             pci_addr;
        size_t                          pci_size;
        struct pci_epc_map              pci_map;

        u16                             qid;
        u16                             depth;
        u16                             vector;
        u16                             head;
        u16                             tail;
        u16                             phase;
        u32                             db;

        size_t                          qes;

        struct nvmet_pci_epf_irq_vector *iv;
        struct workqueue_struct         *iod_wq;
        struct delayed_work             work;
        spinlock_t                      lock;
        struct list_head                list;
};

/*
 * PCI Root Complex (RC) address data segment for mapping an admin or
 * I/O command buffer @buf of @length bytes to the PCI address @pci_addr.
 */
struct nvmet_pci_epf_segment {
        void                            *buf;
        u64                             pci_addr;
        u32                             length;
};

/*
 * Command descriptors.
 */
struct nvmet_pci_epf_iod {
        struct list_head                link;

        struct nvmet_req                req;
        struct nvme_command             cmd;
        struct nvme_completion          cqe;
        unsigned int                    status;

        struct nvmet_pci_epf_ctrl       *ctrl;

        struct nvmet_pci_epf_queue      *sq;
        struct nvmet_pci_epf_queue      *cq;

        /* Data transfer size and direction for the command. */
        size_t                          data_len;
        enum dma_data_direction         dma_dir;

        /*
         * PCI Root Complex (RC) address data segments: if nr_data_segs is 1, we
         * use only @data_seg. Otherwise, the array of segments @data_segs is
         * allocated to manage multiple PCI address data segments. @data_sgl and
         * @data_sgt are used to setup the command request for execution by the
         * target core.
         */
        unsigned int                    nr_data_segs;
        struct nvmet_pci_epf_segment    data_seg;
        struct nvmet_pci_epf_segment    *data_segs;
        struct scatterlist              data_sgl;
        struct sg_table                 data_sgt;

        struct work_struct              work;
        struct completion               done;
};

/*
 * PCI target controller private data.
 */
struct nvmet_pci_epf_ctrl {
        struct nvmet_pci_epf            *nvme_epf;
        struct nvmet_port               *port;
        struct nvmet_ctrl               *tctrl;
        struct device                   *dev;

        unsigned int                    nr_queues;
        struct nvmet_pci_epf_queue      *sq;
        struct nvmet_pci_epf_queue      *cq;
        unsigned int                    sq_ab;

        mempool_t                       iod_pool;
        void                            *bar;
        u64                             cap;
        u32                             cc;
        u32                             csts;

        size_t                          io_sqes;
        size_t                          io_cqes;

        size_t                          mps_shift;
        size_t                          mps;
        size_t                          mps_mask;

        unsigned int                    mdts;

        struct delayed_work             poll_cc;
        struct delayed_work             poll_sqs;

        struct mutex                    irq_lock;
        struct nvmet_pci_epf_irq_vector *irq_vectors;
        unsigned int                    irq_vector_threshold;

        bool                            link_up;
        bool                            enabled;
};

/*
 * PCI EPF driver private data.
 */
struct nvmet_pci_epf {
        struct pci_epf                  *epf;

        const struct pci_epc_features   *epc_features;

        void                            *reg_bar;
        size_t                          msix_table_offset;

        unsigned int                    irq_type;
        unsigned int                    nr_vectors;

        struct nvmet_pci_epf_ctrl       ctrl;

        bool                            dma_enabled;
        struct dma_chan                 *dma_tx_chan;
        struct mutex                    dma_tx_lock;
        struct dma_chan                 *dma_rx_chan;
        struct mutex                    dma_rx_lock;

        struct mutex                    mmio_lock;

        /* PCI endpoint function configfs attributes. */
        struct config_group             group;
        __le16                          portid;
        char                            subsysnqn[NVMF_NQN_SIZE];
        unsigned int                    mdts_kb;
};

static inline u32 nvmet_pci_epf_bar_read32(struct nvmet_pci_epf_ctrl *ctrl,
                                           u32 off)
{
        __le32 *bar_reg = ctrl->bar + off;

        return le32_to_cpu(READ_ONCE(*bar_reg));
}

static inline void nvmet_pci_epf_bar_write32(struct nvmet_pci_epf_ctrl *ctrl,
                                             u32 off, u32 val)
{
        __le32 *bar_reg = ctrl->bar + off;

        WRITE_ONCE(*bar_reg, cpu_to_le32(val));
}

static inline u64 nvmet_pci_epf_bar_read64(struct nvmet_pci_epf_ctrl *ctrl,
                                           u32 off)
{
        return (u64)nvmet_pci_epf_bar_read32(ctrl, off) |
                ((u64)nvmet_pci_epf_bar_read32(ctrl, off + 4) << 32);
}

static inline void nvmet_pci_epf_bar_write64(struct nvmet_pci_epf_ctrl *ctrl,
                                             u32 off, u64 val)
{
        nvmet_pci_epf_bar_write32(ctrl, off, val & 0xFFFFFFFF);
        nvmet_pci_epf_bar_write32(ctrl, off + 4, (val >> 32) & 0xFFFFFFFF);
}

static inline int nvmet_pci_epf_mem_map(struct nvmet_pci_epf *nvme_epf,
                u64 pci_addr, size_t size, struct pci_epc_map *map)
{
        struct pci_epf *epf = nvme_epf->epf;

        return pci_epc_mem_map(epf->epc, epf->func_no, epf->vfunc_no,
                               pci_addr, size, map);
}

static inline void nvmet_pci_epf_mem_unmap(struct nvmet_pci_epf *nvme_epf,
                                           struct pci_epc_map *map)
{
        struct pci_epf *epf = nvme_epf->epf;

        pci_epc_mem_unmap(epf->epc, epf->func_no, epf->vfunc_no, map);
}

struct nvmet_pci_epf_dma_filter {
        struct device *dev;
        u32 dma_mask;
};

static bool nvmet_pci_epf_dma_filter(struct dma_chan *chan, void *arg)
{
        struct nvmet_pci_epf_dma_filter *filter = arg;
        struct dma_slave_caps caps;

        memset(&caps, 0, sizeof(caps));
        dma_get_slave_caps(chan, &caps);

        return chan->device->dev == filter->dev &&
                (filter->dma_mask & caps.directions);
}

static void nvmet_pci_epf_init_dma(struct nvmet_pci_epf *nvme_epf)
{
        struct pci_epf *epf = nvme_epf->epf;
        struct device *dev = &epf->dev;
        struct nvmet_pci_epf_dma_filter filter;
        struct dma_chan *chan;
        dma_cap_mask_t mask;

        mutex_init(&nvme_epf->dma_rx_lock);
        mutex_init(&nvme_epf->dma_tx_lock);

        dma_cap_zero(mask);
        dma_cap_set(DMA_SLAVE, mask);

        filter.dev = epf->epc->dev.parent;
        filter.dma_mask = BIT(DMA_DEV_TO_MEM);

        chan = dma_request_channel(mask, nvmet_pci_epf_dma_filter, &filter);
        if (!chan)
                goto out_dma_no_rx;

        nvme_epf->dma_rx_chan = chan;

        filter.dma_mask = BIT(DMA_MEM_TO_DEV);
        chan = dma_request_channel(mask, nvmet_pci_epf_dma_filter, &filter);
        if (!chan)
                goto out_dma_no_tx;

        nvme_epf->dma_tx_chan = chan;

        nvme_epf->dma_enabled = true;

        dev_dbg(dev, "Using DMA RX channel %s, maximum segment size %u B\n",
                dma_chan_name(nvme_epf->dma_rx_chan),
                dma_get_max_seg_size(dmaengine_get_dma_device(nvme_epf->
                                                              dma_rx_chan)));

        dev_dbg(dev, "Using DMA TX channel %s, maximum segment size %u B\n",
                dma_chan_name(nvme_epf->dma_tx_chan),
                dma_get_max_seg_size(dmaengine_get_dma_device(nvme_epf->
                                                              dma_tx_chan)));

        return;

out_dma_no_tx:
        dma_release_channel(nvme_epf->dma_rx_chan);
        nvme_epf->dma_rx_chan = NULL;

out_dma_no_rx:
        mutex_destroy(&nvme_epf->dma_rx_lock);
        mutex_destroy(&nvme_epf->dma_tx_lock);
        nvme_epf->dma_enabled = false;

        dev_info(&epf->dev, "DMA not supported, falling back to MMIO\n");
}

static void nvmet_pci_epf_deinit_dma(struct nvmet_pci_epf *nvme_epf)
{
        if (!nvme_epf->dma_enabled)
                return;

        dma_release_channel(nvme_epf->dma_tx_chan);
        nvme_epf->dma_tx_chan = NULL;
        dma_release_channel(nvme_epf->dma_rx_chan);
        nvme_epf->dma_rx_chan = NULL;
        mutex_destroy(&nvme_epf->dma_rx_lock);
        mutex_destroy(&nvme_epf->dma_tx_lock);
        nvme_epf->dma_enabled = false;
}

static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf,
                struct nvmet_pci_epf_segment *seg, enum dma_data_direction dir)
{
        struct pci_epf *epf = nvme_epf->epf;
        struct dma_async_tx_descriptor *desc;
        struct dma_slave_config sconf = {};
        struct device *dev = &epf->dev;
        struct device *dma_dev;
        struct dma_chan *chan;
        dma_cookie_t cookie;
        dma_addr_t dma_addr;
        struct mutex *lock;
        int ret;

        switch (dir) {
        case DMA_FROM_DEVICE:
                lock = &nvme_epf->dma_rx_lock;
                chan = nvme_epf->dma_rx_chan;
                sconf.direction = DMA_DEV_TO_MEM;
                sconf.src_addr = seg->pci_addr;
                break;
        case DMA_TO_DEVICE:
                lock = &nvme_epf->dma_tx_lock;
                chan = nvme_epf->dma_tx_chan;
                sconf.direction = DMA_MEM_TO_DEV;
                sconf.dst_addr = seg->pci_addr;
                break;
        default:
                return -EINVAL;
        }

        mutex_lock(lock);

        dma_dev = dmaengine_get_dma_device(chan);
        dma_addr = dma_map_single(dma_dev, seg->buf, seg->length, dir);
        ret = dma_mapping_error(dma_dev, dma_addr);
        if (ret)
                goto unlock;

        ret = dmaengine_slave_config(chan, &sconf);
        if (ret) {
                dev_err(dev, "Failed to configure DMA channel\n");
                goto unmap;
        }

        desc = dmaengine_prep_slave_single(chan, dma_addr, seg->length,
                                           sconf.direction, DMA_CTRL_ACK);
        if (!desc) {
                dev_err(dev, "Failed to prepare DMA\n");
                ret = -EIO;
                goto unmap;
        }

        cookie = dmaengine_submit(desc);
        ret = dma_submit_error(cookie);
        if (ret) {
                dev_err(dev, "Failed to do DMA submit (err=%d)\n", ret);
                goto unmap;
        }

        if (dma_sync_wait(chan, cookie) != DMA_COMPLETE) {
                dev_err(dev, "DMA transfer failed\n");
                ret = -EIO;
        }

        dmaengine_terminate_sync(chan);

unmap:
        dma_unmap_single(dma_dev, dma_addr, seg->length, dir);

unlock:
        mutex_unlock(lock);

        return ret;
}

static int nvmet_pci_epf_mmio_transfer(struct nvmet_pci_epf *nvme_epf,
                struct nvmet_pci_epf_segment *seg, enum dma_data_direction dir)
{
        u64 pci_addr = seg->pci_addr;
        u32 length = seg->length;
        void *buf = seg->buf;
        struct pci_epc_map map;
        int ret = -EINVAL;

        /*
         * Note: MMIO transfers do not need serialization but this is a
         * simple way to avoid using too many mapping windows.
         */
        mutex_lock(&nvme_epf->mmio_lock);

        while (length) {
                ret = nvmet_pci_epf_mem_map(nvme_epf, pci_addr, length, &map);
                if (ret)
                        break;

                switch (dir) {
                case DMA_FROM_DEVICE:
                        memcpy_fromio(buf, map.virt_addr, map.pci_size);
                        break;
                case DMA_TO_DEVICE:
                        memcpy_toio(map.virt_addr, buf, map.pci_size);
                        break;
                default:
                        ret = -EINVAL;
                        goto unlock;
                }

                pci_addr += map.pci_size;
                buf += map.pci_size;
                length -= map.pci_size;

                nvmet_pci_epf_mem_unmap(nvme_epf, &map);
        }

unlock:
        mutex_unlock(&nvme_epf->mmio_lock);

        return ret;
}

static inline int nvmet_pci_epf_transfer_seg(struct nvmet_pci_epf *nvme_epf,
                struct nvmet_pci_epf_segment *seg, enum dma_data_direction dir)
{
        if (nvme_epf->dma_enabled)
                return nvmet_pci_epf_dma_transfer(nvme_epf, seg, dir);

        return nvmet_pci_epf_mmio_transfer(nvme_epf, seg, dir);
}

static inline int nvmet_pci_epf_transfer(struct nvmet_pci_epf_ctrl *ctrl,
                                         void *buf, u64 pci_addr, u32 length,
                                         enum dma_data_direction dir)
{
        struct nvmet_pci_epf_segment seg = {
                .buf = buf,
                .pci_addr = pci_addr,
                .length = length,
        };

        return nvmet_pci_epf_transfer_seg(ctrl->nvme_epf, &seg, dir);
}

static int nvmet_pci_epf_alloc_irq_vectors(struct nvmet_pci_epf_ctrl *ctrl)
{
        ctrl->irq_vectors = kzalloc_objs(struct nvmet_pci_epf_irq_vector,
                                         ctrl->nr_queues);
        if (!ctrl->irq_vectors)
                return -ENOMEM;

        mutex_init(&ctrl->irq_lock);

        return 0;
}

static void nvmet_pci_epf_free_irq_vectors(struct nvmet_pci_epf_ctrl *ctrl)
{
        if (ctrl->irq_vectors) {
                mutex_destroy(&ctrl->irq_lock);
                kfree(ctrl->irq_vectors);
                ctrl->irq_vectors = NULL;
        }
}

static struct nvmet_pci_epf_irq_vector *
nvmet_pci_epf_find_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, u16 vector)
{
        struct nvmet_pci_epf_irq_vector *iv;
        int i;

        lockdep_assert_held(&ctrl->irq_lock);

        for (i = 0; i < ctrl->nr_queues; i++) {
                iv = &ctrl->irq_vectors[i];
                if (iv->ref && iv->vector == vector)
                        return iv;
        }

        return NULL;
}

static struct nvmet_pci_epf_irq_vector *
nvmet_pci_epf_add_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, u16 vector)
{
        struct nvmet_pci_epf_irq_vector *iv;
        int i;

        mutex_lock(&ctrl->irq_lock);

        iv = nvmet_pci_epf_find_irq_vector(ctrl, vector);
        if (iv) {
                iv->ref++;
                goto unlock;
        }

        for (i = 0; i < ctrl->nr_queues; i++) {
                iv = &ctrl->irq_vectors[i];
                if (!iv->ref)
                        break;
        }

        if (WARN_ON_ONCE(!iv))
                goto unlock;

        iv->ref = 1;
        iv->vector = vector;
        iv->nr_irqs = 0;

unlock:
        mutex_unlock(&ctrl->irq_lock);

        return iv;
}

static void nvmet_pci_epf_remove_irq_vector(struct nvmet_pci_epf_ctrl *ctrl,
                                            u16 vector)
{
        struct nvmet_pci_epf_irq_vector *iv;

        mutex_lock(&ctrl->irq_lock);

        iv = nvmet_pci_epf_find_irq_vector(ctrl, vector);
        if (iv) {
                iv->ref--;
                if (!iv->ref) {
                        iv->vector = 0;
                        iv->nr_irqs = 0;
                }
        }

        mutex_unlock(&ctrl->irq_lock);
}

static bool nvmet_pci_epf_should_raise_irq(struct nvmet_pci_epf_ctrl *ctrl,
                struct nvmet_pci_epf_queue *cq, bool force)
{
        struct nvmet_pci_epf_irq_vector *iv = cq->iv;
        bool ret;

        /* IRQ coalescing for the admin queue is not allowed. */
        if (!cq->qid)
                return true;

        if (iv->cd)
                return true;

        if (force) {
                ret = iv->nr_irqs > 0;
        } else {
                iv->nr_irqs++;
                ret = iv->nr_irqs >= ctrl->irq_vector_threshold;
        }
        if (ret)
                iv->nr_irqs = 0;

        return ret;
}

static void nvmet_pci_epf_raise_irq(struct nvmet_pci_epf_ctrl *ctrl,
                struct nvmet_pci_epf_queue *cq, bool force)
{
        struct nvmet_pci_epf *nvme_epf = ctrl->nvme_epf;
        struct pci_epf *epf = nvme_epf->epf;
        int ret = 0;

        if (!test_bit(NVMET_PCI_EPF_Q_LIVE, &cq->flags) ||
            !test_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags))
                return;

        mutex_lock(&ctrl->irq_lock);

        if (!nvmet_pci_epf_should_raise_irq(ctrl, cq, force))
                goto unlock;

        switch (nvme_epf->irq_type) {
        case PCI_IRQ_MSIX:
        case PCI_IRQ_MSI:
                /*
                 * If we fail to raise an MSI or MSI-X interrupt, it is likely
                 * because the host is using legacy INTX IRQs (e.g. BIOS,
                 * grub), but we can fallback to the INTX type only if the
                 * endpoint controller supports this type.
                 */
                ret = pci_epc_raise_irq(epf->epc, epf->func_no, epf->vfunc_no,
                                        nvme_epf->irq_type, cq->vector + 1);
                if (!ret || !nvme_epf->epc_features->intx_capable)
                        break;
                fallthrough;
        case PCI_IRQ_INTX:
                ret = pci_epc_raise_irq(epf->epc, epf->func_no, epf->vfunc_no,
                                        PCI_IRQ_INTX, 0);
                break;
        default:
                WARN_ON_ONCE(1);
                ret = -EINVAL;
                break;
        }

        if (ret)
                dev_err_ratelimited(ctrl->dev,
                                    "CQ[%u]: Failed to raise IRQ (err=%d)\n",
                                    cq->qid, ret);

unlock:
        mutex_unlock(&ctrl->irq_lock);
}

static inline const char *nvmet_pci_epf_iod_name(struct nvmet_pci_epf_iod *iod)
{
        return nvme_opcode_str(iod->sq->qid, iod->cmd.common.opcode);
}

static void nvmet_pci_epf_exec_iod_work(struct work_struct *work);

static struct nvmet_pci_epf_iod *
nvmet_pci_epf_alloc_iod(struct nvmet_pci_epf_queue *sq)
{
        struct nvmet_pci_epf_ctrl *ctrl = sq->ctrl;
        struct nvmet_pci_epf_iod *iod;

        iod = mempool_alloc(&ctrl->iod_pool, GFP_KERNEL);
        if (unlikely(!iod))
                return NULL;

        memset(iod, 0, sizeof(*iod));
        iod->req.cmd = &iod->cmd;
        iod->req.cqe = &iod->cqe;
        iod->req.port = ctrl->port;
        iod->ctrl = ctrl;
        iod->sq = sq;
        iod->cq = &ctrl->cq[sq->qid];
        INIT_LIST_HEAD(&iod->link);
        iod->dma_dir = DMA_NONE;
        INIT_WORK(&iod->work, nvmet_pci_epf_exec_iod_work);
        init_completion(&iod->done);

        return iod;
}

/*
 * Allocate or grow a command table of PCI segments.
 */
static int nvmet_pci_epf_alloc_iod_data_segs(struct nvmet_pci_epf_iod *iod,
                                             int nsegs)
{
        struct nvmet_pci_epf_segment *segs;
        int nr_segs = iod->nr_data_segs + nsegs;

        segs = krealloc(iod->data_segs,
                        nr_segs * sizeof(struct nvmet_pci_epf_segment),
                        GFP_KERNEL | __GFP_ZERO);
        if (!segs)
                return -ENOMEM;

        iod->nr_data_segs = nr_segs;
        iod->data_segs = segs;

        return 0;
}

static void nvmet_pci_epf_free_iod(struct nvmet_pci_epf_iod *iod)
{
        int i;

        if (iod->data_segs) {
                for (i = 0; i < iod->nr_data_segs; i++)
                        kfree(iod->data_segs[i].buf);
                if (iod->data_segs != &iod->data_seg)
                        kfree(iod->data_segs);
        }
        if (iod->data_sgt.nents > 1)
                sg_free_table(&iod->data_sgt);
        mempool_free(iod, &iod->ctrl->iod_pool);
}

static int nvmet_pci_epf_transfer_iod_data(struct nvmet_pci_epf_iod *iod)
{
        struct nvmet_pci_epf *nvme_epf = iod->ctrl->nvme_epf;
        struct nvmet_pci_epf_segment *seg = &iod->data_segs[0];
        int i, ret;

        /* Split the data transfer according to the PCI segments. */
        for (i = 0; i < iod->nr_data_segs; i++, seg++) {
                ret = nvmet_pci_epf_transfer_seg(nvme_epf, seg, iod->dma_dir);
                if (ret) {
                        iod->status = NVME_SC_DATA_XFER_ERROR | NVME_STATUS_DNR;
                        return ret;
                }
        }

        return 0;
}

static inline u32 nvmet_pci_epf_prp_ofst(struct nvmet_pci_epf_ctrl *ctrl,
                                         u64 prp)
{
        return prp & ctrl->mps_mask;
}

static inline size_t nvmet_pci_epf_prp_size(struct nvmet_pci_epf_ctrl *ctrl,
                                            u64 prp)
{
        return ctrl->mps - nvmet_pci_epf_prp_ofst(ctrl, prp);
}

/*
 * Transfer a PRP list from the host and return the number of prps.
 */
static int nvmet_pci_epf_get_prp_list(struct nvmet_pci_epf_ctrl *ctrl, u64 prp,
                                      size_t xfer_len, __le64 *prps)
{
        size_t nr_prps = (xfer_len + ctrl->mps_mask) >> ctrl->mps_shift;
        u32 length;
        int ret;

        /*
         * Compute the number of PRPs required for the number of bytes to
         * transfer (xfer_len). If this number overflows the memory page size
         * with the PRP list pointer specified, only return the space available
         * in the memory page, the last PRP in there will be a PRP list pointer
         * to the remaining PRPs.
         */
        length = min(nvmet_pci_epf_prp_size(ctrl, prp), nr_prps << 3);
        ret = nvmet_pci_epf_transfer(ctrl, prps, prp, length, DMA_FROM_DEVICE);
        if (ret)
                return ret;

        return length >> 3;
}

static int nvmet_pci_epf_iod_parse_prp_list(struct nvmet_pci_epf_ctrl *ctrl,
                                            struct nvmet_pci_epf_iod *iod)
{
        struct nvme_command *cmd = &iod->cmd;
        struct nvmet_pci_epf_segment *seg;
        size_t size = 0, ofst, prp_size, xfer_len;
        size_t transfer_len = iod->data_len;
        int nr_segs, nr_prps = 0;
        u64 pci_addr, prp;
        int i = 0, ret;
        __le64 *prps;

        prps = kzalloc(ctrl->mps, GFP_KERNEL);
        if (!prps)
                goto err_internal;

        /*
         * Allocate PCI segments for the command: this considers the worst case
         * scenario where all prps are discontiguous, so get as many segments
         * as we can have prps. In practice, most of the time, we will have
         * far less PCI segments than prps.
         */
        prp = le64_to_cpu(cmd->common.dptr.prp1);
        if (!prp)
                goto err_invalid_field;

        ofst = nvmet_pci_epf_prp_ofst(ctrl, prp);
        nr_segs = (transfer_len + ofst + ctrl->mps - 1) >> ctrl->mps_shift;

        ret = nvmet_pci_epf_alloc_iod_data_segs(iod, nr_segs);
        if (ret)
                goto err_internal;

        /* Set the first segment using prp1. */
        seg = &iod->data_segs[0];
        seg->pci_addr = prp;
        seg->length = nvmet_pci_epf_prp_size(ctrl, prp);

        size = seg->length;
        pci_addr = prp + size;
        nr_segs = 1;

        /*
         * Now build the PCI address segments using the PRP lists, starting
         * from prp2.
         */
        prp = le64_to_cpu(cmd->common.dptr.prp2);
        if (!prp)
                goto err_invalid_field;

        while (size < transfer_len) {
                xfer_len = transfer_len - size;

                if (!nr_prps) {
                        nr_prps = nvmet_pci_epf_get_prp_list(ctrl, prp,
                                                             xfer_len, prps);
                        if (nr_prps < 0)
                                goto err_internal;

                        i = 0;
                        ofst = 0;
                }

                /* Current entry */
                prp = le64_to_cpu(prps[i]);
                if (!prp)
                        goto err_invalid_field;

                /* Did we reach the last PRP entry of the list? */
                if (xfer_len > ctrl->mps && i == nr_prps - 1) {
                        /* We need more PRPs: PRP is a list pointer. */
                        nr_prps = 0;
                        continue;
                }

                /* Only the first PRP is allowed to have an offset. */
                if (nvmet_pci_epf_prp_ofst(ctrl, prp))
                        goto err_invalid_offset;

                if (prp != pci_addr) {
                        /* Discontiguous prp: new segment. */
                        nr_segs++;
                        if (WARN_ON_ONCE(nr_segs > iod->nr_data_segs))
                                goto err_internal;

                        seg++;
                        seg->pci_addr = prp;
                        seg->length = 0;
                        pci_addr = prp;
                }

                prp_size = min_t(size_t, ctrl->mps, xfer_len);
                seg->length += prp_size;
                pci_addr += prp_size;
                size += prp_size;

                i++;
        }

        iod->nr_data_segs = nr_segs;
        ret = 0;

        if (size != transfer_len) {
                dev_err(ctrl->dev,
                        "PRPs transfer length mismatch: got %zu B, need %zu B\n",
                        size, transfer_len);
                goto err_internal;
        }

        kfree(prps);

        return 0;

err_invalid_offset:
        dev_err(ctrl->dev, "PRPs list invalid offset\n");
        iod->status = NVME_SC_PRP_INVALID_OFFSET | NVME_STATUS_DNR;
        goto err;

err_invalid_field:
        dev_err(ctrl->dev, "PRPs list invalid field\n");
        iod->status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
        goto err;

err_internal:
        dev_err(ctrl->dev, "PRPs list internal error\n");
        iod->status = NVME_SC_INTERNAL | NVME_STATUS_DNR;

err:
        kfree(prps);
        return -EINVAL;
}

static int nvmet_pci_epf_iod_parse_prp_simple(struct nvmet_pci_epf_ctrl *ctrl,
                                              struct nvmet_pci_epf_iod *iod)
{
        struct nvme_command *cmd = &iod->cmd;
        size_t transfer_len = iod->data_len;
        int ret, nr_segs = 1;
        u64 prp1, prp2 = 0;
        size_t prp1_size;

        prp1 = le64_to_cpu(cmd->common.dptr.prp1);
        prp1_size = nvmet_pci_epf_prp_size(ctrl, prp1);

        /* For commands crossing a page boundary, we should have prp2. */
        if (transfer_len > prp1_size) {
                prp2 = le64_to_cpu(cmd->common.dptr.prp2);
                if (!prp2) {
                        iod->status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                        return -EINVAL;
                }
                if (nvmet_pci_epf_prp_ofst(ctrl, prp2)) {
                        iod->status =
                                NVME_SC_PRP_INVALID_OFFSET | NVME_STATUS_DNR;
                        return -EINVAL;
                }
                if (prp2 != prp1 + prp1_size)
                        nr_segs = 2;
        }

        if (nr_segs == 1) {
                iod->nr_data_segs = 1;
                iod->data_segs = &iod->data_seg;
                iod->data_segs[0].pci_addr = prp1;
                iod->data_segs[0].length = transfer_len;
                return 0;
        }

        ret = nvmet_pci_epf_alloc_iod_data_segs(iod, nr_segs);
        if (ret) {
                iod->status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
                return ret;
        }

        iod->data_segs[0].pci_addr = prp1;
        iod->data_segs[0].length = prp1_size;
        iod->data_segs[1].pci_addr = prp2;
        iod->data_segs[1].length = transfer_len - prp1_size;

        return 0;
}

static int nvmet_pci_epf_iod_parse_prps(struct nvmet_pci_epf_iod *iod)
{
        struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl;
        u64 prp1 = le64_to_cpu(iod->cmd.common.dptr.prp1);
        size_t ofst;

        /* Get the PCI address segments for the command using its PRPs. */
        ofst = nvmet_pci_epf_prp_ofst(ctrl, prp1);
        if (ofst & 0x3) {
                iod->status = NVME_SC_PRP_INVALID_OFFSET | NVME_STATUS_DNR;
                return -EINVAL;
        }

        if (iod->data_len + ofst <= ctrl->mps * 2)
                return nvmet_pci_epf_iod_parse_prp_simple(ctrl, iod);

        return nvmet_pci_epf_iod_parse_prp_list(ctrl, iod);
}

/*
 * Transfer an SGL segment from the host and return the number of data
 * descriptors and the next segment descriptor, if any.
 */
static struct nvme_sgl_desc *
nvmet_pci_epf_get_sgl_segment(struct nvmet_pci_epf_ctrl *ctrl,
                              struct nvme_sgl_desc *desc, unsigned int *nr_sgls)
{
        struct nvme_sgl_desc *sgls;
        u32 length = le32_to_cpu(desc->length);
        int nr_descs, ret;
        void *buf;

        buf = kmalloc(length, GFP_KERNEL);
        if (!buf)
                return NULL;

        ret = nvmet_pci_epf_transfer(ctrl, buf, le64_to_cpu(desc->addr), length,
                                     DMA_FROM_DEVICE);
        if (ret) {
                kfree(buf);
                return NULL;
        }

        sgls = buf;
        nr_descs = length / sizeof(struct nvme_sgl_desc);
        if (sgls[nr_descs - 1].type == (NVME_SGL_FMT_SEG_DESC << 4) ||
            sgls[nr_descs - 1].type == (NVME_SGL_FMT_LAST_SEG_DESC << 4)) {
                /*
                 * We have another SGL segment following this one: do not count
                 * it as a regular data SGL descriptor and return it to the
                 * caller.
                 */
                *desc = sgls[nr_descs - 1];
                nr_descs--;
        } else {
                /* We do not have another SGL segment after this one. */
                desc->length = 0;
        }

        *nr_sgls = nr_descs;

        return sgls;
}

static int nvmet_pci_epf_iod_parse_sgl_segments(struct nvmet_pci_epf_ctrl *ctrl,
                                                struct nvmet_pci_epf_iod *iod)
{
        struct nvme_command *cmd = &iod->cmd;
        struct nvme_sgl_desc seg = cmd->common.dptr.sgl;
        struct nvme_sgl_desc *sgls = NULL;
        int n = 0, i, nr_sgls;
        int ret;

        /*
         * We do not support inline data nor keyed SGLs, so we should be seeing
         * only segment descriptors.
         */
        if (seg.type != (NVME_SGL_FMT_SEG_DESC << 4) &&
            seg.type != (NVME_SGL_FMT_LAST_SEG_DESC << 4)) {
                iod->status = NVME_SC_SGL_INVALID_TYPE | NVME_STATUS_DNR;
                return -EIO;
        }

        while (seg.length) {
                sgls = nvmet_pci_epf_get_sgl_segment(ctrl, &seg, &nr_sgls);
                if (!sgls) {
                        iod->status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
                        return -EIO;
                }

                /* Grow the PCI segment table as needed. */
                ret = nvmet_pci_epf_alloc_iod_data_segs(iod, nr_sgls);
                if (ret) {
                        iod->status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
                        goto out;
                }

                /*
                 * Parse the SGL descriptors to build the PCI segment table,
                 * checking the descriptor type as we go.
                 */
                for (i = 0; i < nr_sgls; i++) {
                        if (sgls[i].type != (NVME_SGL_FMT_DATA_DESC << 4)) {
                                iod->status = NVME_SC_SGL_INVALID_TYPE |
                                        NVME_STATUS_DNR;
                                goto out;
                        }
                        iod->data_segs[n].pci_addr = le64_to_cpu(sgls[i].addr);
                        iod->data_segs[n].length = le32_to_cpu(sgls[i].length);
                        n++;
                }

                kfree(sgls);
        }

 out:
        if (iod->status != NVME_SC_SUCCESS) {
                kfree(sgls);
                return -EIO;
        }

        return 0;
}

static int nvmet_pci_epf_iod_parse_sgls(struct nvmet_pci_epf_iod *iod)
{
        struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl;
        struct nvme_sgl_desc *sgl = &iod->cmd.common.dptr.sgl;

        if (sgl->type == (NVME_SGL_FMT_DATA_DESC << 4)) {
                /* Single data descriptor case. */
                iod->nr_data_segs = 1;
                iod->data_segs = &iod->data_seg;
                iod->data_seg.pci_addr = le64_to_cpu(sgl->addr);
                iod->data_seg.length = le32_to_cpu(sgl->length);
                return 0;
        }

        return nvmet_pci_epf_iod_parse_sgl_segments(ctrl, iod);
}

static int nvmet_pci_epf_alloc_iod_data_buf(struct nvmet_pci_epf_iod *iod)
{
        struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl;
        struct nvmet_req *req = &iod->req;
        struct nvmet_pci_epf_segment *seg;
        struct scatterlist *sg;
        int ret, i;

        if (iod->data_len > ctrl->mdts) {
                iod->status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                return -EINVAL;
        }

        /*
         * Get the PCI address segments for the command data buffer using either
         * its SGLs or PRPs.
         */
        if (iod->cmd.common.flags & NVME_CMD_SGL_ALL)
                ret = nvmet_pci_epf_iod_parse_sgls(iod);
        else
                ret = nvmet_pci_epf_iod_parse_prps(iod);
        if (ret)
                return ret;

        /* Get a command buffer using SGLs matching the PCI segments. */
        if (iod->nr_data_segs == 1) {
                sg_init_table(&iod->data_sgl, 1);
                iod->data_sgt.sgl = &iod->data_sgl;
                iod->data_sgt.nents = 1;
                iod->data_sgt.orig_nents = 1;
        } else {
                ret = sg_alloc_table(&iod->data_sgt, iod->nr_data_segs,
                                     GFP_KERNEL);
                if (ret)
                        goto err_nomem;
        }

        for_each_sgtable_sg(&iod->data_sgt, sg, i) {
                seg = &iod->data_segs[i];
                seg->buf = kmalloc(seg->length, GFP_KERNEL);
                if (!seg->buf)
                        goto err_nomem;
                sg_set_buf(sg, seg->buf, seg->length);
        }

        req->transfer_len = iod->data_len;
        req->sg = iod->data_sgt.sgl;
        req->sg_cnt = iod->data_sgt.nents;

        return 0;

err_nomem:
        iod->status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
        return -ENOMEM;
}

static void nvmet_pci_epf_complete_iod(struct nvmet_pci_epf_iod *iod)
{
        struct nvmet_pci_epf_queue *cq = iod->cq;
        unsigned long flags;

        /* Print an error message for failed commands, except AENs. */
        iod->status = le16_to_cpu(iod->cqe.status) >> 1;
        if (iod->status && iod->cmd.common.opcode != nvme_admin_async_event)
                dev_err(iod->ctrl->dev,
                        "CQ[%d]: Command %s (0x%x) status 0x%0x\n",
                        iod->sq->qid, nvmet_pci_epf_iod_name(iod),
                        iod->cmd.common.opcode, iod->status);

        /*
         * Add the command to the list of completed commands and schedule the
         * CQ work.
         */
        spin_lock_irqsave(&cq->lock, flags);
        list_add_tail(&iod->link, &cq->list);
        queue_delayed_work(system_highpri_wq, &cq->work, 0);
        spin_unlock_irqrestore(&cq->lock, flags);
}

static void nvmet_pci_epf_drain_queue(struct nvmet_pci_epf_queue *queue)
{
        struct nvmet_pci_epf_iod *iod;
        unsigned long flags;

        spin_lock_irqsave(&queue->lock, flags);
        while (!list_empty(&queue->list)) {
                iod = list_first_entry(&queue->list, struct nvmet_pci_epf_iod,
                                       link);
                list_del_init(&iod->link);
                nvmet_pci_epf_free_iod(iod);
        }
        spin_unlock_irqrestore(&queue->lock, flags);
}

static int nvmet_pci_epf_add_port(struct nvmet_port *port)
{
        mutex_lock(&nvmet_pci_epf_ports_mutex);
        list_add_tail(&port->entry, &nvmet_pci_epf_ports);
        mutex_unlock(&nvmet_pci_epf_ports_mutex);
        return 0;
}

static void nvmet_pci_epf_remove_port(struct nvmet_port *port)
{
        mutex_lock(&nvmet_pci_epf_ports_mutex);
        list_del_init(&port->entry);
        mutex_unlock(&nvmet_pci_epf_ports_mutex);
}

static struct nvmet_port *
nvmet_pci_epf_find_port(struct nvmet_pci_epf_ctrl *ctrl, __le16 portid)
{
        struct nvmet_port *p, *port = NULL;

        mutex_lock(&nvmet_pci_epf_ports_mutex);
        list_for_each_entry(p, &nvmet_pci_epf_ports, entry) {
                if (p->disc_addr.portid == portid) {
                        port = p;
                        break;
                }
        }
        mutex_unlock(&nvmet_pci_epf_ports_mutex);

        return port;
}

static void nvmet_pci_epf_queue_response(struct nvmet_req *req)
{
        struct nvmet_pci_epf_iod *iod =
                container_of(req, struct nvmet_pci_epf_iod, req);

        iod->status = le16_to_cpu(req->cqe->status) >> 1;

        /*
         * If the command failed or we have no data to transfer, complete the
         * command immediately.
         */
        if (iod->status || !iod->data_len || iod->dma_dir != DMA_TO_DEVICE) {
                nvmet_pci_epf_complete_iod(iod);
                return;
        }

        complete(&iod->done);
}

static u8 nvmet_pci_epf_get_mdts(const struct nvmet_ctrl *tctrl)
{
        struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata;
        int page_shift = NVME_CAP_MPSMIN(tctrl->cap) + 12;

        return ilog2(ctrl->mdts) - page_shift;
}

static u16 nvmet_pci_epf_create_cq(struct nvmet_ctrl *tctrl,
                u16 cqid, u16 flags, u16 qsize, u64 pci_addr, u16 vector)
{
        struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata;
        struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid];
        u16 status;
        int ret;

        if (test_bit(NVMET_PCI_EPF_Q_LIVE, &cq->flags))
                return NVME_SC_QID_INVALID | NVME_STATUS_DNR;

        if (!(flags & NVME_QUEUE_PHYS_CONTIG))
                return NVME_SC_INVALID_QUEUE | NVME_STATUS_DNR;

        cq->pci_addr = pci_addr;
        cq->qid = cqid;
        cq->depth = qsize + 1;
        cq->vector = vector;
        cq->head = 0;
        cq->tail = 0;
        cq->phase = 1;
        cq->db = NVME_REG_DBS + (((cqid * 2) + 1) * sizeof(u32));
        nvmet_pci_epf_bar_write32(ctrl, cq->db, 0);

        if (!cqid)
                cq->qes = sizeof(struct nvme_completion);
        else
                cq->qes = ctrl->io_cqes;
        cq->pci_size = cq->qes * cq->depth;

        if (flags & NVME_CQ_IRQ_ENABLED) {
                cq->iv = nvmet_pci_epf_add_irq_vector(ctrl, vector);
                if (!cq->iv)
                        return NVME_SC_INTERNAL | NVME_STATUS_DNR;
                set_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags);
        }

        status = nvmet_cq_create(tctrl, &cq->nvme_cq, cqid, cq->depth);
        if (status != NVME_SC_SUCCESS)
                goto err;

        /*
         * Map the CQ PCI address space and since PCI endpoint controllers may
         * return a partial mapping, check that the mapping is large enough.
         */
        ret = nvmet_pci_epf_mem_map(ctrl->nvme_epf, cq->pci_addr, cq->pci_size,
                                    &cq->pci_map);
        if (ret) {
                dev_err(ctrl->dev, "Failed to map CQ %u (err=%d)\n",
                        cq->qid, ret);
                goto err_internal;
        }

        if (cq->pci_map.pci_size < cq->pci_size) {
                dev_err(ctrl->dev, "Invalid partial mapping of queue %u\n",
                        cq->qid);
                goto err_unmap_queue;
        }

        set_bit(NVMET_PCI_EPF_Q_LIVE, &cq->flags);

        if (test_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags))
                dev_dbg(ctrl->dev,
                        "CQ[%u]: %u entries of %zu B, IRQ vector %u\n",
                        cqid, qsize, cq->qes, cq->vector);
        else
                dev_dbg(ctrl->dev,
                        "CQ[%u]: %u entries of %zu B, IRQ disabled\n",
                        cqid, qsize, cq->qes);

        return NVME_SC_SUCCESS;

err_unmap_queue:
        nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map);
err_internal:
        status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
err:
        if (test_and_clear_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags))
                nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector);
        return status;
}

static u16 nvmet_pci_epf_delete_cq(struct nvmet_ctrl *tctrl, u16 cqid)
{
        struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata;
        struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid];

        if (!test_and_clear_bit(NVMET_PCI_EPF_Q_LIVE, &cq->flags))
                return NVME_SC_QID_INVALID | NVME_STATUS_DNR;

        cancel_delayed_work_sync(&cq->work);
        nvmet_pci_epf_drain_queue(cq);
        if (test_and_clear_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags))
                nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector);
        nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map);
        nvmet_cq_put(&cq->nvme_cq);

        return NVME_SC_SUCCESS;
}

static u16 nvmet_pci_epf_create_sq(struct nvmet_ctrl *tctrl,
                u16 sqid, u16 cqid, u16 flags, u16 qsize, u64 pci_addr)
{
        struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata;
        struct nvmet_pci_epf_queue *sq = &ctrl->sq[sqid];
        struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid];
        u16 status;

        if (test_bit(NVMET_PCI_EPF_Q_LIVE, &sq->flags))
                return NVME_SC_QID_INVALID | NVME_STATUS_DNR;

        if (!(flags & NVME_QUEUE_PHYS_CONTIG))
                return NVME_SC_INVALID_QUEUE | NVME_STATUS_DNR;

        sq->pci_addr = pci_addr;
        sq->qid = sqid;
        sq->depth = qsize + 1;
        sq->head = 0;
        sq->tail = 0;
        sq->phase = 0;
        sq->db = NVME_REG_DBS + (sqid * 2 * sizeof(u32));
        nvmet_pci_epf_bar_write32(ctrl, sq->db, 0);
        if (!sqid)
                sq->qes = 1UL << NVME_ADM_SQES;
        else
                sq->qes = ctrl->io_sqes;
        sq->pci_size = sq->qes * sq->depth;

        status = nvmet_sq_create(tctrl, &sq->nvme_sq, &cq->nvme_cq, sqid,
                        sq->depth);
        if (status != NVME_SC_SUCCESS)
                return status;

        sq->iod_wq = alloc_workqueue("sq%d_wq", WQ_UNBOUND,
                                min_t(int, sq->depth, WQ_MAX_ACTIVE), sqid);
        if (!sq->iod_wq) {
                dev_err(ctrl->dev, "Failed to create SQ %d work queue\n", sqid);
                status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
                goto out_destroy_sq;
        }

        set_bit(NVMET_PCI_EPF_Q_LIVE, &sq->flags);

        dev_dbg(ctrl->dev, "SQ[%u]: %u entries of %zu B\n",
                sqid, qsize, sq->qes);

        return NVME_SC_SUCCESS;

out_destroy_sq:
        nvmet_sq_destroy(&sq->nvme_sq);
        return status;
}

static u16 nvmet_pci_epf_delete_sq(struct nvmet_ctrl *tctrl, u16 sqid)
{
        struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata;
        struct nvmet_pci_epf_queue *sq = &ctrl->sq[sqid];

        if (!test_and_clear_bit(NVMET_PCI_EPF_Q_LIVE, &sq->flags))
                return NVME_SC_QID_INVALID | NVME_STATUS_DNR;

        destroy_workqueue(sq->iod_wq);
        sq->iod_wq = NULL;

        nvmet_pci_epf_drain_queue(sq);

        if (sq->nvme_sq.ctrl)
                nvmet_sq_destroy(&sq->nvme_sq);

        return NVME_SC_SUCCESS;
}

static u16 nvmet_pci_epf_get_feat(const struct nvmet_ctrl *tctrl,
                                  u8 feat, void *data)
{
        struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata;
        struct nvmet_feat_arbitration *arb;
        struct nvmet_feat_irq_coalesce *irqc;
        struct nvmet_feat_irq_config *irqcfg;
        struct nvmet_pci_epf_irq_vector *iv;
        u16 status;

        switch (feat) {
        case NVME_FEAT_ARBITRATION:
                arb = data;
                if (!ctrl->sq_ab)
                        arb->ab = 0x7;
                else
                        arb->ab = ilog2(ctrl->sq_ab);
                return NVME_SC_SUCCESS;

        case NVME_FEAT_IRQ_COALESCE:
                irqc = data;
                irqc->thr = ctrl->irq_vector_threshold;
                irqc->time = 0;
                return NVME_SC_SUCCESS;

        case NVME_FEAT_IRQ_CONFIG:
                irqcfg = data;
                mutex_lock(&ctrl->irq_lock);
                iv = nvmet_pci_epf_find_irq_vector(ctrl, irqcfg->iv);
                if (iv) {
                        irqcfg->cd = iv->cd;
                        status = NVME_SC_SUCCESS;
                } else {
                        status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                }
                mutex_unlock(&ctrl->irq_lock);
                return status;

        default:
                return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
        }
}

static u16 nvmet_pci_epf_set_feat(const struct nvmet_ctrl *tctrl,
                                  u8 feat, void *data)
{
        struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata;
        struct nvmet_feat_arbitration *arb;
        struct nvmet_feat_irq_coalesce *irqc;
        struct nvmet_feat_irq_config *irqcfg;
        struct nvmet_pci_epf_irq_vector *iv;
        u16 status;

        switch (feat) {
        case NVME_FEAT_ARBITRATION:
                arb = data;
                if (arb->ab == 0x7)
                        ctrl->sq_ab = 0;
                else
                        ctrl->sq_ab = 1 << arb->ab;
                return NVME_SC_SUCCESS;

        case NVME_FEAT_IRQ_COALESCE:
                /*
                 * Since we do not implement precise IRQ coalescing timing,
                 * ignore the time field.
                 */
                irqc = data;
                ctrl->irq_vector_threshold = irqc->thr + 1;
                return NVME_SC_SUCCESS;

        case NVME_FEAT_IRQ_CONFIG:
                irqcfg = data;
                mutex_lock(&ctrl->irq_lock);
                iv = nvmet_pci_epf_find_irq_vector(ctrl, irqcfg->iv);
                if (iv) {
                        iv->cd = irqcfg->cd;
                        status = NVME_SC_SUCCESS;
                } else {
                        status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
                }
                mutex_unlock(&ctrl->irq_lock);
                return status;

        default:
                return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
        }
}

static const struct nvmet_fabrics_ops nvmet_pci_epf_fabrics_ops = {
        .owner          = THIS_MODULE,
        .type           = NVMF_TRTYPE_PCI,
        .add_port       = nvmet_pci_epf_add_port,
        .remove_port    = nvmet_pci_epf_remove_port,
        .queue_response = nvmet_pci_epf_queue_response,
        .get_mdts       = nvmet_pci_epf_get_mdts,
        .create_cq      = nvmet_pci_epf_create_cq,
        .delete_cq      = nvmet_pci_epf_delete_cq,
        .create_sq      = nvmet_pci_epf_create_sq,
        .delete_sq      = nvmet_pci_epf_delete_sq,
        .get_feature    = nvmet_pci_epf_get_feat,
        .set_feature    = nvmet_pci_epf_set_feat,
};

static void nvmet_pci_epf_cq_work(struct work_struct *work);

static void nvmet_pci_epf_init_queue(struct nvmet_pci_epf_ctrl *ctrl,
                                     unsigned int qid, bool sq)
{
        struct nvmet_pci_epf_queue *queue;

        if (sq) {
                queue = &ctrl->sq[qid];
        } else {
                queue = &ctrl->cq[qid];
                INIT_DELAYED_WORK(&queue->work, nvmet_pci_epf_cq_work);
        }
        queue->ctrl = ctrl;
        queue->qid = qid;
        spin_lock_init(&queue->lock);
        INIT_LIST_HEAD(&queue->list);
}

static int nvmet_pci_epf_alloc_queues(struct nvmet_pci_epf_ctrl *ctrl)
{
        unsigned int qid;

        ctrl->sq = kzalloc_objs(struct nvmet_pci_epf_queue, ctrl->nr_queues);
        if (!ctrl->sq)
                return -ENOMEM;

        ctrl->cq = kzalloc_objs(struct nvmet_pci_epf_queue, ctrl->nr_queues);
        if (!ctrl->cq) {
                kfree(ctrl->sq);
                ctrl->sq = NULL;
                return -ENOMEM;
        }

        for (qid = 0; qid < ctrl->nr_queues; qid++) {
                nvmet_pci_epf_init_queue(ctrl, qid, true);
                nvmet_pci_epf_init_queue(ctrl, qid, false);
        }

        return 0;
}

static void nvmet_pci_epf_free_queues(struct nvmet_pci_epf_ctrl *ctrl)
{
        kfree(ctrl->sq);
        ctrl->sq = NULL;
        kfree(ctrl->cq);
        ctrl->cq = NULL;
}

static void nvmet_pci_epf_exec_iod_work(struct work_struct *work)
{
        struct nvmet_pci_epf_iod *iod =
                container_of(work, struct nvmet_pci_epf_iod, work);
        struct nvmet_req *req = &iod->req;
        int ret;

        if (!iod->ctrl->link_up) {
                nvmet_pci_epf_free_iod(iod);
                return;
        }

        if (!test_bit(NVMET_PCI_EPF_Q_LIVE, &iod->sq->flags)) {
                iod->status = NVME_SC_QID_INVALID | NVME_STATUS_DNR;
                goto complete;
        }

        /*
         * If nvmet_req_init() fails (e.g., unsupported opcode) it will call
         * __nvmet_req_complete() internally which will call
         * nvmet_pci_epf_queue_response() and will complete the command directly.
         */
        if (!nvmet_req_init(req, &iod->sq->nvme_sq, &nvmet_pci_epf_fabrics_ops))
                return;

        iod->data_len = nvmet_req_transfer_len(req);
        if (iod->data_len) {
                /*
                 * Get the data DMA transfer direction. Here "device" means the
                 * PCI root-complex host.
                 */
                if (nvme_is_write(&iod->cmd))
                        iod->dma_dir = DMA_FROM_DEVICE;
                else
                        iod->dma_dir = DMA_TO_DEVICE;

                /*
                 * Setup the command data buffer and get the command data from
                 * the host if needed.
                 */
                ret = nvmet_pci_epf_alloc_iod_data_buf(iod);
                if (!ret && iod->dma_dir == DMA_FROM_DEVICE)
                        ret = nvmet_pci_epf_transfer_iod_data(iod);
                if (ret) {
                        nvmet_req_uninit(req);
                        goto complete;
                }
        }

        req->execute(req);

        /*
         * If we do not have data to transfer after the command execution
         * finishes, nvmet_pci_epf_queue_response() will complete the command
         * directly. No need to wait for the completion in this case.
         */
        if (!iod->data_len || iod->dma_dir != DMA_TO_DEVICE)
                return;

        wait_for_completion(&iod->done);

        if (iod->status != NVME_SC_SUCCESS)
                return;

        WARN_ON_ONCE(!iod->data_len || iod->dma_dir != DMA_TO_DEVICE);
        nvmet_pci_epf_transfer_iod_data(iod);

complete:
        nvmet_pci_epf_complete_iod(iod);
}

static int nvmet_pci_epf_process_sq(struct nvmet_pci_epf_ctrl *ctrl,
                                    struct nvmet_pci_epf_queue *sq)
{
        struct nvmet_pci_epf_iod *iod;
        int ret, n = 0;
        u16 head = sq->head;

        sq->tail = nvmet_pci_epf_bar_read32(ctrl, sq->db);
        while (head != sq->tail && (!ctrl->sq_ab || n < ctrl->sq_ab)) {
                iod = nvmet_pci_epf_alloc_iod(sq);
                if (!iod)
                        break;

                /* Get the NVMe command submitted by the host. */
                ret = nvmet_pci_epf_transfer(ctrl, &iod->cmd,
                                             sq->pci_addr + head * sq->qes,
                                             sq->qes, DMA_FROM_DEVICE);
                if (ret) {
                        /* Not much we can do... */
                        nvmet_pci_epf_free_iod(iod);
                        break;
                }

                dev_dbg(ctrl->dev, "SQ[%u]: head %u, tail %u, command %s\n",
                        sq->qid, head, sq->tail,
                        nvmet_pci_epf_iod_name(iod));

                head++;
                if (head == sq->depth)
                        head = 0;
                WRITE_ONCE(sq->head, head);
                n++;

                queue_work_on(WORK_CPU_UNBOUND, sq->iod_wq, &iod->work);

                sq->tail = nvmet_pci_epf_bar_read32(ctrl, sq->db);
        }

        return n;
}

static void nvmet_pci_epf_poll_sqs_work(struct work_struct *work)
{
        struct nvmet_pci_epf_ctrl *ctrl =
                container_of(work, struct nvmet_pci_epf_ctrl, poll_sqs.work);
        struct nvmet_pci_epf_queue *sq;
        unsigned long limit = jiffies;
        unsigned long last = 0;
        int i, nr_sqs;

        while (ctrl->link_up && ctrl->enabled) {
                nr_sqs = 0;
                /* Do round-robin arbitration. */
                for (i = 0; i < ctrl->nr_queues; i++) {
                        sq = &ctrl->sq[i];
                        if (!test_bit(NVMET_PCI_EPF_Q_LIVE, &sq->flags))
                                continue;
                        if (nvmet_pci_epf_process_sq(ctrl, sq))
                                nr_sqs++;
                }

                /*
                 * If we have been running for a while, reschedule to let other
                 * tasks run and to avoid RCU stalls.
                 */
                if (time_is_before_jiffies(limit + secs_to_jiffies(1))) {
                        cond_resched();
                        limit = jiffies;
                        continue;
                }

                if (nr_sqs) {
                        last = jiffies;
                        continue;
                }

                /*
                 * If we have not received any command on any queue for more
                 * than NVMET_PCI_EPF_SQ_POLL_IDLE, assume we are idle and
                 * reschedule. This avoids "burning" a CPU when the controller
                 * is idle for a long time.
                 */
                if (time_is_before_jiffies(last + NVMET_PCI_EPF_SQ_POLL_IDLE))
                        break;

                cpu_relax();
        }

        schedule_delayed_work(&ctrl->poll_sqs, NVMET_PCI_EPF_SQ_POLL_INTERVAL);
}

static void nvmet_pci_epf_cq_work(struct work_struct *work)
{
        struct nvmet_pci_epf_queue *cq =
                container_of(work, struct nvmet_pci_epf_queue, work.work);
        struct nvmet_pci_epf_ctrl *ctrl = cq->ctrl;
        struct nvme_completion *cqe;
        struct nvmet_pci_epf_iod *iod;
        unsigned long flags;
        int ret = 0, n = 0;

        while (test_bit(NVMET_PCI_EPF_Q_LIVE, &cq->flags) && ctrl->link_up) {

                /* Check that the CQ is not full. */
                cq->head = nvmet_pci_epf_bar_read32(ctrl, cq->db);
                if (cq->head == cq->tail + 1) {
                        ret = -EAGAIN;
                        break;
                }

                spin_lock_irqsave(&cq->lock, flags);
                iod = list_first_entry_or_null(&cq->list,
                                               struct nvmet_pci_epf_iod, link);
                if (iod)
                        list_del_init(&iod->link);
                spin_unlock_irqrestore(&cq->lock, flags);

                if (!iod)
                        break;

                /*
                 * Post the IOD completion entry. If the IOD request was
                 * executed (req->execute() called), the CQE is already
                 * initialized. However, the IOD may have been failed before
                 * that, leaving the CQE not properly initialized. So always
                 * initialize it here.
                 */
                cqe = &iod->cqe;
                cqe->sq_head = cpu_to_le16(READ_ONCE(iod->sq->head));
                cqe->sq_id = cpu_to_le16(iod->sq->qid);
                cqe->command_id = iod->cmd.common.command_id;
                cqe->status = cpu_to_le16((iod->status << 1) | cq->phase);

                dev_dbg(ctrl->dev,
                        "CQ[%u]: %s status 0x%x, result 0x%llx, head %u, tail %u, phase %u\n",
                        cq->qid, nvmet_pci_epf_iod_name(iod), iod->status,
                        le64_to_cpu(cqe->result.u64), cq->head, cq->tail,
                        cq->phase);

                memcpy_toio(cq->pci_map.virt_addr + cq->tail * cq->qes,
                            cqe, cq->qes);

                cq->tail++;
                if (cq->tail >= cq->depth) {
                        cq->tail = 0;
                        cq->phase ^= 1;
                }

                nvmet_pci_epf_free_iod(iod);

                /* Signal the host. */
                nvmet_pci_epf_raise_irq(ctrl, cq, false);
                n++;
        }

        /*
         * We do not support precise IRQ coalescing time (100ns units as per
         * NVMe specifications). So if we have posted completion entries without
         * reaching the interrupt coalescing threshold, raise an interrupt.
         */
        if (n)
                nvmet_pci_epf_raise_irq(ctrl, cq, true);

        if (ret < 0)
                queue_delayed_work(system_highpri_wq, &cq->work,
                                   NVMET_PCI_EPF_CQ_RETRY_INTERVAL);
}

static void nvmet_pci_epf_clear_ctrl_config(struct nvmet_pci_epf_ctrl *ctrl)
{
        struct nvmet_ctrl *tctrl = ctrl->tctrl;

        /* Initialize controller status. */
        tctrl->csts = 0;
        ctrl->csts = 0;
        nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CSTS, ctrl->csts);

        /* Initialize controller configuration and start polling. */
        tctrl->cc = 0;
        ctrl->cc = 0;
        nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CC, ctrl->cc);
}

static int nvmet_pci_epf_enable_ctrl(struct nvmet_pci_epf_ctrl *ctrl)
{
        u64 pci_addr, asq, acq;
        u32 aqa;
        u16 status, qsize;

        if (ctrl->enabled)
                return 0;

        dev_info(ctrl->dev, "Enabling controller\n");

        ctrl->mps_shift = nvmet_cc_mps(ctrl->cc) + 12;
        ctrl->mps = 1UL << ctrl->mps_shift;
        ctrl->mps_mask = ctrl->mps - 1;

        ctrl->io_sqes = 1UL << nvmet_cc_iosqes(ctrl->cc);
        if (ctrl->io_sqes < sizeof(struct nvme_command)) {
                dev_err(ctrl->dev, "Unsupported I/O SQES %zu (need %zu)\n",
                        ctrl->io_sqes, sizeof(struct nvme_command));
                goto err;
        }

        ctrl->io_cqes = 1UL << nvmet_cc_iocqes(ctrl->cc);
        if (ctrl->io_cqes < sizeof(struct nvme_completion)) {
                dev_err(ctrl->dev, "Unsupported I/O CQES %zu (need %zu)\n",
                        ctrl->io_cqes, sizeof(struct nvme_completion));
                goto err;
        }

        /* Create the admin queue. */
        aqa = nvmet_pci_epf_bar_read32(ctrl, NVME_REG_AQA);
        asq = nvmet_pci_epf_bar_read64(ctrl, NVME_REG_ASQ);
        acq = nvmet_pci_epf_bar_read64(ctrl, NVME_REG_ACQ);

        qsize = (aqa & 0x0fff0000) >> 16;
        pci_addr = acq & GENMASK_ULL(63, 12);
        status = nvmet_pci_epf_create_cq(ctrl->tctrl, 0,
                                NVME_CQ_IRQ_ENABLED | NVME_QUEUE_PHYS_CONTIG,
                                qsize, pci_addr, 0);
        if (status != NVME_SC_SUCCESS) {
                dev_err(ctrl->dev, "Failed to create admin completion queue\n");
                goto err;
        }

        qsize = aqa & 0x00000fff;
        pci_addr = asq & GENMASK_ULL(63, 12);
        status = nvmet_pci_epf_create_sq(ctrl->tctrl, 0, 0,
                        NVME_QUEUE_PHYS_CONTIG, qsize, pci_addr);
        if (status != NVME_SC_SUCCESS) {
                dev_err(ctrl->dev, "Failed to create admin submission queue\n");
                nvmet_pci_epf_delete_cq(ctrl->tctrl, 0);
                goto err;
        }

        ctrl->sq_ab = NVMET_PCI_EPF_SQ_AB;
        ctrl->irq_vector_threshold = NVMET_PCI_EPF_IV_THRESHOLD;
        ctrl->enabled = true;
        ctrl->csts = NVME_CSTS_RDY;

        /* Start polling the controller SQs. */
        schedule_delayed_work(&ctrl->poll_sqs, 0);

        return 0;

err:
        nvmet_pci_epf_clear_ctrl_config(ctrl);
        return -EINVAL;
}

static void nvmet_pci_epf_disable_ctrl(struct nvmet_pci_epf_ctrl *ctrl,
                                       bool shutdown)
{
        int qid;

        if (!ctrl->enabled)
                return;

        dev_info(ctrl->dev, "%s controller\n",
                 shutdown ? "Shutting down" : "Disabling");

        ctrl->enabled = false;
        cancel_delayed_work_sync(&ctrl->poll_sqs);

        /* Delete all I/O queues first. */
        for (qid = 1; qid < ctrl->nr_queues; qid++)
                nvmet_pci_epf_delete_sq(ctrl->tctrl, qid);

        for (qid = 1; qid < ctrl->nr_queues; qid++)
                nvmet_pci_epf_delete_cq(ctrl->tctrl, qid);

        /* Delete the admin queue last. */
        nvmet_pci_epf_delete_sq(ctrl->tctrl, 0);
        nvmet_pci_epf_delete_cq(ctrl->tctrl, 0);

        ctrl->csts &= ~NVME_CSTS_RDY;
        if (shutdown) {
                ctrl->csts |= NVME_CSTS_SHST_CMPLT;
                ctrl->cc &= ~NVME_CC_ENABLE;
                nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CC, ctrl->cc);
        }
}

static void nvmet_pci_epf_poll_cc_work(struct work_struct *work)
{
        struct nvmet_pci_epf_ctrl *ctrl =
                container_of(work, struct nvmet_pci_epf_ctrl, poll_cc.work);
        u32 old_cc, new_cc;
        int ret;

        if (!ctrl->tctrl)
                return;

        old_cc = ctrl->cc;
        new_cc = nvmet_pci_epf_bar_read32(ctrl, NVME_REG_CC);
        if (new_cc == old_cc)
                goto reschedule_work;

        ctrl->cc = new_cc;

        if (nvmet_cc_en(new_cc) && !nvmet_cc_en(old_cc)) {
                ret = nvmet_pci_epf_enable_ctrl(ctrl);
                if (ret)
                        goto reschedule_work;
        }

        if (!nvmet_cc_en(new_cc) && nvmet_cc_en(old_cc))
                nvmet_pci_epf_disable_ctrl(ctrl, false);

        if (nvmet_cc_shn(new_cc) && !nvmet_cc_shn(old_cc))
                nvmet_pci_epf_disable_ctrl(ctrl, true);

        if (!nvmet_cc_shn(new_cc) && nvmet_cc_shn(old_cc))
                ctrl->csts &= ~NVME_CSTS_SHST_CMPLT;

        nvmet_update_cc(ctrl->tctrl, ctrl->cc);
        nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CSTS, ctrl->csts);

reschedule_work:
        schedule_delayed_work(&ctrl->poll_cc, NVMET_PCI_EPF_CC_POLL_INTERVAL);
}

static void nvmet_pci_epf_init_bar(struct nvmet_pci_epf_ctrl *ctrl)
{
        struct nvmet_ctrl *tctrl = ctrl->tctrl;

        ctrl->bar = ctrl->nvme_epf->reg_bar;

        /* Copy the target controller capabilities as a base. */
        ctrl->cap = tctrl->cap;

        /* Contiguous Queues Required (CQR). */
        ctrl->cap |= 0x1ULL << 16;

        /* Set Doorbell stride to 4B (DSTRB). */
        ctrl->cap &= ~GENMASK_ULL(35, 32);

        /* Clear NVM Subsystem Reset Supported (NSSRS). */
        ctrl->cap &= ~(0x1ULL << 36);

        /* Clear Boot Partition Support (BPS). */
        ctrl->cap &= ~(0x1ULL << 45);

        /* Clear Persistent Memory Region Supported (PMRS). */
        ctrl->cap &= ~(0x1ULL << 56);

        /* Clear Controller Memory Buffer Supported (CMBS). */
        ctrl->cap &= ~(0x1ULL << 57);

        nvmet_pci_epf_bar_write64(ctrl, NVME_REG_CAP, ctrl->cap);
        nvmet_pci_epf_bar_write32(ctrl, NVME_REG_VS, tctrl->subsys->ver);

        nvmet_pci_epf_clear_ctrl_config(ctrl);
}

static int nvmet_pci_epf_create_ctrl(struct nvmet_pci_epf *nvme_epf,
                                     unsigned int max_nr_queues)
{
        struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl;
        struct nvmet_alloc_ctrl_args args = {};
        char hostnqn[NVMF_NQN_SIZE];
        uuid_t id;
        int ret;

        memset(ctrl, 0, sizeof(*ctrl));
        ctrl->dev = &nvme_epf->epf->dev;
        mutex_init(&ctrl->irq_lock);
        ctrl->nvme_epf = nvme_epf;
        ctrl->mdts = nvme_epf->mdts_kb * SZ_1K;
        INIT_DELAYED_WORK(&ctrl->poll_cc, nvmet_pci_epf_poll_cc_work);
        INIT_DELAYED_WORK(&ctrl->poll_sqs, nvmet_pci_epf_poll_sqs_work);

        ret = mempool_init_kmalloc_pool(&ctrl->iod_pool,
                                        max_nr_queues * NVMET_MAX_QUEUE_SIZE,
                                        sizeof(struct nvmet_pci_epf_iod));
        if (ret) {
                dev_err(ctrl->dev, "Failed to initialize IOD mempool\n");
                return ret;
        }

        ctrl->port = nvmet_pci_epf_find_port(ctrl, nvme_epf->portid);
        if (!ctrl->port) {
                dev_err(ctrl->dev, "Port not found\n");
                ret = -EINVAL;
                goto out_mempool_exit;
        }

        /* Create the target controller. */
        uuid_gen(&id);
        snprintf(hostnqn, NVMF_NQN_SIZE,
                 "nqn.2014-08.org.nvmexpress:uuid:%pUb", &id);
        args.port = ctrl->port;
        args.subsysnqn = nvme_epf->subsysnqn;
        memset(&id, 0, sizeof(uuid_t));
        args.hostid = &id;
        args.hostnqn = hostnqn;
        args.ops = &nvmet_pci_epf_fabrics_ops;

        ctrl->tctrl = nvmet_alloc_ctrl(&args);
        if (!ctrl->tctrl) {
                dev_err(ctrl->dev, "Failed to create target controller\n");
                ret = -ENOMEM;
                goto out_mempool_exit;
        }
        ctrl->tctrl->drvdata = ctrl;

        /* We do not support protection information for now. */
        if (ctrl->tctrl->pi_support) {
                dev_err(ctrl->dev,
                        "Protection information (PI) is not supported\n");
                ret = -ENOTSUPP;
                goto out_put_ctrl;
        }

        /* Allocate our queues, up to the maximum number. */
        ctrl->nr_queues = min(ctrl->tctrl->subsys->max_qid + 1, max_nr_queues);
        ret = nvmet_pci_epf_alloc_queues(ctrl);
        if (ret)
                goto out_put_ctrl;

        /*
         * Allocate the IRQ vectors descriptors. We cannot have more than the
         * maximum number of queues.
         */
        ret = nvmet_pci_epf_alloc_irq_vectors(ctrl);
        if (ret)
                goto out_free_queues;

        dev_info(ctrl->dev,
                 "New PCI ctrl \"%s\", %u I/O queues, mdts %u B\n",
                 ctrl->tctrl->subsys->subsysnqn, ctrl->nr_queues - 1,
                 ctrl->mdts);

        /* Initialize BAR 0 using the target controller CAP. */
        nvmet_pci_epf_init_bar(ctrl);

        return 0;

out_free_queues:
        nvmet_pci_epf_free_queues(ctrl);
out_put_ctrl:
        nvmet_ctrl_put(ctrl->tctrl);
        ctrl->tctrl = NULL;
out_mempool_exit:
        mempool_exit(&ctrl->iod_pool);
        return ret;
}

static void nvmet_pci_epf_start_ctrl(struct nvmet_pci_epf_ctrl *ctrl)
{

        dev_info(ctrl->dev, "PCI link up\n");
        ctrl->link_up = true;

        schedule_delayed_work(&ctrl->poll_cc, NVMET_PCI_EPF_CC_POLL_INTERVAL);
}

static void nvmet_pci_epf_stop_ctrl(struct nvmet_pci_epf_ctrl *ctrl)
{
        dev_info(ctrl->dev, "PCI link down\n");
        ctrl->link_up = false;

        cancel_delayed_work_sync(&ctrl->poll_cc);

        nvmet_pci_epf_disable_ctrl(ctrl, false);
        nvmet_pci_epf_clear_ctrl_config(ctrl);
}

static void nvmet_pci_epf_destroy_ctrl(struct nvmet_pci_epf_ctrl *ctrl)
{
        if (!ctrl->tctrl)
                return;

        dev_info(ctrl->dev, "Destroying PCI ctrl \"%s\"\n",
                 ctrl->tctrl->subsys->subsysnqn);

        nvmet_pci_epf_stop_ctrl(ctrl);

        nvmet_pci_epf_free_queues(ctrl);
        nvmet_pci_epf_free_irq_vectors(ctrl);

        nvmet_ctrl_put(ctrl->tctrl);
        ctrl->tctrl = NULL;

        mempool_exit(&ctrl->iod_pool);
}

static int nvmet_pci_epf_configure_bar(struct nvmet_pci_epf *nvme_epf)
{
        struct pci_epf *epf = nvme_epf->epf;
        const struct pci_epc_features *epc_features = nvme_epf->epc_features;
        size_t reg_size, reg_bar_size;
        size_t msix_table_size = 0;

        /*
         * The first free BAR will be our register BAR and per NVMe
         * specifications, it must be BAR 0.
         */
        if (pci_epc_get_first_free_bar(epc_features) != BAR_0) {
                dev_err(&epf->dev, "BAR 0 is not free\n");
                return -ENODEV;
        }

        /*
         * While NVMe PCIe Transport Specification 1.1, section 2.1.10, claims
         * that the BAR0 type is Implementation Specific, in NVMe 1.1, the type
         * is required to be 64-bit. Thus, for interoperability, always set the
         * type to 64-bit. In the rare case that the PCI EPC does not support
         * configuring BAR0 as 64-bit, the call to pci_epc_set_bar() will fail,
         * and we will return failure back to the user.
         */
        epf->bar[BAR_0].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;

        /*
         * Calculate the size of the register bar: NVMe registers first with
         * enough space for the doorbells, followed by the MSI-X table
         * if supported.
         */
        reg_size = NVME_REG_DBS + (NVMET_NR_QUEUES * 2 * sizeof(u32));
        reg_size = ALIGN(reg_size, 8);

        if (epc_features->msix_capable) {
                size_t pba_size;

                msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts;
                nvme_epf->msix_table_offset = reg_size;
                pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8);

                reg_size += msix_table_size + pba_size;
        }

        if (epc_features->bar[BAR_0].type == BAR_FIXED) {
                if (reg_size > epc_features->bar[BAR_0].fixed_size) {
                        dev_err(&epf->dev,
                                "BAR 0 size %llu B too small, need %zu B\n",
                                epc_features->bar[BAR_0].fixed_size,
                                reg_size);
                        return -ENOMEM;
                }
                reg_bar_size = epc_features->bar[BAR_0].fixed_size;
        } else {
                reg_bar_size = ALIGN(reg_size, max(epc_features->align, 4096));
        }

        nvme_epf->reg_bar = pci_epf_alloc_space(epf, reg_bar_size, BAR_0,
                                                epc_features, PRIMARY_INTERFACE);
        if (!nvme_epf->reg_bar) {
                dev_err(&epf->dev, "Failed to allocate BAR 0\n");
                return -ENOMEM;
        }
        memset(nvme_epf->reg_bar, 0, reg_bar_size);

        return 0;
}

static void nvmet_pci_epf_free_bar(struct nvmet_pci_epf *nvme_epf)
{
        struct pci_epf *epf = nvme_epf->epf;

        if (!nvme_epf->reg_bar)
                return;

        pci_epf_free_space(epf, nvme_epf->reg_bar, BAR_0, PRIMARY_INTERFACE);
        nvme_epf->reg_bar = NULL;
}

static void nvmet_pci_epf_clear_bar(struct nvmet_pci_epf *nvme_epf)
{
        struct pci_epf *epf = nvme_epf->epf;

        pci_epc_clear_bar(epf->epc, epf->func_no, epf->vfunc_no,
                          &epf->bar[BAR_0]);
}

static int nvmet_pci_epf_init_irq(struct nvmet_pci_epf *nvme_epf)
{
        const struct pci_epc_features *epc_features = nvme_epf->epc_features;
        struct pci_epf *epf = nvme_epf->epf;
        int ret;

        /* Enable MSI-X if supported, otherwise, use MSI. */
        if (epc_features->msix_capable && epf->msix_interrupts) {
                ret = pci_epc_set_msix(epf->epc, epf->func_no, epf->vfunc_no,
                                       epf->msix_interrupts, BAR_0,
                                       nvme_epf->msix_table_offset);
                if (ret) {
                        dev_err(&epf->dev, "Failed to configure MSI-X\n");
                        return ret;
                }

                nvme_epf->nr_vectors = epf->msix_interrupts;
                nvme_epf->irq_type = PCI_IRQ_MSIX;

                return 0;
        }

        if (epc_features->msi_capable && epf->msi_interrupts) {
                ret = pci_epc_set_msi(epf->epc, epf->func_no, epf->vfunc_no,
                                      epf->msi_interrupts);
                if (ret) {
                        dev_err(&epf->dev, "Failed to configure MSI\n");
                        return ret;
                }

                nvme_epf->nr_vectors = epf->msi_interrupts;
                nvme_epf->irq_type = PCI_IRQ_MSI;

                return 0;
        }

        /* MSI and MSI-X are not supported: fall back to INTx. */
        nvme_epf->nr_vectors = 1;
        nvme_epf->irq_type = PCI_IRQ_INTX;

        return 0;
}

static int nvmet_pci_epf_epc_init(struct pci_epf *epf)
{
        struct nvmet_pci_epf *nvme_epf = epf_get_drvdata(epf);
        const struct pci_epc_features *epc_features = nvme_epf->epc_features;
        struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl;
        unsigned int max_nr_queues = NVMET_NR_QUEUES;
        int ret;

        /* For now, do not support virtual functions. */
        if (epf->vfunc_no > 0) {
                dev_err(&epf->dev, "Virtual functions are not supported\n");
                return -EINVAL;
        }

        /*
         * Cap the maximum number of queues we can support on the controller
         * with the number of IRQs we can use.
         */
        if (epc_features->msix_capable && epf->msix_interrupts) {
                dev_info(&epf->dev,
                         "PCI endpoint controller supports MSI-X, %u vectors\n",
                         epf->msix_interrupts);
                max_nr_queues = min(max_nr_queues, epf->msix_interrupts);
        } else if (epc_features->msi_capable && epf->msi_interrupts) {
                dev_info(&epf->dev,
                         "PCI endpoint controller supports MSI, %u vectors\n",
                         epf->msi_interrupts);
                max_nr_queues = min(max_nr_queues, epf->msi_interrupts);
        }

        if (max_nr_queues < 2) {
                dev_err(&epf->dev, "Invalid maximum number of queues %u\n",
                        max_nr_queues);
                return -EINVAL;
        }

        /* Create the target controller. */
        ret = nvmet_pci_epf_create_ctrl(nvme_epf, max_nr_queues);
        if (ret) {
                dev_err(&epf->dev,
                        "Failed to create NVMe PCI target controller (err=%d)\n",
                        ret);
                return ret;
        }

        nvmet_pci_epf_init_dma(nvme_epf);

        /* Set device ID, class, etc. */
        epf->header->vendorid = ctrl->tctrl->subsys->vendor_id;
        epf->header->subsys_vendor_id = ctrl->tctrl->subsys->subsys_vendor_id;
        ret = pci_epc_write_header(epf->epc, epf->func_no, epf->vfunc_no,
                                   epf->header);
        if (ret) {
                dev_err(&epf->dev,
                        "Failed to write configuration header (err=%d)\n", ret);
                goto out_destroy_ctrl;
        }

        ret = pci_epc_set_bar(epf->epc, epf->func_no, epf->vfunc_no,
                              &epf->bar[BAR_0]);
        if (ret) {
                dev_err(&epf->dev, "Failed to set BAR 0 (err=%d)\n", ret);
                goto out_destroy_ctrl;
        }

        /*
         * Enable interrupts and start polling the controller BAR if we do not
         * have a link up notifier.
         */
        ret = nvmet_pci_epf_init_irq(nvme_epf);
        if (ret)
                goto out_clear_bar;

        if (!epc_features->linkup_notifier)
                nvmet_pci_epf_start_ctrl(&nvme_epf->ctrl);

        return 0;

out_clear_bar:
        nvmet_pci_epf_clear_bar(nvme_epf);
out_destroy_ctrl:
        nvmet_pci_epf_destroy_ctrl(&nvme_epf->ctrl);
        return ret;
}

static void nvmet_pci_epf_epc_deinit(struct pci_epf *epf)
{
        struct nvmet_pci_epf *nvme_epf = epf_get_drvdata(epf);
        struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl;

        nvmet_pci_epf_destroy_ctrl(ctrl);

        nvmet_pci_epf_deinit_dma(nvme_epf);
        nvmet_pci_epf_clear_bar(nvme_epf);
}

static int nvmet_pci_epf_link_up(struct pci_epf *epf)
{
        struct nvmet_pci_epf *nvme_epf = epf_get_drvdata(epf);
        struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl;

        nvmet_pci_epf_start_ctrl(ctrl);

        return 0;
}

static int nvmet_pci_epf_link_down(struct pci_epf *epf)
{
        struct nvmet_pci_epf *nvme_epf = epf_get_drvdata(epf);
        struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl;

        nvmet_pci_epf_stop_ctrl(ctrl);

        return 0;
}

static const struct pci_epc_event_ops nvmet_pci_epf_event_ops = {
        .epc_init = nvmet_pci_epf_epc_init,
        .epc_deinit = nvmet_pci_epf_epc_deinit,
        .link_up = nvmet_pci_epf_link_up,
        .link_down = nvmet_pci_epf_link_down,
};

static int nvmet_pci_epf_bind(struct pci_epf *epf)
{
        struct nvmet_pci_epf *nvme_epf = epf_get_drvdata(epf);
        const struct pci_epc_features *epc_features;
        struct pci_epc *epc = epf->epc;
        int ret;

        if (WARN_ON_ONCE(!epc))
                return -EINVAL;

        epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
        if (!epc_features) {
                dev_err(&epf->dev, "epc_features not implemented\n");
                return -EOPNOTSUPP;
        }
        nvme_epf->epc_features = epc_features;

        ret = nvmet_pci_epf_configure_bar(nvme_epf);
        if (ret)
                return ret;

        return 0;
}

static void nvmet_pci_epf_unbind(struct pci_epf *epf)
{
        struct nvmet_pci_epf *nvme_epf = epf_get_drvdata(epf);
        struct pci_epc *epc = epf->epc;

        nvmet_pci_epf_destroy_ctrl(&nvme_epf->ctrl);

        if (epc->init_complete) {
                nvmet_pci_epf_deinit_dma(nvme_epf);
                nvmet_pci_epf_clear_bar(nvme_epf);
        }

        nvmet_pci_epf_free_bar(nvme_epf);
}

static struct pci_epf_header nvme_epf_pci_header = {
        .vendorid       = PCI_ANY_ID,
        .deviceid       = PCI_ANY_ID,
        .progif_code    = 0x02, /* NVM Express */
        .baseclass_code = PCI_BASE_CLASS_STORAGE,
        .subclass_code  = 0x08, /* Non-Volatile Memory controller */
        .interrupt_pin  = PCI_INTERRUPT_INTA,
};

static int nvmet_pci_epf_probe(struct pci_epf *epf,
                               const struct pci_epf_device_id *id)
{
        struct nvmet_pci_epf *nvme_epf;
        int ret;

        nvme_epf = devm_kzalloc(&epf->dev, sizeof(*nvme_epf), GFP_KERNEL);
        if (!nvme_epf)
                return -ENOMEM;

        ret = devm_mutex_init(&epf->dev, &nvme_epf->mmio_lock);
        if (ret)
                return ret;

        nvme_epf->epf = epf;
        nvme_epf->mdts_kb = NVMET_PCI_EPF_MDTS_KB;

        epf->event_ops = &nvmet_pci_epf_event_ops;
        epf->header = &nvme_epf_pci_header;
        epf_set_drvdata(epf, nvme_epf);

        return 0;
}

#define to_nvme_epf(epf_group)  \
        container_of(epf_group, struct nvmet_pci_epf, group)

static ssize_t nvmet_pci_epf_portid_show(struct config_item *item, char *page)
{
        struct config_group *group = to_config_group(item);
        struct nvmet_pci_epf *nvme_epf = to_nvme_epf(group);

        return sysfs_emit(page, "%u\n", le16_to_cpu(nvme_epf->portid));
}

static ssize_t nvmet_pci_epf_portid_store(struct config_item *item,
                                          const char *page, size_t len)
{
        struct config_group *group = to_config_group(item);
        struct nvmet_pci_epf *nvme_epf = to_nvme_epf(group);
        u16 portid;

        /* Do not allow setting this when the function is already started. */
        if (nvme_epf->ctrl.tctrl)
                return -EBUSY;

        if (!len)
                return -EINVAL;

        if (kstrtou16(page, 0, &portid))
                return -EINVAL;

        nvme_epf->portid = cpu_to_le16(portid);

        return len;
}

CONFIGFS_ATTR(nvmet_pci_epf_, portid);

static ssize_t nvmet_pci_epf_subsysnqn_show(struct config_item *item,
                                            char *page)
{
        struct config_group *group = to_config_group(item);
        struct nvmet_pci_epf *nvme_epf = to_nvme_epf(group);

        return sysfs_emit(page, "%s\n", nvme_epf->subsysnqn);
}

static ssize_t nvmet_pci_epf_subsysnqn_store(struct config_item *item,
                                             const char *page, size_t len)
{
        struct config_group *group = to_config_group(item);
        struct nvmet_pci_epf *nvme_epf = to_nvme_epf(group);

        /* Do not allow setting this when the function is already started. */
        if (nvme_epf->ctrl.tctrl)
                return -EBUSY;

        if (!len)
                return -EINVAL;

        strscpy(nvme_epf->subsysnqn, page, len);

        return len;
}

CONFIGFS_ATTR(nvmet_pci_epf_, subsysnqn);

static ssize_t nvmet_pci_epf_mdts_kb_show(struct config_item *item, char *page)
{
        struct config_group *group = to_config_group(item);
        struct nvmet_pci_epf *nvme_epf = to_nvme_epf(group);

        return sysfs_emit(page, "%u\n", nvme_epf->mdts_kb);
}

static ssize_t nvmet_pci_epf_mdts_kb_store(struct config_item *item,
                                           const char *page, size_t len)
{
        struct config_group *group = to_config_group(item);
        struct nvmet_pci_epf *nvme_epf = to_nvme_epf(group);
        unsigned long mdts_kb;
        int ret;

        if (nvme_epf->ctrl.tctrl)
                return -EBUSY;

        ret = kstrtoul(page, 0, &mdts_kb);
        if (ret)
                return ret;
        if (!mdts_kb)
                mdts_kb = NVMET_PCI_EPF_MDTS_KB;
        else if (mdts_kb > NVMET_PCI_EPF_MAX_MDTS_KB)
                mdts_kb = NVMET_PCI_EPF_MAX_MDTS_KB;

        if (!is_power_of_2(mdts_kb))
                return -EINVAL;

        nvme_epf->mdts_kb = mdts_kb;

        return len;
}

CONFIGFS_ATTR(nvmet_pci_epf_, mdts_kb);

static struct configfs_attribute *nvmet_pci_epf_attrs[] = {
        &nvmet_pci_epf_attr_portid,
        &nvmet_pci_epf_attr_subsysnqn,
        &nvmet_pci_epf_attr_mdts_kb,
        NULL,
};

static const struct config_item_type nvmet_pci_epf_group_type = {
        .ct_attrs       = nvmet_pci_epf_attrs,
        .ct_owner       = THIS_MODULE,
};

static struct config_group *nvmet_pci_epf_add_cfs(struct pci_epf *epf,
                                                  struct config_group *group)
{
        struct nvmet_pci_epf *nvme_epf = epf_get_drvdata(epf);

        config_group_init_type_name(&nvme_epf->group, "nvme",
                                    &nvmet_pci_epf_group_type);

        return &nvme_epf->group;
}

static const struct pci_epf_device_id nvmet_pci_epf_ids[] = {
        { .name = "nvmet_pci_epf" },
        {},
};

static struct pci_epf_ops nvmet_pci_epf_ops = {
        .bind   = nvmet_pci_epf_bind,
        .unbind = nvmet_pci_epf_unbind,
        .add_cfs = nvmet_pci_epf_add_cfs,
};

static struct pci_epf_driver nvmet_pci_epf_driver = {
        .driver.name    = "nvmet_pci_epf",
        .probe          = nvmet_pci_epf_probe,
        .id_table       = nvmet_pci_epf_ids,
        .ops            = &nvmet_pci_epf_ops,
        .owner          = THIS_MODULE,
};

static int __init nvmet_pci_epf_init_module(void)
{
        int ret;

        ret = pci_epf_register_driver(&nvmet_pci_epf_driver);
        if (ret)
                return ret;

        ret = nvmet_register_transport(&nvmet_pci_epf_fabrics_ops);
        if (ret) {
                pci_epf_unregister_driver(&nvmet_pci_epf_driver);
                return ret;
        }

        return 0;
}

static void __exit nvmet_pci_epf_cleanup_module(void)
{
        nvmet_unregister_transport(&nvmet_pci_epf_fabrics_ops);
        pci_epf_unregister_driver(&nvmet_pci_epf_driver);
}

module_init(nvmet_pci_epf_init_module);
module_exit(nvmet_pci_epf_cleanup_module);

MODULE_DESCRIPTION("NVMe PCI Endpoint Function target driver");
MODULE_AUTHOR("Damien Le Moal <dlemoal@kernel.org>");
MODULE_LICENSE("GPL");