/*
 * Copyright (c) 2016-2018 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * Primary header file
 * NVME softc and structural definitions
 */

#if defined(__DragonFly__)
#include "nvme_dragonfly.h"
#else
#error "build for OS unknown"
#endif
#include "nvme_fw.h"
#include "nvme_log.h"
#include "nvme_ident.h"
#include "nvme_ns.h"
#include "nvme_chipset.h"
#include "nvme_ioctl.h"

MALLOC_DECLARE(M_NVME);

/*
 * Choose some reasonable limit, even if the hardware supports more.
 */
#define NVME_MAX_QUEUES         1024
#define NVME_MAX_NAMESPACES     1024

struct nvme_queue;
struct nvme_softc;
struct nvme_softns;

/*
 * Device matching array for special attach/detach codings.
 */
typedef struct {
        pci_vendor_id_t  vendor;
        pci_product_id_t product;
        int             (*attach)(device_t dev);
        int             (*detach)(device_t dev);
        char            *name;
} nvme_device_t;

/*
 * Kernel-level request structure.  This structure allows the driver to
 * construct, issue, wait for, and process responses to commands.  Each
 * queue manages its own request bank.
 *
 * In order to disconnect the request structure from the hardware queue
 * mechanism itself, to reduce SMP conflicts and interactions, and allow
 * command/response processing to block without interfering with queue
 * operations, this structure embeds a copy of the HW command and response
 * structures instead of referencing the ones in the actual hardware queues.
 * These will be copied to/from the actual queue entries by lower-level
 * chipset code.
 *
 * Requests are associated with particular queues, completions can occur on
 * any queue.  Requests associated with the admin queue conveniently include
 * an additional 4K 'info' block suitable for DMA.
 */
typedef struct nvme_request {
        struct nvme_request *next_avail;
        struct nvme_subqueue *subq;     /* which queue is submission on */
        struct nvme_comqueue *comq;     /* which queue is completion on */
        uint32_t        state;
        uint32_t        cmd_id;         /* reqary[] index */
        int             waiting;
        nvme_allcmd_t   cmd;            /* hw submit structure for entry */
        nvme_allres_t   res;            /* hw completion structure for entry */
        nvme_admin_data_t *info;        /* DMA data (admin request only) */
        bus_addr_t      pinfo;          /* phys address for PRP */

        /*
         * Aux fields to keep track of bio and other data, depending on
         * the callback.  If the callback is NULL the caller will poll for
         * completion.
         */
        void            (*callback)(struct nvme_request *req, struct lock *lk);
        struct nvme_softns *nsc;
        struct bio      *bio;
} nvme_request_t;

#define NVME_REQ_AVAIL          0
#define NVME_REQ_ALLOCATED      1
#define NVME_REQ_SUBMITTED      2
#define NVME_REQ_COMPLETED      3

typedef struct nvme_subqueue {
        /*
         * Driver stuff
         */
        struct lock     lk;             /* queue lock controls access */
        struct nvme_softc *sc;
        nvme_request_t  *dump_req;      /* one request is set aside */
        nvme_request_t  *first_avail;
        nvme_request_t  *reqary;
        uint32_t        nqe;            /* #of queue entries */
        uint16_t        qid;            /* which queue# are we on? */
        uint16_t        comqid;         /* we are tied to this completion qu */
        uint32_t        subq_doorbell_reg;
        uint32_t        subq_head;      /* start of active requests */
        uint32_t        subq_tail;      /* new requests */
        uint32_t        unsubmitted;    /* #unsubmitted requests */
        int             unused01;
        int             signal_requeue;

        /*
         * DMA resources
         */
        bus_dmamap_t    sque_map;
        bus_dmamap_t    prps_map;
        nvme_allcmd_t   *ksubq;         /* kernel-mapped addresses */
        uint64_t        *kprps;         /* enough PRPs per request for */
                                        /* MAXPHYS bytes worth of mappings */
        bus_addr_t      psubq;          /* physical addresses */
        bus_addr_t      pprps;

        /*
         * Additional DMA resources for admin queue (A NVME_MAX_ADMIN_BUFFER
         * sized buffer for each queue entry).
         */
        bus_dmamap_t    adm_map;
        bus_addr_t      pdatapgs;
        nvme_admin_data_t *kdatapgs;
} nvme_subqueue_t;

typedef struct nvme_comqueue {
        /*
         * Driver stuff
         */
        struct lock     lk;             /* queue lock controls access */
        struct nvme_softc *sc;
        uint32_t        nqe;            /* #of queue entries */
        uint16_t        phase;          /* phase to match (res->tail.status) */
        uint16_t        qid;            /* which queue# are we on? */
        uint32_t        comq_doorbell_reg;
        uint32_t        comq_head;      /* consume responses */
        uint32_t        comq_tail;

        /*
         * DMA resources
         */
        bus_dmamap_t    cque_map;
        nvme_allres_t   *kcomq;
        bus_addr_t      pcomq;
} nvme_comqueue_t;

typedef struct nvme_softns {
        struct nvme_softc *sc;
        nvme_ident_ns_data_t idns;
        struct bio_queue_head bioq;     /* excess BIOs */
        struct lock     lk;             /* mostly for bioq handling */
        int             state;
        int             unit;
        uint32_t        nsid;
        uint32_t        blksize;
        struct disk     disk;           /* disk attachment */
        struct devstat  stats;
        cdev_t          cdev;           /* disk device (cdev) */
} nvme_softns_t;

#define NVME_NSC_STATE_UNATTACHED       0
#define NVME_NSC_STATE_ATTACHED         1


typedef struct nvme_softc {
        TAILQ_ENTRY(nvme_softc) entry;  /* global list */
        device_t        dev;            /* core device */
        const nvme_device_t *ad;        /* quirk detection etc */
        thread_t        admintd;
        uint32_t        maxqe;
        uint64_t        cap;
        uint32_t        vers;
        uint32_t        dstrd4;         /* doorbell stride */
        uint32_t        entimo;         /* enable timeout in ticks */
        uint16_t        niosubqs;       /* #of I/O submission queues */
        uint16_t        niocomqs;       /* #of I/O completion queues */
        uint16_t        dumpqno;
        uint16_t        eventqno;
        uint16_t        qmap[SMP_MAXCPU][2];
        uint32_t        flags;
        nvme_subqueue_t subqueues[NVME_MAX_QUEUES];
        nvme_comqueue_t comqueues[NVME_MAX_QUEUES];
        int             cputovect[SMP_MAXCPU];

        /*
         * bio/disk layer tracking
         */
        ulong           opencnt;

        /*
         * admin queue irq resources
         * register map resources
         */
        struct resource *regs;
        struct resource *bar4;
        bus_space_tag_t iot;
        bus_space_handle_t ioh;
        int             rid_regs;
        int             rid_bar4;

        int             nirqs;
        int             irq_type;
        struct resource *irq[NVME_MAX_QUEUES];
        int             rid_irq[NVME_MAX_QUEUES];
        void            *irq_handle[NVME_MAX_QUEUES];

        /*
         * dma tags
         */
        bus_dma_tag_t   prps_tag;       /* worst-case PRP table(s) per queue */
        bus_dma_tag_t   sque_tag;       /* submission queue */
        bus_dma_tag_t   cque_tag;       /* completion queue */
        bus_dma_tag_t   adm_tag;        /* DMA data buffers for admin cmds */
        size_t          prp_bytes;
        size_t          cmd_bytes;
        size_t          res_bytes;
        size_t          adm_bytes;

        /*
         * Admin thread and controller identify data
         */
        uint32_t        admin_signal;
        struct lock     admin_lk;
        struct lock     ioctl_lk;
        int             (*admin_func)(struct nvme_softc *);
        nvme_ident_ctlr_data_t idctlr;
        nvme_softns_t   *nscary[NVME_MAX_NAMESPACES];
        int             nscmax;
} nvme_softc_t;

#define NVME_SC_ATTACHED        0x00000001
#define NVME_SC_UNLOADING       0x00000002

#define ADMIN_SIG_STOP          0x00000001
#define ADMIN_SIG_RUNNING       0x00000002
#define ADMIN_SIG_PROBED        0x00000004
#define ADMIN_SIG_REQUEUE       0x00000008
#define ADMIN_SIG_RUN_MASK      (ADMIN_SIG_STOP | ADMIN_SIG_REQUEUE)

#define NVME_QMAP_RD            0
#define NVME_QMAP_WR            1

typedef struct nvme_status_buf {
        char buf[64];
} nvme_status_buf_t;

/*
 * Prototypes
 */
const nvme_device_t *nvme_lookup_device(device_t dev);
void nvme_os_sleep(int ms);
int nvme_os_softsleep(void);
void nvme_os_hardsleep(int us);
u_int32_t nvme_read(nvme_softc_t *sc, bus_size_t r);
u_int64_t nvme_read8(nvme_softc_t *sc, bus_size_t r);
void nvme_write(nvme_softc_t *sc, bus_size_t r, u_int32_t v);
void nvme_write8(nvme_softc_t *sc, bus_size_t r, u_int64_t v);
int nvme_enable(nvme_softc_t *sc, int enable);
int nvme_alloc_subqueue(nvme_softc_t *sc, uint16_t qid);
int nvme_alloc_comqueue(nvme_softc_t *sc, uint16_t qid);
void nvme_free_subqueue(nvme_softc_t *sc, uint16_t qid);
void nvme_free_comqueue(nvme_softc_t *sc, uint16_t qid);

nvme_request_t *nvme_get_admin_request(nvme_softc_t *sc, uint8_t opcode);
nvme_request_t *nvme_get_request(nvme_subqueue_t *queue, uint8_t opcode,
                        char *kva, size_t bytes);
nvme_request_t *nvme_get_dump_request(nvme_subqueue_t *queue, uint8_t opcode,
                        char *kva, size_t bytes);
void nvme_submit_request(nvme_request_t *req);
int nvme_wait_request(nvme_request_t *req);
int nvme_poll_request(nvme_request_t *req);
void nvme_put_request(nvme_request_t *req);
void nvme_put_dump_request(nvme_request_t *req);
void nvme_poll_completions(nvme_comqueue_t *queue, struct lock *lk);

int nvme_start_admin_thread(nvme_softc_t *sc);
void nvme_stop_admin_thread(nvme_softc_t *sc);

int nvme_create_subqueue(nvme_softc_t *sc, uint16_t qid);
int nvme_create_comqueue(nvme_softc_t *sc, uint16_t qid);
int nvme_delete_subqueue(nvme_softc_t *sc, uint16_t qid);
int nvme_delete_comqueue(nvme_softc_t *sc, uint16_t qid);
int nvme_issue_shutdown(nvme_softc_t *sc, int dopoll);

void nvme_disk_attach(nvme_softns_t *nsc);
void nvme_disk_detach(nvme_softns_t *nsc);
void nvme_disk_requeues(nvme_softc_t *sc);
int nvme_alloc_disk_unit(void);

int nvme_getlog_ioctl(nvme_softc_t *sc, nvme_getlog_ioctl_t *ioc);

void nvme_intr(void *arg);
size_t string_cleanup(char *str, int domiddle);