/*-
 *   BSD LICENSE
 *
 *   Copyright (c) Intel Corporation. All rights reserved.
 *   Copyright (c) 2017, Western Digital Corporation or its affiliates.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __NVME_INTERNAL_H__
#define __NVME_INTERNAL_H__

#include "nvme_common.h"
#include "nvme_pci.h"
#include "nvme_intel.h"
#include "nvme_mem.h"

#ifndef __HAIKU__
#include <pthread.h>
#include <sys/user.h> /* PAGE_SIZE */
#else
#include "nvme_platform.h"
#endif

/*
 * List functions.
 */
#define LIST_FOREACH_SAFE(var, head, field, tvar)                       \
        for ((var) = LIST_FIRST((head));                                \
             (var) && ((tvar) = LIST_NEXT((var), field), 1);            \
             (var) = (tvar))

/*
 * Tail queue functions.
 */
#define TAILQ_FOREACH_SAFE(var, head, field, tvar)                      \
        for ((var) = TAILQ_FIRST((head));                               \
             (var) && ((tvar) = TAILQ_NEXT((var), field), 1);           \
             (var) = (tvar))

#define INTEL_DC_P3X00_DEVID    0x09538086

#define NVME_TIMEOUT_INFINITE   UINT64_MAX

/*
 * Some Intel devices support vendor-unique read latency log page even
 * though the log page directory says otherwise.
 */
#define NVME_INTEL_QUIRK_READ_LATENCY   0x1

/*
 * Some Intel devices support vendor-unique write latency log page even
 * though the log page directory says otherwise.
 */
#define NVME_INTEL_QUIRK_WRITE_LATENCY  0x2

/*
 * Some controllers need a delay before starting to check the device
 * readiness, which is done by reading the controller status register rdy bit.
 */
#define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4

/*
 * Some controllers need a delay once the controller status register rdy bit
 * switches from 0 to 1.
 */
#define NVME_QUIRK_DELAY_AFTER_RDY      0x8

/*
 * Queues may consist of a contiguous block of physical
 * memory or optionally a non-contiguous set of physical
 * memory pages (defined by a Physical Region Pages List)
 */
#define NVME_MAX_PRP_LIST_ENTRIES       (506)

/*
 * For commands requiring more than 2 PRP entries, one PRP will be
 * embedded in the command (prp1), and the rest of the PRP entries
 * will be in a list pointed to by the command (prp2).  This means
 * that real max number of PRP entries we support is 506+1, which
 * results in a max xfer size of 506*PAGE_SIZE.
 */
#define NVME_MAX_XFER_SIZE      NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE

#define NVME_ADMIN_TRACKERS             (16)
#define NVME_ADMIN_ENTRIES              (128)

/*
 * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
 * queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
 * will allow outstanding on an I/O qpair at any time. The only advantage in
 * having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
 * the contents of the submission and completion queues, it will show a longer
 * history of data.
 */
#define NVME_IO_ENTRIES                 (1024U)
#define NVME_IO_TRACKERS                (128U)
#define NVME_IO_ENTRIES_VS_TRACKERS_RATIO (NVME_IO_ENTRIES / NVME_IO_TRACKERS)

/*
 * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL
 * segment.
 */
#define NVME_MAX_SGL_DESCRIPTORS        (253)

/*
 * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
 * for each controller.
 */

#define NVME_MAX_ASYNC_EVENTS           (8)

/*
 * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
 * define specifies the maximum number of queues this driver will actually
 * try to configure, if available.
 */
#define DEFAULT_MAX_IO_QUEUES           (1024)

/*
 * Maximum of times a failed command can be retried.
 */
#define NVME_MAX_RETRY_COUNT            (3)

/*
 * I/O queue type.
 */
enum nvme_io_queue_type {

        NVME_IO_QTYPE_INVALID = 0,
        NVME_IO_SUBMISSION_QUEUE,
        NVME_IO_COMPLETION_QUEUE,
};

enum nvme_payload_type {

        NVME_PAYLOAD_TYPE_INVALID = 0,

        /*
         * nvme_request::u.payload.contig_buffer is valid for this request.
         */
        NVME_PAYLOAD_TYPE_CONTIG,

        /*
         * nvme_request::u.sgl is valid for this request
         */
        NVME_PAYLOAD_TYPE_SGL,
};

/*
 * Controller support flags.
 */
enum nvme_ctrlr_flags {

        /*
         * The SGL is supported.
         */
        NVME_CTRLR_SGL_SUPPORTED = 0x1,

};

/*
 * Descriptor for a request data payload.
 *
 * This struct is arranged so that it fits nicely in struct nvme_request.
 */
struct __attribute__((packed)) nvme_payload {

        union {
                /*
                 * Virtual memory address of a single
                 * physically contiguous buffer
                 */
                void *contig;

                /*
                 * Call back functions for retrieving physical
                 * addresses for scattered payloads.
                 */
                struct {
                        nvme_req_reset_sgl_cb reset_sgl_fn;
                        nvme_req_next_sge_cb next_sge_fn;
                        void *cb_arg;
                } sgl;
        } u;

        /*
         * Virtual memory address of a single physically
         * contiguous metadata buffer
         */
        void *md;

        /*
         * Payload type.
         */
        uint8_t type;

};

struct nvme_request {

        /*
         * NVMe command: must be aligned on 64B.
         */
        struct nvme_cmd                  cmd;

        /*
         * Data payload for this request's command.
         */
        struct nvme_payload              payload;

        uint8_t                          retries;

        /*
         * Number of child requests still outstanding for this
         * request which was split into multiple child requests.
         */
        uint8_t                          child_reqs;
        uint32_t                         payload_size;

        /*
         * Offset in bytes from the beginning of payload for this request.
         * This is used for I/O commands that are split into multiple requests.
         */
        uint32_t                         payload_offset;
        uint32_t                         md_offset;

        nvme_cmd_cb                      cb_fn;
        void                             *cb_arg;

        /*
         * The following members should not be reordered with members
         * above.  These members are only needed when splitting
         * requests which is done rarely, and the driver is careful
         * to not touch the following fields until a split operation is
         * needed, to avoid touching an extra cacheline.
         */

        /*
         * Points to the outstanding child requests for a parent request.
         * Only valid if a request was split into multiple child
         * requests, and is not initialized for non-split requests.
         */
        TAILQ_HEAD(, nvme_request)       children;

        /*
         * Linked-list pointers for a child request in its parent's list.
         */
        TAILQ_ENTRY(nvme_request)        child_tailq;

        /*
         * For queueing in qpair queued_req or free_req.
         */
        struct nvme_qpair                *qpair;
        STAILQ_ENTRY(nvme_request)       stailq;

        /*
         * Points to a parent request if part of a split request,
         * NULL otherwise.
         */
        struct nvme_request              *parent;

        /*
         * Completion status for a parent request.  Initialized to all 0's
         * (SUCCESS) before child requests are submitted.  If a child
         * request completes with error, the error status is copied here,
         * to ensure that the parent request is also completed with error
         * status once all child requests are completed.
         */
        struct nvme_cpl                  parent_status;

} __attribute__((aligned(64)));

struct nvme_completion_poll_status {
        struct nvme_cpl         cpl;
        bool                    done;
};

struct nvme_async_event_request {
        struct nvme_ctrlr       *ctrlr;
        struct nvme_request     *req;
        struct nvme_cpl         cpl;
};

struct nvme_tracker {

        LIST_ENTRY(nvme_tracker)        list;

        struct nvme_request             *req;
#if INTPTR_MAX == INT32_MAX
        int32_t __pad[3];
#elif !defined(INTPTR_MAX)
#       error Need definition of INTPTR_MAX!
#endif

        uint16_t                        cid;

        uint16_t                        rsvd1: 15;
        uint16_t                        active: 1;

        uint32_t                        rsvd2;

        uint64_t                        prp_sgl_bus_addr;

        union {
                uint64_t                        prp[NVME_MAX_PRP_LIST_ENTRIES];
                struct nvme_sgl_descriptor      sgl[NVME_MAX_SGL_DESCRIPTORS];
        } u;

        uint64_t                        rsvd3;
};

/*
 * struct nvme_tracker must be exactly 4K so that the prp[] array does not
 * cross a page boundery and so that there is no padding required to meet
 * alignment requirements.
 */
nvme_static_assert(sizeof(struct nvme_tracker) == 4096,
                   "nvme_tracker is not 4K");
nvme_static_assert((offsetof(struct nvme_tracker, u.sgl) & 7) == 0,
                   "SGL must be Qword aligned");

struct nvme_qpair {
        /*
         * Guards access to this structure.
         */
        pthread_mutex_t                         lock;

        volatile uint32_t               *sq_tdbl;
        volatile uint32_t               *cq_hdbl;

        /*
         * Submission queue
         */
        struct nvme_cmd                 *cmd;

        /*
         * Completion queue
         */
        struct nvme_cpl                 *cpl;

        LIST_HEAD(, nvme_tracker)       free_tr;
        LIST_HEAD(, nvme_tracker)       outstanding_tr;

        /*
         * Array of trackers indexed by command ID.
         */
        uint16_t                        trackers;
        struct nvme_tracker             *tr;

        struct nvme_request             *reqs;
        unsigned int                    num_reqs;
        STAILQ_HEAD(, nvme_request)     free_req;
        STAILQ_HEAD(, nvme_request)     queued_req;

        uint16_t                        id;

        uint16_t                        entries;
        uint16_t                        sq_tail;
        uint16_t                        cq_head;

        uint8_t                         phase;

        bool                            enabled;
        bool                            sq_in_cmb;

        /*
         * Fields below this point should not be touched on the
         * normal I/O happy path.
         */

        uint8_t                         qprio;

        struct nvme_ctrlr               *ctrlr;

        /* List entry for nvme_ctrlr::free_io_qpairs and active_io_qpairs */
        TAILQ_ENTRY(nvme_qpair)         tailq;

        phys_addr_t                     cmd_bus_addr;
        phys_addr_t                     cpl_bus_addr;
};

struct nvme_ns {

        struct nvme_ctrlr               *ctrlr;

        uint32_t                        stripe_size;
        uint32_t                        sector_size;

        uint32_t                        md_size;
        uint32_t                        pi_type;

        uint32_t                        sectors_per_max_io;
        uint32_t                        sectors_per_stripe;

        uint16_t                        id;
        uint16_t                        flags;

        int                             open_count;

};

/*
 * State of struct nvme_ctrlr (in particular, during initialization).
 */
enum nvme_ctrlr_state {

        /*
         * Controller has not been initialized yet.
         */
        NVME_CTRLR_STATE_INIT = 0,

        /*
         * Waiting for CSTS.RDY to transition from 0 to 1
         * so that CC.EN may be set to 0.
         */
        NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,

        /*
         * Waiting for CSTS.RDY to transition from 1 to 0
         * so that CC.EN may be set to 1.
         */
        NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,

        /*
         * Waiting for CSTS.RDY to transition from 0 to 1
         * after enabling the controller.
         */
        NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,

        /*
         * Controller initialization has completed and
         * the controller is ready.
         */
        NVME_CTRLR_STATE_READY
};

/*
 * One of these per allocated PCI device.
 */
struct nvme_ctrlr {

        /*
         * NVMe MMIO register space.
         */
        volatile struct nvme_registers  *regs;

        /*
         * Array of I/O queue pairs.
         */
        struct nvme_qpair               *ioq;

        /*
         * Size of the array of I/O queue pairs.
         */
        unsigned int                    io_queues;

        /*
         * Maximum I/O queue pairs.
         */
        unsigned int                    max_io_queues;

        /*
         * Number of I/O queue pairs enabled
         */
        unsigned int                    enabled_io_qpairs;

        /*
         * Maximum entries for I/O qpairs
         */
        unsigned int                    io_qpairs_max_entries;

        /*
         * Array of namespace IDs.
         */
        unsigned int                    nr_ns;
        struct nvme_ns                  *ns;

        /*
         * Controller state.
         */
        bool                            resetting;
        bool                            failed;

        /*
         * Controller support flags.
         */
        uint64_t                        flags;

        /*
         * Cold data (not accessed in normal I/O path) is after this point.
         */
        enum nvme_ctrlr_state           state;
        uint64_t                        state_timeout_ms;

        /*
         * All the log pages supported.
         */
        bool                            log_page_supported[256];

        /*
         * All the features supported.
         */
        bool                            feature_supported[256];

        /*
         * Associated PCI device information.
         */
        struct pci_device               *pci_dev;

        /*
         * Maximum i/o size in bytes.
         */
        uint32_t                        max_xfer_size;

        /*
         * Minimum page size supported by this controller in bytes.
         */
        uint32_t                        min_page_size;

        /*
         * Stride in uint32_t units between doorbell registers
         * (1 = 4 bytes, 2 = 8 bytes, ...).
         */
        uint32_t                        doorbell_stride_u32;

        uint32_t                        num_aers;
        struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
        nvme_aer_cb                     aer_cb_fn;
        void                            *aer_cb_arg;

        /*
         * Admin queue pair.
         */
        struct nvme_qpair               adminq;

        /*
         * Guards access to the controller itself.
         */
        pthread_mutex_t                 lock;

        /*
         * Identify Controller data.
         */
        struct nvme_ctrlr_data          cdata;

        /*
         * Array of Identify Namespace data.
         * Stored separately from ns since nsdata should
         * not normally be accessed during I/O.
         */
        struct nvme_ns_data             *nsdata;

        TAILQ_HEAD(, nvme_qpair)        free_io_qpairs;
        TAILQ_HEAD(, nvme_qpair)        active_io_qpairs;

        /*
         * Controller option set on open.
         */
        struct nvme_ctrlr_opts          opts;

        /*
         * BAR mapping address which contains controller memory buffer.
         */
        void                            *cmb_bar_virt_addr;

        /*
         * BAR physical address which contains controller memory buffer.
         */
        uint64_t                        cmb_bar_phys_addr;

        /*
         * Controller memory buffer size in Bytes.
         */
        uint64_t                        cmb_size;

        /*
         * Current offset of controller memory buffer.
         */
        uint64_t                        cmb_current_offset;

        /*
         * Quirks flags.
         */
        unsigned int                    quirks;

        /*
         * For controller list.
         */
        LIST_ENTRY(nvme_ctrlr)          link;

} __attribute__((aligned(PAGE_SIZE)));

/*
 * Admin functions.
 */
extern int nvme_admin_identify_ctrlr(struct nvme_ctrlr *ctrlr,
                                     struct nvme_ctrlr_data *cdata);

extern int nvme_admin_get_feature(struct nvme_ctrlr *ctrlr,
                                  enum nvme_feat_sel sel,
                                  enum nvme_feat feature,
                                  uint32_t cdw11, uint32_t *attributes);

extern int nvme_admin_set_feature(struct nvme_ctrlr *ctrlr,
                                  bool save,
                                  enum nvme_feat feature,
                                  uint32_t cdw11, uint32_t cdw12,
                                  uint32_t cdw13, uint32_t cdw14, uint32_t cdw15,
                                  void *buf, uint32_t len,
                                  uint32_t *attributes);

extern int nvme_admin_format_nvm(struct nvme_ctrlr *ctrlr,
                                 unsigned int nsid,
                                 struct nvme_format *format);

extern int nvme_admin_get_log_page(struct nvme_ctrlr *ctrlr,
                                   uint8_t log_page, uint32_t nsid,
                                   void *payload, uint32_t payload_size);

extern int nvme_admin_abort_cmd(struct nvme_ctrlr *ctrlr,
                                uint16_t cid, uint16_t sqid);

extern int nvme_admin_create_ioq(struct nvme_ctrlr *ctrlr,
                                 struct nvme_qpair *io_que,
                                 enum nvme_io_queue_type io_qtype);

extern int nvme_admin_delete_ioq(struct nvme_ctrlr *ctrlr,
                                 struct nvme_qpair *qpair,
                                 enum nvme_io_queue_type io_qtype);

extern int nvme_admin_identify_ns(struct nvme_ctrlr *ctrlr,
                                  uint16_t nsid,
                                  struct nvme_ns_data *nsdata);

extern int nvme_admin_attach_ns(struct nvme_ctrlr *ctrlr,
                                uint32_t nsid,
                                struct nvme_ctrlr_list *clist);

extern int nvme_admin_detach_ns(struct nvme_ctrlr *ctrlr,
                                uint32_t nsid,
                                struct nvme_ctrlr_list *clist);

extern int nvme_admin_create_ns(struct nvme_ctrlr *ctrlr,
                                struct nvme_ns_data *nsdata,
                                unsigned int *nsid);

extern int nvme_admin_delete_ns(struct nvme_ctrlr *ctrlr,
                                unsigned int nsid);

extern int nvme_admin_fw_commit(struct nvme_ctrlr *ctrlr,
                                const struct nvme_fw_commit *fw_commit);

extern int nvme_admin_fw_image_dl(struct nvme_ctrlr *ctrlr,
                                  void *fw, uint32_t size, uint32_t offset);

extern void nvme_request_completion_poll_cb(void *arg,
                                            const struct nvme_cpl *cpl);

extern struct nvme_ctrlr *nvme_ctrlr_attach(struct pci_device *pci_dev,
                                            struct nvme_ctrlr_opts *opts);

extern void nvme_ctrlr_detach(struct nvme_ctrlr *ctrlr);

extern int nvme_qpair_construct(struct nvme_ctrlr *ctrlr,
                                struct nvme_qpair *qpair, enum nvme_qprio qprio,
                                uint16_t entries, uint16_t trackers);

extern void nvme_qpair_destroy(struct nvme_qpair *qpair);
extern void nvme_qpair_enable(struct nvme_qpair *qpair);
extern void nvme_qpair_disable(struct nvme_qpair *qpair);
extern int  nvme_qpair_submit_request(struct nvme_qpair *qpair,
                                      struct nvme_request *req);
extern void nvme_qpair_reset(struct nvme_qpair *qpair);
extern void nvme_qpair_fail(struct nvme_qpair *qpair);

extern int nvme_request_pool_construct(struct nvme_qpair *qpair);

extern void nvme_request_pool_destroy(struct nvme_qpair *qpair);

extern struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair,
                      const struct nvme_payload *payload, uint32_t payload_size,
                      nvme_cmd_cb cb_fn, void *cb_arg);

extern struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair,
                                                       nvme_cmd_cb cb_fn,
                                                       void *cb_arg);

extern struct nvme_request *
nvme_request_allocate_contig(struct nvme_qpair *qpair,
                             void *buffer, uint32_t payload_size,
                             nvme_cmd_cb cb_fn, void *cb_arg);

extern void nvme_request_free(struct nvme_request *req);
extern void nvme_request_free_locked(struct nvme_request *req);

extern void nvme_request_add_child(struct nvme_request *parent,
                                   struct nvme_request *child);

extern void nvme_request_remove_child(struct nvme_request *parent,
                                      struct nvme_request *child);

extern unsigned int nvme_ctrlr_get_quirks(struct pci_device *pdev);

extern int nvme_ns_construct(struct nvme_ctrlr *ctrlr,
                             struct nvme_ns *ns, unsigned int id);

/*
 * Registers mmio access.
 */
#define nvme_reg_mmio_read_4(sc, reg)           \
        nvme_mmio_read_4((__u32 *)&(sc)->regs->reg)

#define nvme_reg_mmio_read_8(sc, reg)           \
        nvme_mmio_read_8((__u64 *)&(sc)->regs->reg)

#define nvme_reg_mmio_write_4(sc, reg, val)     \
        nvme_mmio_write_4((__u32 *)&(sc)->regs->reg, val)

#define nvme_reg_mmio_write_8(sc, reg, val)     \
        nvme_mmio_write_8((__u64 *)&(sc)->regs->reg, val)

#endif /* __NVME_INTERNAL_H__ */