/*-
 * Copyright (C) 2012 Intel Corporation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
#ifndef __IOAT_INTERNAL_H__
#define __IOAT_INTERNAL_H__

#include <sys/_task.h>

#define DEVICE2SOFTC(dev)       ((struct ioat_softc *) device_get_softc(dev))
#define KTR_IOAT                KTR_SPARE3

#define ioat_read_chancnt(ioat) \
        ioat_read_1((ioat), IOAT_CHANCNT_OFFSET)

#define ioat_read_xfercap(ioat) \
        (ioat_read_1((ioat), IOAT_XFERCAP_OFFSET) & IOAT_XFERCAP_VALID_MASK)

#define ioat_write_intrctrl(ioat, value) \
        ioat_write_1((ioat), IOAT_INTRCTRL_OFFSET, (value))

#define ioat_read_cbver(ioat) \
        (ioat_read_1((ioat), IOAT_CBVER_OFFSET) & 0xFF)

#define ioat_read_dmacapability(ioat) \
        ioat_read_4((ioat), IOAT_DMACAPABILITY_OFFSET)

#define ioat_write_chanctrl(ioat, value) \
        ioat_write_2((ioat), IOAT_CHANCTRL_OFFSET, (value))

static __inline uint64_t
ioat_bus_space_read_8_lower_first(bus_space_tag_t tag,
    bus_space_handle_t handle, bus_size_t offset)
{
        return (bus_space_read_4(tag, handle, offset) |
            ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32);
}

static __inline void
ioat_bus_space_write_8_lower_first(bus_space_tag_t tag,
    bus_space_handle_t handle, bus_size_t offset, uint64_t val)
{
        bus_space_write_4(tag, handle, offset, val);
        bus_space_write_4(tag, handle, offset + 4, val >> 32);
}

#ifdef __i386__
#define ioat_bus_space_read_8 ioat_bus_space_read_8_lower_first
#define ioat_bus_space_write_8 ioat_bus_space_write_8_lower_first
#else
#define ioat_bus_space_read_8(tag, handle, offset) \
        bus_space_read_8((tag), (handle), (offset))
#define ioat_bus_space_write_8(tag, handle, offset, val) \
        bus_space_write_8((tag), (handle), (offset), (val))
#endif

#define ioat_read_1(ioat, offset) \
        bus_space_read_1((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset))

#define ioat_read_2(ioat, offset) \
        bus_space_read_2((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset))

#define ioat_read_4(ioat, offset) \
        bus_space_read_4((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset))

#define ioat_read_8(ioat, offset) \
        ioat_bus_space_read_8((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset))

#define ioat_read_double_4(ioat, offset) \
        ioat_bus_space_read_8_lower_first((ioat)->pci_bus_tag, \
            (ioat)->pci_bus_handle, (offset))

#define ioat_write_1(ioat, offset, value) \
        bus_space_write_1((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset), (value))

#define ioat_write_2(ioat, offset, value) \
        bus_space_write_2((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset), (value))

#define ioat_write_4(ioat, offset, value) \
        bus_space_write_4((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset), (value))

#define ioat_write_8(ioat, offset, value) \
        ioat_bus_space_write_8((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \
            (offset), (value))

#define ioat_write_double_4(ioat, offset, value) \
        ioat_bus_space_write_8_lower_first((ioat)->pci_bus_tag, \
            (ioat)->pci_bus_handle, (offset), (value))

MALLOC_DECLARE(M_IOAT);

SYSCTL_DECL(_hw_ioat);

extern int g_ioat_debug_level;

struct generic_dma_control {
        uint32_t int_enable:1;
        uint32_t src_snoop_disable:1;
        uint32_t dest_snoop_disable:1;
        uint32_t completion_update:1;
        uint32_t fence:1;
        uint32_t reserved1:1;
        uint32_t src_page_break:1;
        uint32_t dest_page_break:1;
        uint32_t bundle:1;
        uint32_t dest_dca:1;
        uint32_t hint:1;
        uint32_t reserved2:13;
        uint32_t op:8;
};

struct ioat_generic_hw_descriptor {
        uint32_t size;
        union {
                uint32_t control_raw;
                struct generic_dma_control control_generic;
        } u;
        uint64_t src_addr;
        uint64_t dest_addr;
        uint64_t next;
        uint64_t reserved[4];
};

struct ioat_dma_hw_descriptor {
        uint32_t size;
        union {
                uint32_t control_raw;
                struct generic_dma_control control_generic;
                struct {
                        uint32_t int_enable:1;
                        uint32_t src_snoop_disable:1;
                        uint32_t dest_snoop_disable:1;
                        uint32_t completion_update:1;
                        uint32_t fence:1;
                        uint32_t null:1;
                        uint32_t src_page_break:1;
                        uint32_t dest_page_break:1;
                        uint32_t bundle:1;
                        uint32_t dest_dca:1;
                        uint32_t hint:1;
                        uint32_t reserved:13;
                        #define IOAT_OP_COPY 0x00
                        uint32_t op:8;
                } control;
        } u;
        uint64_t src_addr;
        uint64_t dest_addr;
        uint64_t next;
        uint64_t next_src_addr;
        uint64_t next_dest_addr;
        uint64_t user1;
        uint64_t user2;
};

struct ioat_fill_hw_descriptor {
        uint32_t size;
        union {
                uint32_t control_raw;
                struct generic_dma_control control_generic;
                struct {
                        uint32_t int_enable:1;
                        uint32_t reserved:1;
                        uint32_t dest_snoop_disable:1;
                        uint32_t completion_update:1;
                        uint32_t fence:1;
                        uint32_t reserved2:2;
                        uint32_t dest_page_break:1;
                        uint32_t bundle:1;
                        uint32_t reserved3:15;
                        #define IOAT_OP_FILL 0x01
                        uint32_t op:8;
                } control;
        } u;
        uint64_t src_data;
        uint64_t dest_addr;
        uint64_t next;
        uint64_t reserved;
        uint64_t next_dest_addr;
        uint64_t user1;
        uint64_t user2;
};

struct ioat_crc32_hw_descriptor {
        uint32_t size;
        union {
                uint32_t control_raw;
                struct generic_dma_control control_generic;
                struct {
                        uint32_t int_enable:1;
                        uint32_t src_snoop_disable:1;
                        uint32_t dest_snoop_disable:1;
                        uint32_t completion_update:1;
                        uint32_t fence:1;
                        uint32_t reserved1:3;
                        uint32_t bundle:1;
                        uint32_t dest_dca:1;
                        uint32_t hint:1;
                        uint32_t use_seed:1;
                        /*
                         * crc_location:
                         * For IOAT_OP_MOVECRC_TEST and IOAT_OP_CRC_TEST:
                         * 0: comparison value is pointed to by CRC Address
                         *    field.
                         * 1: comparison value follows data in wire format
                         *    ("inverted reflected bit order") in the 4 bytes
                         *    following the source data.
                         *
                         * For IOAT_OP_CRC_STORE:
                         * 0: Result will be stored at location pointed to by
                         *    CRC Address field (in wire format).
                         * 1: Result will be stored directly following the
                         *    source data.
                         *
                         * For IOAT_OP_MOVECRC_STORE:
                         * 0: Result will be stored at location pointed to by
                         *    CRC Address field (in wire format).
                         * 1: Result will be stored directly following the
                         *    *destination* data.
                         */
                        uint32_t crc_location:1;
                        uint32_t reserved2:11;
                        /*
                         * MOVECRC - Move data in the same way as standard copy
                         * operation, but also compute CRC32.
                         *
                         * CRC - Only compute CRC on source data.
                         *
                         * There is a CRC accumulator register in the hardware.
                         * If 'initial' is set, it is initialized to the value
                         * in 'seed.'
                         *
                         * In all modes, these operators accumulate size bytes
                         * at src_addr into the running CRC32C.
                         *
                         * Store mode emits the accumulated CRC, in wire
                         * format, as specified by the crc_location bit above.
                         *
                         * Test mode compares the accumulated CRC against the
                         * reference CRC, as described in crc_location above.
                         * On failure, halts the DMA engine with a CRC error
                         * status.
                         */
                        #define IOAT_OP_MOVECRC         0x41
                        #define IOAT_OP_MOVECRC_TEST    0x42
                        #define IOAT_OP_MOVECRC_STORE   0x43
                        #define IOAT_OP_CRC             0x81
                        #define IOAT_OP_CRC_TEST        0x82
                        #define IOAT_OP_CRC_STORE       0x83
                        uint32_t op:8;
                } control;
        } u;
        uint64_t src_addr;
        uint64_t dest_addr;
        uint64_t next;
        uint64_t next_src_addr;
        uint64_t next_dest_addr;
        uint32_t seed;
        uint32_t reserved;
        uint64_t crc_address;
};

struct ioat_xor_hw_descriptor {
        uint32_t size;
        union {
                uint32_t control_raw;
                struct generic_dma_control control_generic;
                struct {
                        uint32_t int_enable:1;
                        uint32_t src_snoop_disable:1;
                        uint32_t dest_snoop_disable:1;
                        uint32_t completion_update:1;
                        uint32_t fence:1;
                        uint32_t src_count:3;
                        uint32_t bundle:1;
                        uint32_t dest_dca:1;
                        uint32_t hint:1;
                        uint32_t reserved:13;
                        #define IOAT_OP_XOR 0x87
                        #define IOAT_OP_XOR_VAL 0x88
                        uint32_t op:8;
                } control;
        } u;
        uint64_t src_addr;
        uint64_t dest_addr;
        uint64_t next;
        uint64_t src_addr2;
        uint64_t src_addr3;
        uint64_t src_addr4;
        uint64_t src_addr5;
};

struct ioat_xor_ext_hw_descriptor {
        uint64_t src_addr6;
        uint64_t src_addr7;
        uint64_t src_addr8;
        uint64_t next;
        uint64_t reserved[4];
};

struct ioat_pq_hw_descriptor {
        uint32_t size;
        union {
                uint32_t control_raw;
                struct generic_dma_control control_generic;
                struct {
                        uint32_t int_enable:1;
                        uint32_t src_snoop_disable:1;
                        uint32_t dest_snoop_disable:1;
                        uint32_t completion_update:1;
                        uint32_t fence:1;
                        uint32_t src_count:3;
                        uint32_t bundle:1;
                        uint32_t dest_dca:1;
                        uint32_t hint:1;
                        uint32_t p_disable:1;
                        uint32_t q_disable:1;
                        uint32_t reserved:11;
                        #define IOAT_OP_PQ 0x89
                        #define IOAT_OP_PQ_VAL 0x8a
                        uint32_t op:8;
                } control;
        } u;
        uint64_t src_addr;
        uint64_t p_addr;
        uint64_t next;
        uint64_t src_addr2;
        uint64_t src_addr3;
        uint8_t  coef[8];
        uint64_t q_addr;
};

struct ioat_pq_ext_hw_descriptor {
        uint64_t src_addr4;
        uint64_t src_addr5;
        uint64_t src_addr6;
        uint64_t next;
        uint64_t src_addr7;
        uint64_t src_addr8;
        uint64_t reserved[2];
};

struct ioat_pq_update_hw_descriptor {
        uint32_t size;
        union {
                uint32_t control_raw;
                struct generic_dma_control control_generic;
                struct {
                        uint32_t int_enable:1;
                        uint32_t src_snoop_disable:1;
                        uint32_t dest_snoop_disable:1;
                        uint32_t completion_update:1;
                        uint32_t fence:1;
                        uint32_t src_cnt:3;
                        uint32_t bundle:1;
                        uint32_t dest_dca:1;
                        uint32_t hint:1;
                        uint32_t p_disable:1;
                        uint32_t q_disable:1;
                        uint32_t reserved:3;
                        uint32_t coef:8;
                        #define IOAT_OP_PQ_UP 0x8b
                        uint32_t op:8;
                } control;
        } u;
        uint64_t src_addr;
        uint64_t p_addr;
        uint64_t next;
        uint64_t src_addr2;
        uint64_t p_src;
        uint64_t q_src;
        uint64_t q_addr;
};

struct ioat_raw_hw_descriptor {
        uint64_t field[8];
};

struct bus_dmadesc {
        bus_dmaengine_callback_t callback_fn;
        void                     *callback_arg;
};

struct ioat_descriptor {
        struct bus_dmadesc      bus_dmadesc;
        uint32_t                id;
        bus_dmamap_t            src_dmamap;
        bus_dmamap_t            dst_dmamap;
        bus_dmamap_t            src2_dmamap;
        bus_dmamap_t            dst2_dmamap;
};

/* Unused by this driver at this time. */
#define IOAT_OP_MARKER          0x84

/*
 * Deprecated OPs -- v3 DMA generates an abort if given these.  And this driver
 * doesn't support anything older than v3.
 */
#define IOAT_OP_OLD_XOR         0x85
#define IOAT_OP_OLD_XOR_VAL     0x86

/* One of these per allocated PCI device. */
struct ioat_softc {
        bus_dmaengine_t         dmaengine;
#define to_ioat_softc(_dmaeng)                                          \
({                                                                      \
        bus_dmaengine_t *_p = (_dmaeng);                                \
        (struct ioat_softc *)((char *)_p -                              \
            offsetof(struct ioat_softc, dmaengine));                    \
})

        device_t                device;
        int                     domain;
        int                     cpu;
        int                     version;
        unsigned                chan_idx;

        bus_space_tag_t         pci_bus_tag;
        bus_space_handle_t      pci_bus_handle;
        struct resource         *pci_resource;
        int                     pci_resource_id;
        uint32_t                max_xfer_size;
        uint32_t                capabilities;
        uint32_t                ring_size_order;
        uint16_t                intrdelay_max;
        uint16_t                cached_intrdelay;

        int                     rid;
        struct resource         *res;
        void                    *tag;

        bus_dma_tag_t           hw_desc_tag;
        bus_dmamap_t            hw_desc_map;

        bus_dma_tag_t           data_tag;

        bus_dma_tag_t           comp_update_tag;
        bus_dmamap_t            comp_update_map;
        uint64_t                *comp_update;
        bus_addr_t              comp_update_bus_addr;

        boolean_t               quiescing;
        boolean_t               destroying;
        boolean_t               is_submitter_processing;
        boolean_t               intrdelay_supported;
        boolean_t               resetting;              /* submit_lock */
        boolean_t               resetting_cleanup;      /* cleanup_lock */

        struct ioat_descriptor  *ring;

        union ioat_hw_descriptor {
                struct ioat_generic_hw_descriptor       generic;
                struct ioat_dma_hw_descriptor           dma;
                struct ioat_fill_hw_descriptor          fill;
                struct ioat_crc32_hw_descriptor         crc32;
                struct ioat_xor_hw_descriptor           xor;
                struct ioat_xor_ext_hw_descriptor       xor_ext;
                struct ioat_pq_hw_descriptor            pq;
                struct ioat_pq_ext_hw_descriptor        pq_ext;
                struct ioat_raw_hw_descriptor           raw;
        } *hw_desc_ring;
        bus_addr_t              hw_desc_bus_addr;
#define RING_PHYS_ADDR(sc, i)   (sc)->hw_desc_bus_addr + \
    (((i) % (1 << (sc)->ring_size_order)) * sizeof(struct ioat_dma_hw_descriptor))

        struct mtx_padalign     submit_lock;
        struct callout          poll_timer;
        struct task             reset_task;
        struct mtx_padalign     cleanup_lock;

        uint32_t                refcnt;
        uint32_t                head;
        uint32_t                acq_head;
        uint32_t                tail;
        bus_addr_t              last_seen;

        struct {
                uint64_t        interrupts;
                uint64_t        descriptors_processed;
                uint64_t        descriptors_error;
                uint64_t        descriptors_submitted;

                uint32_t        channel_halts;
                uint32_t        last_halt_chanerr;
        } stats;
};

void ioat_test_attach(void);
void ioat_test_detach(void);

/*
 * XXX DO NOT USE this routine for obtaining the current completed descriptor.
 *
 * The double_4 read on ioat<3.3 appears to result in torn reads.  And v3.2
 * hardware is still commonplace (Broadwell Xeon has it).  Instead, use the
 * device-pushed *comp_update.
 *
 * It is safe to use ioat_get_chansts() for the low status bits.
 */
static inline uint64_t
ioat_get_chansts(struct ioat_softc *ioat)
{
        uint64_t status;

        if (ioat->version >= IOAT_VER_3_3)
                status = ioat_read_8(ioat, IOAT_CHANSTS_OFFSET);
        else
                /* Must read lower 4 bytes before upper 4 bytes. */
                status = ioat_read_double_4(ioat, IOAT_CHANSTS_OFFSET);
        return (status);
}

static inline void
ioat_write_chancmp(struct ioat_softc *ioat, uint64_t addr)
{

        if (ioat->version >= IOAT_VER_3_3)
                ioat_write_8(ioat, IOAT_CHANCMP_OFFSET_LOW, addr);
        else
                ioat_write_double_4(ioat, IOAT_CHANCMP_OFFSET_LOW, addr);
}

static inline void
ioat_write_chainaddr(struct ioat_softc *ioat, uint64_t addr)
{

        if (ioat->version >= IOAT_VER_3_3)
                ioat_write_8(ioat, IOAT_CHAINADDR_OFFSET_LOW, addr);
        else
                ioat_write_double_4(ioat, IOAT_CHAINADDR_OFFSET_LOW, addr);
}

static inline boolean_t
is_ioat_active(uint64_t status)
{
        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
}

static inline boolean_t
is_ioat_idle(uint64_t status)
{
        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_IDLE);
}

static inline boolean_t
is_ioat_halted(uint64_t status)
{
        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
}

static inline boolean_t
is_ioat_suspended(uint64_t status)
{
        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
}

static inline void
ioat_suspend(struct ioat_softc *ioat)
{
        ioat_write_1(ioat, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_SUSPEND);
}

static inline void
ioat_reset(struct ioat_softc *ioat)
{
        ioat_write_1(ioat, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET);
}

static inline boolean_t
ioat_reset_pending(struct ioat_softc *ioat)
{
        uint8_t cmd;

        cmd = ioat_read_1(ioat, IOAT_CHANCMD_OFFSET);
        return ((cmd & IOAT_CHANCMD_RESET) != 0);
}

#endif /* __IOAT_INTERNAL_H__ */