// SPDX-License-Identifier: GPL-2.0
/* ldc.c: Logical Domain Channel link-layer protocol driver.
 *
 * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
 */

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/scatterlist.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/init.h>
#include <linux/bitmap.h>
#include <asm/iommu-common.h>

#include <asm/hypervisor.h>
#include <asm/iommu.h>
#include <asm/page.h>
#include <asm/ldc.h>
#include <asm/mdesc.h>

#define DRV_MODULE_NAME         "ldc"
#define PFX DRV_MODULE_NAME     ": "
#define DRV_MODULE_VERSION      "1.1"
#define DRV_MODULE_RELDATE      "July 22, 2008"

#define COOKIE_PGSZ_CODE        0xf000000000000000ULL
#define COOKIE_PGSZ_CODE_SHIFT  60ULL


static char version[] =
        DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

/* Packet header layout for unreliable and reliable mode frames.
 * When in RAW mode, packets are simply straight 64-byte payloads
 * with no headers.
 */
struct ldc_packet {
        u8                      type;
#define LDC_CTRL                0x01
#define LDC_DATA                0x02
#define LDC_ERR                 0x10

        u8                      stype;
#define LDC_INFO                0x01
#define LDC_ACK                 0x02
#define LDC_NACK                0x04

        u8                      ctrl;
#define LDC_VERS                0x01 /* Link Version            */
#define LDC_RTS                 0x02 /* Request To Send         */
#define LDC_RTR                 0x03 /* Ready To Receive        */
#define LDC_RDX                 0x04 /* Ready for Data eXchange */
#define LDC_CTRL_MSK            0x0f

        u8                      env;
#define LDC_LEN                 0x3f
#define LDC_FRAG_MASK           0xc0
#define LDC_START               0x40
#define LDC_STOP                0x80

        u32                     seqid;

        union {
                u8              u_data[LDC_PACKET_SIZE - 8];
                struct {
                        u32     pad;
                        u32     ackid;
                        u8      r_data[LDC_PACKET_SIZE - 8 - 8];
                } r;
        } u;
};

struct ldc_version {
        u16 major;
        u16 minor;
};

/* Ordered from largest major to lowest.  */
static struct ldc_version ver_arr[] = {
        { .major = 1, .minor = 0 },
};

#define LDC_DEFAULT_MTU                 (4 * LDC_PACKET_SIZE)
#define LDC_DEFAULT_NUM_ENTRIES         (PAGE_SIZE / LDC_PACKET_SIZE)

struct ldc_channel;

struct ldc_mode_ops {
        int (*write)(struct ldc_channel *, const void *, unsigned int);
        int (*read)(struct ldc_channel *, void *, unsigned int);
};

static const struct ldc_mode_ops raw_ops;
static const struct ldc_mode_ops nonraw_ops;
static const struct ldc_mode_ops stream_ops;

int ldom_domaining_enabled;

struct ldc_iommu {
        /* Protects ldc_unmap.  */
        spinlock_t                      lock;
        struct ldc_mtable_entry         *page_table;
        struct iommu_map_table          iommu_map_table;
};

struct ldc_channel {
        /* Protects all operations that depend upon channel state.  */
        spinlock_t                      lock;

        unsigned long                   id;

        u8                              *mssbuf;
        u32                             mssbuf_len;
        u32                             mssbuf_off;

        struct ldc_packet               *tx_base;
        unsigned long                   tx_head;
        unsigned long                   tx_tail;
        unsigned long                   tx_num_entries;
        unsigned long                   tx_ra;

        unsigned long                   tx_acked;

        struct ldc_packet               *rx_base;
        unsigned long                   rx_head;
        unsigned long                   rx_tail;
        unsigned long                   rx_num_entries;
        unsigned long                   rx_ra;

        u32                             rcv_nxt;
        u32                             snd_nxt;

        unsigned long                   chan_state;

        struct ldc_channel_config       cfg;
        void                            *event_arg;

        const struct ldc_mode_ops       *mops;

        struct ldc_iommu                iommu;

        struct ldc_version              ver;

        u8                              hs_state;
#define LDC_HS_CLOSED                   0x00
#define LDC_HS_OPEN                     0x01
#define LDC_HS_GOTVERS                  0x02
#define LDC_HS_SENTRTR                  0x03
#define LDC_HS_GOTRTR                   0x04
#define LDC_HS_COMPLETE                 0x10

        u8                              flags;
#define LDC_FLAG_ALLOCED_QUEUES         0x01
#define LDC_FLAG_REGISTERED_QUEUES      0x02
#define LDC_FLAG_REGISTERED_IRQS        0x04
#define LDC_FLAG_RESET                  0x10

        u8                              mss;
        u8                              state;

#define LDC_IRQ_NAME_MAX                32
        char                            rx_irq_name[LDC_IRQ_NAME_MAX];
        char                            tx_irq_name[LDC_IRQ_NAME_MAX];

        struct hlist_head               mh_list;

        struct hlist_node               list;
};

#define ldcdbg(TYPE, f, a...) \
do {    if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
                printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
} while (0)

#define LDC_ABORT(lp)   ldc_abort((lp), __func__)

static const char *state_to_str(u8 state)
{
        switch (state) {
        case LDC_STATE_INVALID:
                return "INVALID";
        case LDC_STATE_INIT:
                return "INIT";
        case LDC_STATE_BOUND:
                return "BOUND";
        case LDC_STATE_READY:
                return "READY";
        case LDC_STATE_CONNECTED:
                return "CONNECTED";
        default:
                return "<UNKNOWN>";
        }
}

static unsigned long __advance(unsigned long off, unsigned long num_entries)
{
        off += LDC_PACKET_SIZE;
        if (off == (num_entries * LDC_PACKET_SIZE))
                off = 0;

        return off;
}

static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
{
        return __advance(off, lp->rx_num_entries);
}

static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
{
        return __advance(off, lp->tx_num_entries);
}

static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
                                                  unsigned long *new_tail)
{
        struct ldc_packet *p;
        unsigned long t;

        t = tx_advance(lp, lp->tx_tail);
        if (t == lp->tx_head)
                return NULL;

        *new_tail = t;

        p = lp->tx_base;
        return p + (lp->tx_tail / LDC_PACKET_SIZE);
}

/* When we are in reliable or stream mode, have to track the next packet
 * we haven't gotten an ACK for in the TX queue using tx_acked.  We have
 * to be careful not to stomp over the queue past that point.  During
 * the handshake, we don't have TX data packets pending in the queue
 * and that's why handshake_get_tx_packet() need not be mindful of
 * lp->tx_acked.
 */
static unsigned long head_for_data(struct ldc_channel *lp)
{
        if (lp->cfg.mode == LDC_MODE_STREAM)
                return lp->tx_acked;
        return lp->tx_head;
}

static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
{
        unsigned long limit, tail, new_tail, diff;
        unsigned int mss;

        limit = head_for_data(lp);
        tail = lp->tx_tail;
        new_tail = tx_advance(lp, tail);
        if (new_tail == limit)
                return 0;

        if (limit > new_tail)
                diff = limit - new_tail;
        else
                diff = (limit +
                        ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
        diff /= LDC_PACKET_SIZE;
        mss = lp->mss;

        if (diff * mss < size)
                return 0;

        return 1;
}

static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
                                             unsigned long *new_tail)
{
        struct ldc_packet *p;
        unsigned long h, t;

        h = head_for_data(lp);
        t = tx_advance(lp, lp->tx_tail);
        if (t == h)
                return NULL;

        *new_tail = t;

        p = lp->tx_base;
        return p + (lp->tx_tail / LDC_PACKET_SIZE);
}

static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
{
        unsigned long orig_tail = lp->tx_tail;
        int limit = 1000;

        lp->tx_tail = tail;
        while (limit-- > 0) {
                unsigned long err;

                err = sun4v_ldc_tx_set_qtail(lp->id, tail);
                if (!err)
                        return 0;

                if (err != HV_EWOULDBLOCK) {
                        lp->tx_tail = orig_tail;
                        return -EINVAL;
                }
                udelay(1);
        }

        lp->tx_tail = orig_tail;
        return -EBUSY;
}

/* This just updates the head value in the hypervisor using
 * a polling loop with a timeout.  The caller takes care of
 * upating software state representing the head change, if any.
 */
static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
{
        int limit = 1000;

        while (limit-- > 0) {
                unsigned long err;

                err = sun4v_ldc_rx_set_qhead(lp->id, head);
                if (!err)
                        return 0;

                if (err != HV_EWOULDBLOCK)
                        return -EINVAL;

                udelay(1);
        }

        return -EBUSY;
}

static int send_tx_packet(struct ldc_channel *lp,
                          struct ldc_packet *p,
                          unsigned long new_tail)
{
        BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));

        return set_tx_tail(lp, new_tail);
}

static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
                                                 u8 stype, u8 ctrl,
                                                 void *data, int dlen,
                                                 unsigned long *new_tail)
{
        struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);

        if (p) {
                memset(p, 0, sizeof(*p));
                p->type = LDC_CTRL;
                p->stype = stype;
                p->ctrl = ctrl;
                if (data)
                        memcpy(p->u.u_data, data, dlen);
        }
        return p;
}

static int start_handshake(struct ldc_channel *lp)
{
        struct ldc_packet *p;
        struct ldc_version *ver;
        unsigned long new_tail;

        ver = &ver_arr[0];

        ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
               ver->major, ver->minor);

        p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
                                   ver, sizeof(*ver), &new_tail);
        if (p) {
                int err = send_tx_packet(lp, p, new_tail);
                if (!err)
                        lp->flags &= ~LDC_FLAG_RESET;
                return err;
        }
        return -EBUSY;
}

static int send_version_nack(struct ldc_channel *lp,
                             u16 major, u16 minor)
{
        struct ldc_packet *p;
        struct ldc_version ver;
        unsigned long new_tail;

        ver.major = major;
        ver.minor = minor;

        p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
                                   &ver, sizeof(ver), &new_tail);
        if (p) {
                ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
                       ver.major, ver.minor);

                return send_tx_packet(lp, p, new_tail);
        }
        return -EBUSY;
}

static int send_version_ack(struct ldc_channel *lp,
                            struct ldc_version *vp)
{
        struct ldc_packet *p;
        unsigned long new_tail;

        p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
                                   vp, sizeof(*vp), &new_tail);
        if (p) {
                ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
                       vp->major, vp->minor);

                return send_tx_packet(lp, p, new_tail);
        }
        return -EBUSY;
}

static int send_rts(struct ldc_channel *lp)
{
        struct ldc_packet *p;
        unsigned long new_tail;

        p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
                                   &new_tail);
        if (p) {
                p->env = lp->cfg.mode;
                p->seqid = 0;
                lp->rcv_nxt = 0;

                ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
                       p->env, p->seqid);

                return send_tx_packet(lp, p, new_tail);
        }
        return -EBUSY;
}

static int send_rtr(struct ldc_channel *lp)
{
        struct ldc_packet *p;
        unsigned long new_tail;

        p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
                                   &new_tail);
        if (p) {
                p->env = lp->cfg.mode;
                p->seqid = 0;

                ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
                       p->env, p->seqid);

                return send_tx_packet(lp, p, new_tail);
        }
        return -EBUSY;
}

static int send_rdx(struct ldc_channel *lp)
{
        struct ldc_packet *p;
        unsigned long new_tail;

        p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
                                   &new_tail);
        if (p) {
                p->env = 0;
                p->seqid = ++lp->snd_nxt;
                p->u.r.ackid = lp->rcv_nxt;

                ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
                       p->env, p->seqid, p->u.r.ackid);

                return send_tx_packet(lp, p, new_tail);
        }
        return -EBUSY;
}

static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
{
        struct ldc_packet *p;
        unsigned long new_tail;
        int err;

        p = data_get_tx_packet(lp, &new_tail);
        if (!p)
                return -EBUSY;
        memset(p, 0, sizeof(*p));
        p->type = data_pkt->type;
        p->stype = LDC_NACK;
        p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
        p->seqid = lp->snd_nxt + 1;
        p->u.r.ackid = lp->rcv_nxt;

        ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
               p->type, p->ctrl, p->seqid, p->u.r.ackid);

        err = send_tx_packet(lp, p, new_tail);
        if (!err)
                lp->snd_nxt++;

        return err;
}

static int ldc_abort(struct ldc_channel *lp, const char *msg)
{
        unsigned long hv_err;

        ldcdbg(STATE, "ABORT[%s]\n", msg);
        ldc_print(lp);

        /* We report but do not act upon the hypervisor errors because
         * there really isn't much we can do if they fail at this point.
         */
        hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
        if (hv_err)
                printk(KERN_ERR PFX "ldc_abort: "
                       "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
                       lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);

        hv_err = sun4v_ldc_tx_get_state(lp->id,
                                        &lp->tx_head,
                                        &lp->tx_tail,
                                        &lp->chan_state);
        if (hv_err)
                printk(KERN_ERR PFX "ldc_abort: "
                       "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
                       lp->id, hv_err);

        hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
        if (hv_err)
                printk(KERN_ERR PFX "ldc_abort: "
                       "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
                       lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);

        /* Refetch the RX queue state as well, because we could be invoked
         * here in the queue processing context.
         */
        hv_err = sun4v_ldc_rx_get_state(lp->id,
                                        &lp->rx_head,
                                        &lp->rx_tail,
                                        &lp->chan_state);
        if (hv_err)
                printk(KERN_ERR PFX "ldc_abort: "
                       "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
                       lp->id, hv_err);

        return -ECONNRESET;
}

static struct ldc_version *find_by_major(u16 major)
{
        struct ldc_version *ret = NULL;
        int i;

        for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
                struct ldc_version *v = &ver_arr[i];
                if (v->major <= major) {
                        ret = v;
                        break;
                }
        }
        return ret;
}

static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
{
        struct ldc_version *vap;
        int err;

        ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
               vp->major, vp->minor);

        if (lp->hs_state == LDC_HS_GOTVERS) {
                lp->hs_state = LDC_HS_OPEN;
                memset(&lp->ver, 0, sizeof(lp->ver));
        }

        vap = find_by_major(vp->major);
        if (!vap) {
                err = send_version_nack(lp, 0, 0);
        } else if (vap->major != vp->major) {
                err = send_version_nack(lp, vap->major, vap->minor);
        } else {
                struct ldc_version ver = *vp;
                if (ver.minor > vap->minor)
                        ver.minor = vap->minor;
                err = send_version_ack(lp, &ver);
                if (!err) {
                        lp->ver = ver;
                        lp->hs_state = LDC_HS_GOTVERS;
                }
        }
        if (err)
                return LDC_ABORT(lp);

        return 0;
}

static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
{
        ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
               vp->major, vp->minor);

        if (lp->hs_state == LDC_HS_GOTVERS) {
                if (lp->ver.major != vp->major ||
                    lp->ver.minor != vp->minor)
                        return LDC_ABORT(lp);
        } else {
                lp->ver = *vp;
                lp->hs_state = LDC_HS_GOTVERS;
        }
        if (send_rts(lp))
                return LDC_ABORT(lp);
        return 0;
}

static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
{
        struct ldc_version *vap;
        struct ldc_packet *p;
        unsigned long new_tail;

        if (vp->major == 0 && vp->minor == 0)
                return LDC_ABORT(lp);

        vap = find_by_major(vp->major);
        if (!vap)
                return LDC_ABORT(lp);

        p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
                                           vap, sizeof(*vap),
                                           &new_tail);
        if (!p)
                return LDC_ABORT(lp);

        return send_tx_packet(lp, p, new_tail);
}

static int process_version(struct ldc_channel *lp,
                           struct ldc_packet *p)
{
        struct ldc_version *vp;

        vp = (struct ldc_version *) p->u.u_data;

        switch (p->stype) {
        case LDC_INFO:
                return process_ver_info(lp, vp);

        case LDC_ACK:
                return process_ver_ack(lp, vp);

        case LDC_NACK:
                return process_ver_nack(lp, vp);

        default:
                return LDC_ABORT(lp);
        }
}

static int process_rts(struct ldc_channel *lp,
                       struct ldc_packet *p)
{
        ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
               p->stype, p->seqid, p->env);

        if (p->stype     != LDC_INFO       ||
            lp->hs_state != LDC_HS_GOTVERS ||
            p->env       != lp->cfg.mode)
                return LDC_ABORT(lp);

        lp->snd_nxt = p->seqid;
        lp->rcv_nxt = p->seqid;
        lp->hs_state = LDC_HS_SENTRTR;
        if (send_rtr(lp))
                return LDC_ABORT(lp);

        return 0;
}

static int process_rtr(struct ldc_channel *lp,
                       struct ldc_packet *p)
{
        ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
               p->stype, p->seqid, p->env);

        if (p->stype     != LDC_INFO ||
            p->env       != lp->cfg.mode)
                return LDC_ABORT(lp);

        lp->snd_nxt = p->seqid;
        lp->hs_state = LDC_HS_COMPLETE;
        ldc_set_state(lp, LDC_STATE_CONNECTED);
        send_rdx(lp);

        return LDC_EVENT_UP;
}

static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
{
        return lp->rcv_nxt + 1 == seqid;
}

static int process_rdx(struct ldc_channel *lp,
                       struct ldc_packet *p)
{
        ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
               p->stype, p->seqid, p->env, p->u.r.ackid);

        if (p->stype != LDC_INFO ||
            !(rx_seq_ok(lp, p->seqid)))
                return LDC_ABORT(lp);

        lp->rcv_nxt = p->seqid;

        lp->hs_state = LDC_HS_COMPLETE;
        ldc_set_state(lp, LDC_STATE_CONNECTED);

        return LDC_EVENT_UP;
}

static int process_control_frame(struct ldc_channel *lp,
                                 struct ldc_packet *p)
{
        switch (p->ctrl) {
        case LDC_VERS:
                return process_version(lp, p);

        case LDC_RTS:
                return process_rts(lp, p);

        case LDC_RTR:
                return process_rtr(lp, p);

        case LDC_RDX:
                return process_rdx(lp, p);

        default:
                return LDC_ABORT(lp);
        }
}

static int process_error_frame(struct ldc_channel *lp,
                               struct ldc_packet *p)
{
        return LDC_ABORT(lp);
}

static int process_data_ack(struct ldc_channel *lp,
                            struct ldc_packet *ack)
{
        unsigned long head = lp->tx_acked;
        u32 ackid = ack->u.r.ackid;

        while (1) {
                struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);

                head = tx_advance(lp, head);

                if (p->seqid == ackid) {
                        lp->tx_acked = head;
                        return 0;
                }
                if (head == lp->tx_tail)
                        return LDC_ABORT(lp);
        }

        return 0;
}

static void send_events(struct ldc_channel *lp, unsigned int event_mask)
{
        if (event_mask & LDC_EVENT_RESET)
                lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
        if (event_mask & LDC_EVENT_UP)
                lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
        if (event_mask & LDC_EVENT_DATA_READY)
                lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
}

static irqreturn_t ldc_rx(int irq, void *dev_id)
{
        struct ldc_channel *lp = dev_id;
        unsigned long orig_state, flags;
        unsigned int event_mask;

        spin_lock_irqsave(&lp->lock, flags);

        orig_state = lp->chan_state;

        /* We should probably check for hypervisor errors here and
         * reset the LDC channel if we get one.
         */
        sun4v_ldc_rx_get_state(lp->id,
                               &lp->rx_head,
                               &lp->rx_tail,
                               &lp->chan_state);

        ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
               orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);

        event_mask = 0;

        if (lp->cfg.mode == LDC_MODE_RAW &&
            lp->chan_state == LDC_CHANNEL_UP) {
                lp->hs_state = LDC_HS_COMPLETE;
                ldc_set_state(lp, LDC_STATE_CONNECTED);

                /*
                 * Generate an LDC_EVENT_UP event if the channel
                 * was not already up.
                 */
                if (orig_state != LDC_CHANNEL_UP) {
                        event_mask |= LDC_EVENT_UP;
                        orig_state = lp->chan_state;
                }
        }

        /* If we are in reset state, flush the RX queue and ignore
         * everything.
         */
        if (lp->flags & LDC_FLAG_RESET) {
                (void) ldc_rx_reset(lp);
                goto out;
        }

        /* Once we finish the handshake, we let the ldc_read()
         * paths do all of the control frame and state management.
         * Just trigger the callback.
         */
        if (lp->hs_state == LDC_HS_COMPLETE) {
handshake_complete:
                if (lp->chan_state != orig_state) {
                        unsigned int event = LDC_EVENT_RESET;

                        if (lp->chan_state == LDC_CHANNEL_UP)
                                event = LDC_EVENT_UP;

                        event_mask |= event;
                }
                if (lp->rx_head != lp->rx_tail)
                        event_mask |= LDC_EVENT_DATA_READY;

                goto out;
        }

        if (lp->chan_state != orig_state)
                goto out;

        while (lp->rx_head != lp->rx_tail) {
                struct ldc_packet *p;
                unsigned long new;
                int err;

                p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);

                switch (p->type) {
                case LDC_CTRL:
                        err = process_control_frame(lp, p);
                        if (err > 0)
                                event_mask |= err;
                        break;

                case LDC_DATA:
                        event_mask |= LDC_EVENT_DATA_READY;
                        err = 0;
                        break;

                case LDC_ERR:
                        err = process_error_frame(lp, p);
                        break;

                default:
                        err = LDC_ABORT(lp);
                        break;
                }

                if (err < 0)
                        break;

                new = lp->rx_head;
                new += LDC_PACKET_SIZE;
                if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
                        new = 0;
                lp->rx_head = new;

                err = __set_rx_head(lp, new);
                if (err < 0) {
                        (void) LDC_ABORT(lp);
                        break;
                }
                if (lp->hs_state == LDC_HS_COMPLETE)
                        goto handshake_complete;
        }

out:
        spin_unlock_irqrestore(&lp->lock, flags);

        send_events(lp, event_mask);

        return IRQ_HANDLED;
}

static irqreturn_t ldc_tx(int irq, void *dev_id)
{
        struct ldc_channel *lp = dev_id;
        unsigned long flags, orig_state;
        unsigned int event_mask = 0;

        spin_lock_irqsave(&lp->lock, flags);

        orig_state = lp->chan_state;

        /* We should probably check for hypervisor errors here and
         * reset the LDC channel if we get one.
         */
        sun4v_ldc_tx_get_state(lp->id,
                               &lp->tx_head,
                               &lp->tx_tail,
                               &lp->chan_state);

        ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
               orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);

        if (lp->cfg.mode == LDC_MODE_RAW &&
            lp->chan_state == LDC_CHANNEL_UP) {
                lp->hs_state = LDC_HS_COMPLETE;
                ldc_set_state(lp, LDC_STATE_CONNECTED);

                /*
                 * Generate an LDC_EVENT_UP event if the channel
                 * was not already up.
                 */
                if (orig_state != LDC_CHANNEL_UP) {
                        event_mask |= LDC_EVENT_UP;
                        orig_state = lp->chan_state;
                }
        }

        spin_unlock_irqrestore(&lp->lock, flags);

        send_events(lp, event_mask);

        return IRQ_HANDLED;
}

/* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
 * XXX that addition and removal from the ldc_channel_list has
 * XXX atomicity, otherwise the __ldc_channel_exists() check is
 * XXX totally pointless as another thread can slip into ldc_alloc()
 * XXX and add a channel with the same ID.  There also needs to be
 * XXX a spinlock for ldc_channel_list.
 */
static HLIST_HEAD(ldc_channel_list);

static int __ldc_channel_exists(unsigned long id)
{
        struct ldc_channel *lp;

        hlist_for_each_entry(lp, &ldc_channel_list, list) {
                if (lp->id == id)
                        return 1;
        }
        return 0;
}

static int alloc_queue(const char *name, unsigned long num_entries,
                       struct ldc_packet **base, unsigned long *ra)
{
        unsigned long size, order;
        void *q;

        size = num_entries * LDC_PACKET_SIZE;
        order = get_order(size);

        q = (void *) __get_free_pages(GFP_KERNEL, order);
        if (!q) {
                printk(KERN_ERR PFX "Alloc of %s queue failed with "
                       "size=%lu order=%lu\n", name, size, order);
                return -ENOMEM;
        }

        memset(q, 0, PAGE_SIZE << order);

        *base = q;
        *ra = __pa(q);

        return 0;
}

static void free_queue(unsigned long num_entries, struct ldc_packet *q)
{
        unsigned long size, order;

        if (!q)
                return;

        size = num_entries * LDC_PACKET_SIZE;
        order = get_order(size);

        free_pages((unsigned long)q, order);
}

static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
{
        u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
        /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */

        cookie &= ~COOKIE_PGSZ_CODE;

        return (cookie >> (13ULL + (szcode * 3ULL)));
}

static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
                      unsigned long entry, unsigned long npages)
{
        struct ldc_mtable_entry *base;
        unsigned long i, shift;

        shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
        base = iommu->page_table + entry;
        for (i = 0; i < npages; i++) {
                if (base->cookie)
                        sun4v_ldc_revoke(id, cookie + (i << shift),
                                         base->cookie);
                base->mte = 0;
        }
}

/* XXX Make this configurable... XXX */
#define LDC_IOTABLE_SIZE        (8 * 1024)

static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
{
        unsigned long sz, num_tsb_entries, tsbsize, order;
        struct ldc_iommu *ldc_iommu = &lp->iommu;
        struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
        struct ldc_mtable_entry *table;
        unsigned long hv_err;
        int err;

        num_tsb_entries = LDC_IOTABLE_SIZE;
        tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
        spin_lock_init(&ldc_iommu->lock);

        sz = num_tsb_entries / 8;
        sz = (sz + 7UL) & ~7UL;
        iommu->map = kzalloc(sz, GFP_KERNEL);
        if (!iommu->map) {
                printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
                return -ENOMEM;
        }
        iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
                            NULL, false /* no large pool */,
                            1 /* npools */,
                            true /* skip span boundary check */);

        order = get_order(tsbsize);

        table = (struct ldc_mtable_entry *)
                __get_free_pages(GFP_KERNEL, order);
        err = -ENOMEM;
        if (!table) {
                printk(KERN_ERR PFX "Alloc of MTE table failed, "
                       "size=%lu order=%lu\n", tsbsize, order);
                goto out_free_map;
        }

        memset(table, 0, PAGE_SIZE << order);

        ldc_iommu->page_table = table;

        hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
                                         num_tsb_entries);
        err = -EINVAL;
        if (hv_err)
                goto out_free_table;

        return 0;

out_free_table:
        free_pages((unsigned long) table, order);
        ldc_iommu->page_table = NULL;

out_free_map:
        kfree(iommu->map);
        iommu->map = NULL;

        return err;
}

static void ldc_iommu_release(struct ldc_channel *lp)
{
        struct ldc_iommu *ldc_iommu = &lp->iommu;
        struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
        unsigned long num_tsb_entries, tsbsize, order;

        (void) sun4v_ldc_set_map_table(lp->id, 0, 0);

        num_tsb_entries = iommu->poolsize * iommu->nr_pools;
        tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
        order = get_order(tsbsize);

        free_pages((unsigned long) ldc_iommu->page_table, order);
        ldc_iommu->page_table = NULL;

        kfree(iommu->map);
        iommu->map = NULL;
}

struct ldc_channel *ldc_alloc(unsigned long id,
                              const struct ldc_channel_config *cfgp,
                              void *event_arg,
                              const char *name)
{
        struct ldc_channel *lp;
        const struct ldc_mode_ops *mops;
        unsigned long dummy1, dummy2, hv_err;
        u8 mss, *mssbuf;
        int err;

        err = -ENODEV;
        if (!ldom_domaining_enabled)
                goto out_err;

        err = -EINVAL;
        if (!cfgp)
                goto out_err;
        if (!name)
                goto out_err;

        switch (cfgp->mode) {
        case LDC_MODE_RAW:
                mops = &raw_ops;
                mss = LDC_PACKET_SIZE;
                break;

        case LDC_MODE_UNRELIABLE:
                mops = &nonraw_ops;
                mss = LDC_PACKET_SIZE - 8;
                break;

        case LDC_MODE_STREAM:
                mops = &stream_ops;
                mss = LDC_PACKET_SIZE - 8 - 8;
                break;

        default:
                goto out_err;
        }

        if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
                goto out_err;

        hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
        err = -ENODEV;
        if (hv_err == HV_ECHANNEL)
                goto out_err;

        err = -EEXIST;
        if (__ldc_channel_exists(id))
                goto out_err;

        mssbuf = NULL;

        lp = kzalloc_obj(*lp);
        err = -ENOMEM;
        if (!lp)
                goto out_err;

        spin_lock_init(&lp->lock);

        lp->id = id;

        err = ldc_iommu_init(name, lp);
        if (err)
                goto out_free_ldc;

        lp->mops = mops;
        lp->mss = mss;

        lp->cfg = *cfgp;
        if (!lp->cfg.mtu)
                lp->cfg.mtu = LDC_DEFAULT_MTU;

        if (lp->cfg.mode == LDC_MODE_STREAM) {
                mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
                if (!mssbuf) {
                        err = -ENOMEM;
                        goto out_free_iommu;
                }
                lp->mssbuf = mssbuf;
        }

        lp->event_arg = event_arg;

        /* XXX allow setting via ldc_channel_config to override defaults
         * XXX or use some formula based upon mtu
         */
        lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
        lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;

        err = alloc_queue("TX", lp->tx_num_entries,
                          &lp->tx_base, &lp->tx_ra);
        if (err)
                goto out_free_mssbuf;

        err = alloc_queue("RX", lp->rx_num_entries,
                          &lp->rx_base, &lp->rx_ra);
        if (err)
                goto out_free_txq;

        lp->flags |= LDC_FLAG_ALLOCED_QUEUES;

        lp->hs_state = LDC_HS_CLOSED;
        ldc_set_state(lp, LDC_STATE_INIT);

        INIT_HLIST_NODE(&lp->list);
        hlist_add_head(&lp->list, &ldc_channel_list);

        INIT_HLIST_HEAD(&lp->mh_list);

        snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
        snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);

        err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
                          lp->rx_irq_name, lp);
        if (err)
                goto out_free_txq;

        err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
                          lp->tx_irq_name, lp);
        if (err) {
                free_irq(lp->cfg.rx_irq, lp);
                goto out_free_txq;
        }

        return lp;

out_free_txq:
        free_queue(lp->tx_num_entries, lp->tx_base);

out_free_mssbuf:
        kfree(mssbuf);

out_free_iommu:
        ldc_iommu_release(lp);

out_free_ldc:
        kfree(lp);

out_err:
        return ERR_PTR(err);
}
EXPORT_SYMBOL(ldc_alloc);

void ldc_unbind(struct ldc_channel *lp)
{
        if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
                free_irq(lp->cfg.rx_irq, lp);
                free_irq(lp->cfg.tx_irq, lp);
                lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
        }

        if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
                sun4v_ldc_tx_qconf(lp->id, 0, 0);
                sun4v_ldc_rx_qconf(lp->id, 0, 0);
                lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
        }
        if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
                free_queue(lp->tx_num_entries, lp->tx_base);
                free_queue(lp->rx_num_entries, lp->rx_base);
                lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
        }

        ldc_set_state(lp, LDC_STATE_INIT);
}
EXPORT_SYMBOL(ldc_unbind);

void ldc_free(struct ldc_channel *lp)
{
        ldc_unbind(lp);
        hlist_del(&lp->list);
        kfree(lp->mssbuf);
        ldc_iommu_release(lp);

        kfree(lp);
}
EXPORT_SYMBOL(ldc_free);

/* Bind the channel.  This registers the LDC queues with
 * the hypervisor and puts the channel into a pseudo-listening
 * state.  This does not initiate a handshake, ldc_connect() does
 * that.
 */
int ldc_bind(struct ldc_channel *lp)
{
        unsigned long hv_err, flags;
        int err = -EINVAL;

        if (lp->state != LDC_STATE_INIT)
                return -EINVAL;

        spin_lock_irqsave(&lp->lock, flags);

        enable_irq(lp->cfg.rx_irq);
        enable_irq(lp->cfg.tx_irq);

        lp->flags |= LDC_FLAG_REGISTERED_IRQS;

        err = -ENODEV;
        hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
        if (hv_err)
                goto out_free_irqs;

        hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
        if (hv_err)
                goto out_free_irqs;

        hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
        if (hv_err)
                goto out_unmap_tx;

        hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
        if (hv_err)
                goto out_unmap_tx;

        lp->flags |= LDC_FLAG_REGISTERED_QUEUES;

        hv_err = sun4v_ldc_tx_get_state(lp->id,
                                        &lp->tx_head,
                                        &lp->tx_tail,
                                        &lp->chan_state);
        err = -EBUSY;
        if (hv_err)
                goto out_unmap_rx;

        lp->tx_acked = lp->tx_head;

        lp->hs_state = LDC_HS_OPEN;
        ldc_set_state(lp, LDC_STATE_BOUND);

        if (lp->cfg.mode == LDC_MODE_RAW) {
                /*
                 * There is no handshake in RAW mode, so handshake
                 * is completed.
                 */
                lp->hs_state = LDC_HS_COMPLETE;
        }

        spin_unlock_irqrestore(&lp->lock, flags);

        return 0;

out_unmap_rx:
        lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
        sun4v_ldc_rx_qconf(lp->id, 0, 0);

out_unmap_tx:
        sun4v_ldc_tx_qconf(lp->id, 0, 0);

out_free_irqs:
        lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
        free_irq(lp->cfg.tx_irq, lp);
        free_irq(lp->cfg.rx_irq, lp);

        spin_unlock_irqrestore(&lp->lock, flags);

        return err;
}
EXPORT_SYMBOL(ldc_bind);

int ldc_connect(struct ldc_channel *lp)
{
        unsigned long flags;
        int err;

        if (lp->cfg.mode == LDC_MODE_RAW)
                return -EINVAL;

        spin_lock_irqsave(&lp->lock, flags);

        if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
            !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
            lp->hs_state != LDC_HS_OPEN)
                err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
        else
                err = start_handshake(lp);

        spin_unlock_irqrestore(&lp->lock, flags);

        return err;
}
EXPORT_SYMBOL(ldc_connect);

int ldc_disconnect(struct ldc_channel *lp)
{
        unsigned long hv_err, flags;
        int err;

        if (lp->cfg.mode == LDC_MODE_RAW)
                return -EINVAL;

        if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
            !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
                return -EINVAL;

        spin_lock_irqsave(&lp->lock, flags);

        err = -ENODEV;
        hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
        if (hv_err)
                goto out_err;

        hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
        if (hv_err)
                goto out_err;

        hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
        if (hv_err)
                goto out_err;

        hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
        if (hv_err)
                goto out_err;

        ldc_set_state(lp, LDC_STATE_BOUND);
        lp->hs_state = LDC_HS_OPEN;
        lp->flags |= LDC_FLAG_RESET;

        spin_unlock_irqrestore(&lp->lock, flags);

        return 0;

out_err:
        sun4v_ldc_tx_qconf(lp->id, 0, 0);
        sun4v_ldc_rx_qconf(lp->id, 0, 0);
        free_irq(lp->cfg.tx_irq, lp);
        free_irq(lp->cfg.rx_irq, lp);
        lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
                       LDC_FLAG_REGISTERED_QUEUES);
        ldc_set_state(lp, LDC_STATE_INIT);

        spin_unlock_irqrestore(&lp->lock, flags);

        return err;
}
EXPORT_SYMBOL(ldc_disconnect);

int ldc_state(struct ldc_channel *lp)
{
        return lp->state;
}
EXPORT_SYMBOL(ldc_state);

void ldc_set_state(struct ldc_channel *lp, u8 state)
{
        ldcdbg(STATE, "STATE (%s) --> (%s)\n",
               state_to_str(lp->state),
               state_to_str(state));

        lp->state = state;
}
EXPORT_SYMBOL(ldc_set_state);

int ldc_mode(struct ldc_channel *lp)
{
        return lp->cfg.mode;
}
EXPORT_SYMBOL(ldc_mode);

int ldc_rx_reset(struct ldc_channel *lp)
{
        return __set_rx_head(lp, lp->rx_tail);
}
EXPORT_SYMBOL(ldc_rx_reset);

void __ldc_print(struct ldc_channel *lp, const char *caller)
{
        pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
                "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
                "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
                "\trcv_nxt=%u snd_nxt=%u\n",
                caller, lp->id, lp->flags, state_to_str(lp->state),
                lp->chan_state, lp->hs_state,
                lp->rx_head, lp->rx_tail, lp->rx_num_entries,
                lp->tx_head, lp->tx_tail, lp->tx_num_entries,
                lp->rcv_nxt, lp->snd_nxt);
}
EXPORT_SYMBOL(__ldc_print);

static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
{
        struct ldc_packet *p;
        unsigned long new_tail, hv_err;
        int err;

        hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
                                        &lp->chan_state);
        if (unlikely(hv_err))
                return -EBUSY;

        if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
                return LDC_ABORT(lp);

        if (size > LDC_PACKET_SIZE)
                return -EMSGSIZE;

        p = data_get_tx_packet(lp, &new_tail);
        if (!p)
                return -EAGAIN;

        memcpy(p, buf, size);

        err = send_tx_packet(lp, p, new_tail);
        if (!err)
                err = size;

        return err;
}

static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
{
        struct ldc_packet *p;
        unsigned long hv_err, new;
        int err;

        if (size < LDC_PACKET_SIZE)
                return -EINVAL;

        hv_err = sun4v_ldc_rx_get_state(lp->id,
                                        &lp->rx_head,
                                        &lp->rx_tail,
                                        &lp->chan_state);
        if (hv_err)
                return LDC_ABORT(lp);

        if (lp->chan_state == LDC_CHANNEL_DOWN ||
            lp->chan_state == LDC_CHANNEL_RESETTING)
                return -ECONNRESET;

        if (lp->rx_head == lp->rx_tail)
                return 0;

        p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
        memcpy(buf, p, LDC_PACKET_SIZE);

        new = rx_advance(lp, lp->rx_head);
        lp->rx_head = new;

        err = __set_rx_head(lp, new);
        if (err < 0)
                err = -ECONNRESET;
        else
                err = LDC_PACKET_SIZE;

        return err;
}

static const struct ldc_mode_ops raw_ops = {
        .write          =       write_raw,
        .read           =       read_raw,
};

static int write_nonraw(struct ldc_channel *lp, const void *buf,
                        unsigned int size)
{
        unsigned long hv_err, tail;
        unsigned int copied;
        u32 seq;
        int err;

        hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
                                        &lp->chan_state);
        if (unlikely(hv_err))
                return -EBUSY;

        if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
                return LDC_ABORT(lp);

        if (!tx_has_space_for(lp, size))
                return -EAGAIN;

        seq = lp->snd_nxt;
        copied = 0;
        tail = lp->tx_tail;
        while (copied < size) {
                struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
                u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
                            p->u.u_data :
                            p->u.r.r_data);
                int data_len;

                p->type = LDC_DATA;
                p->stype = LDC_INFO;
                p->ctrl = 0;

                data_len = size - copied;
                if (data_len > lp->mss)
                        data_len = lp->mss;

                BUG_ON(data_len > LDC_LEN);

                p->env = (data_len |
                          (copied == 0 ? LDC_START : 0) |
                          (data_len == size - copied ? LDC_STOP : 0));

                p->seqid = ++seq;

                ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
                       p->type,
                       p->stype,
                       p->ctrl,
                       p->env,
                       p->seqid);

                memcpy(data, buf, data_len);
                buf += data_len;
                copied += data_len;

                tail = tx_advance(lp, tail);
        }

        err = set_tx_tail(lp, tail);
        if (!err) {
                lp->snd_nxt = seq;
                err = size;
        }

        return err;
}

static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
                      struct ldc_packet *first_frag)
{
        int err;

        if (first_frag)
                lp->rcv_nxt = first_frag->seqid - 1;

        err = send_data_nack(lp, p);
        if (err)
                return err;

        err = ldc_rx_reset(lp);
        if (err < 0)
                return LDC_ABORT(lp);

        return 0;
}

static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
{
        if (p->stype & LDC_ACK) {
                int err = process_data_ack(lp, p);
                if (err)
                        return err;
        }
        if (p->stype & LDC_NACK)
                return LDC_ABORT(lp);

        return 0;
}

static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
{
        unsigned long dummy;
        int limit = 1000;

        ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
               cur_head, lp->rx_head, lp->rx_tail);
        while (limit-- > 0) {
                unsigned long hv_err;

                hv_err = sun4v_ldc_rx_get_state(lp->id,
                                                &dummy,
                                                &lp->rx_tail,
                                                &lp->chan_state);
                if (hv_err)
                        return LDC_ABORT(lp);

                if (lp->chan_state == LDC_CHANNEL_DOWN ||
                    lp->chan_state == LDC_CHANNEL_RESETTING)
                        return -ECONNRESET;

                if (cur_head != lp->rx_tail) {
                        ldcdbg(DATA, "DATA WAIT DONE "
                               "head[%lx] tail[%lx] chan_state[%lx]\n",
                               dummy, lp->rx_tail, lp->chan_state);
                        return 0;
                }

                udelay(1);
        }
        return -EAGAIN;
}

static int rx_set_head(struct ldc_channel *lp, unsigned long head)
{
        int err = __set_rx_head(lp, head);

        if (err < 0)
                return LDC_ABORT(lp);

        lp->rx_head = head;
        return 0;
}

static void send_data_ack(struct ldc_channel *lp)
{
        unsigned long new_tail;
        struct ldc_packet *p;

        p = data_get_tx_packet(lp, &new_tail);
        if (likely(p)) {
                int err;

                memset(p, 0, sizeof(*p));
                p->type = LDC_DATA;
                p->stype = LDC_ACK;
                p->ctrl = 0;
                p->seqid = lp->snd_nxt + 1;
                p->u.r.ackid = lp->rcv_nxt;

                err = send_tx_packet(lp, p, new_tail);
                if (!err)
                        lp->snd_nxt++;
        }
}

static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
{
        struct ldc_packet *first_frag;
        unsigned long hv_err, new;
        int err, copied;

        hv_err = sun4v_ldc_rx_get_state(lp->id,
                                        &lp->rx_head,
                                        &lp->rx_tail,
                                        &lp->chan_state);
        if (hv_err)
                return LDC_ABORT(lp);

        if (lp->chan_state == LDC_CHANNEL_DOWN ||
            lp->chan_state == LDC_CHANNEL_RESETTING)
                return -ECONNRESET;

        if (lp->rx_head == lp->rx_tail)
                return 0;

        first_frag = NULL;
        copied = err = 0;
        new = lp->rx_head;
        while (1) {
                struct ldc_packet *p;
                int pkt_len;

                BUG_ON(new == lp->rx_tail);
                p = lp->rx_base + (new / LDC_PACKET_SIZE);

                ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
                       "rcv_nxt[%08x]\n",
                       p->type,
                       p->stype,
                       p->ctrl,
                       p->env,
                       p->seqid,
                       p->u.r.ackid,
                       lp->rcv_nxt);

                if (unlikely(!rx_seq_ok(lp, p->seqid))) {
                        err = rx_bad_seq(lp, p, first_frag);
                        copied = 0;
                        break;
                }

                if (p->type & LDC_CTRL) {
                        err = process_control_frame(lp, p);
                        if (err < 0)
                                break;
                        err = 0;
                }

                lp->rcv_nxt = p->seqid;

                /*
                 * If this is a control-only packet, there is nothing
                 * else to do but advance the rx queue since the packet
                 * was already processed above.
                 */
                if (!(p->type & LDC_DATA)) {
                        new = rx_advance(lp, new);
                        break;
                }
                if (p->stype & (LDC_ACK | LDC_NACK)) {
                        err = data_ack_nack(lp, p);
                        if (err)
                                break;
                }
                if (!(p->stype & LDC_INFO)) {
                        new = rx_advance(lp, new);
                        err = rx_set_head(lp, new);
                        if (err)
                                break;
                        goto no_data;
                }

                pkt_len = p->env & LDC_LEN;

                /* Every initial packet starts with the START bit set.
                 *
                 * Singleton packets will have both START+STOP set.
                 *
                 * Fragments will have START set in the first frame, STOP
                 * set in the last frame, and neither bit set in middle
                 * frames of the packet.
                 *
                 * Therefore if we are at the beginning of a packet and
                 * we don't see START, or we are in the middle of a fragmented
                 * packet and do see START, we are unsynchronized and should
                 * flush the RX queue.
                 */
                if ((first_frag == NULL && !(p->env & LDC_START)) ||
                    (first_frag != NULL &&  (p->env & LDC_START))) {
                        if (!first_frag)
                                new = rx_advance(lp, new);

                        err = rx_set_head(lp, new);
                        if (err)
                                break;

                        if (!first_frag)
                                goto no_data;
                }
                if (!first_frag)
                        first_frag = p;

                if (pkt_len > size - copied) {
                        /* User didn't give us a big enough buffer,
                         * what to do?  This is a pretty serious error.
                         *
                         * Since we haven't updated the RX ring head to
                         * consume any of the packets, signal the error
                         * to the user and just leave the RX ring alone.
                         *
                         * This seems the best behavior because this allows
                         * a user of the LDC layer to start with a small
                         * RX buffer for ldc_read() calls and use -EMSGSIZE
                         * as a cue to enlarge its read buffer.
                         */
                        err = -EMSGSIZE;
                        break;
                }

                /* Ok, we are gonna eat this one.  */
                new = rx_advance(lp, new);

                memcpy(buf,
                       (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
                        p->u.u_data : p->u.r.r_data), pkt_len);
                buf += pkt_len;
                copied += pkt_len;

                if (p->env & LDC_STOP)
                        break;

no_data:
                if (new == lp->rx_tail) {
                        err = rx_data_wait(lp, new);
                        if (err)
                                break;
                }
        }

        if (!err)
                err = rx_set_head(lp, new);

        if (err && first_frag)
                lp->rcv_nxt = first_frag->seqid - 1;

        if (!err) {
                err = copied;
                if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
                        send_data_ack(lp);
        }

        return err;
}

static const struct ldc_mode_ops nonraw_ops = {
        .write          =       write_nonraw,
        .read           =       read_nonraw,
};

static int write_stream(struct ldc_channel *lp, const void *buf,
                        unsigned int size)
{
        if (size > lp->cfg.mtu)
                size = lp->cfg.mtu;
        return write_nonraw(lp, buf, size);
}

static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
{
        if (!lp->mssbuf_len) {
                int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
                if (err < 0)
                        return err;

                lp->mssbuf_len = err;
                lp->mssbuf_off = 0;
        }

        if (size > lp->mssbuf_len)
                size = lp->mssbuf_len;
        memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);

        lp->mssbuf_off += size;
        lp->mssbuf_len -= size;

        return size;
}

static const struct ldc_mode_ops stream_ops = {
        .write          =       write_stream,
        .read           =       read_stream,
};

int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
{
        unsigned long flags;
        int err;

        if (!buf)
                return -EINVAL;

        if (!size)
                return 0;

        spin_lock_irqsave(&lp->lock, flags);

        if (lp->hs_state != LDC_HS_COMPLETE)
                err = -ENOTCONN;
        else
                err = lp->mops->write(lp, buf, size);

        spin_unlock_irqrestore(&lp->lock, flags);

        return err;
}
EXPORT_SYMBOL(ldc_write);

int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
{
        unsigned long flags;
        int err;

        ldcdbg(RX, "%s: entered size=%d\n", __func__, size);

        if (!buf)
                return -EINVAL;

        if (!size)
                return 0;

        spin_lock_irqsave(&lp->lock, flags);

        if (lp->hs_state != LDC_HS_COMPLETE)
                err = -ENOTCONN;
        else
                err = lp->mops->read(lp, buf, size);

        spin_unlock_irqrestore(&lp->lock, flags);

        ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
               lp->cfg.mode, lp->rx_head, lp->rx_tail, err);

        return err;
}
EXPORT_SYMBOL(ldc_read);

static u64 pagesize_code(void)
{
        switch (PAGE_SIZE) {
        default:
        case (8ULL * 1024ULL):
                return 0;
        case (64ULL * 1024ULL):
                return 1;
        case (512ULL * 1024ULL):
                return 2;
        case (4ULL * 1024ULL * 1024ULL):
                return 3;
        case (32ULL * 1024ULL * 1024ULL):
                return 4;
        case (256ULL * 1024ULL * 1024ULL):
                return 5;
        }
}

static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
{
        return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
                (index << PAGE_SHIFT) |
                page_offset);
}


static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
                                             unsigned long npages)
{
        long entry;

        entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
                                      npages, NULL, (unsigned long)-1, 0);
        if (unlikely(entry == IOMMU_ERROR_CODE))
                return NULL;

        return iommu->page_table + entry;
}

static u64 perm_to_mte(unsigned int map_perm)
{
        u64 mte_base;

        mte_base = pagesize_code();

        if (map_perm & LDC_MAP_SHADOW) {
                if (map_perm & LDC_MAP_R)
                        mte_base |= LDC_MTE_COPY_R;
                if (map_perm & LDC_MAP_W)
                        mte_base |= LDC_MTE_COPY_W;
        }
        if (map_perm & LDC_MAP_DIRECT) {
                if (map_perm & LDC_MAP_R)
                        mte_base |= LDC_MTE_READ;
                if (map_perm & LDC_MAP_W)
                        mte_base |= LDC_MTE_WRITE;
                if (map_perm & LDC_MAP_X)
                        mte_base |= LDC_MTE_EXEC;
        }
        if (map_perm & LDC_MAP_IO) {
                if (map_perm & LDC_MAP_R)
                        mte_base |= LDC_MTE_IOMMU_R;
                if (map_perm & LDC_MAP_W)
                        mte_base |= LDC_MTE_IOMMU_W;
        }

        return mte_base;
}

static int pages_in_region(unsigned long base, long len)
{
        int count = 0;

        do {
                unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;

                len -= (new - base);
                base = new;
                count++;
        } while (len > 0);

        return count;
}

struct cookie_state {
        struct ldc_mtable_entry         *page_table;
        struct ldc_trans_cookie         *cookies;
        u64                             mte_base;
        u64                             prev_cookie;
        u32                             pte_idx;
        u32                             nc;
};

static void fill_cookies(struct cookie_state *sp, unsigned long pa,
                         unsigned long off, unsigned long len)
{
        do {
                unsigned long tlen, new = pa + PAGE_SIZE;
                u64 this_cookie;

                sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;

                tlen = PAGE_SIZE;
                if (off)
                        tlen = PAGE_SIZE - off;
                if (tlen > len)
                        tlen = len;

                this_cookie = make_cookie(sp->pte_idx,
                                          pagesize_code(), off);

                off = 0;

                if (this_cookie == sp->prev_cookie) {
                        sp->cookies[sp->nc - 1].cookie_size += tlen;
                } else {
                        sp->cookies[sp->nc].cookie_addr = this_cookie;
                        sp->cookies[sp->nc].cookie_size = tlen;
                        sp->nc++;
                }
                sp->prev_cookie = this_cookie + tlen;

                sp->pte_idx++;

                len -= tlen;
                pa = new;
        } while (len > 0);
}

static int sg_count_one(struct scatterlist *sg)
{
        unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
        long len = sg->length;

        if ((sg->offset | len) & (8UL - 1))
                return -EFAULT;

        return pages_in_region(base + sg->offset, len);
}

static int sg_count_pages(struct scatterlist *sg, int num_sg)
{
        int count;
        int i;

        count = 0;
        for (i = 0; i < num_sg; i++) {
                int err = sg_count_one(sg + i);
                if (err < 0)
                        return err;
                count += err;
        }

        return count;
}

int ldc_map_sg(struct ldc_channel *lp,
               struct scatterlist *sg, int num_sg,
               struct ldc_trans_cookie *cookies, int ncookies,
               unsigned int map_perm)
{
        unsigned long i, npages;
        struct ldc_mtable_entry *base;
        struct cookie_state state;
        struct ldc_iommu *iommu;
        int err;
        struct scatterlist *s;

        if (map_perm & ~LDC_MAP_ALL)
                return -EINVAL;

        err = sg_count_pages(sg, num_sg);
        if (err < 0)
                return err;

        npages = err;
        if (err > ncookies)
                return -EMSGSIZE;

        iommu = &lp->iommu;

        base = alloc_npages(iommu, npages);

        if (!base)
                return -ENOMEM;

        state.page_table = iommu->page_table;
        state.cookies = cookies;
        state.mte_base = perm_to_mte(map_perm);
        state.prev_cookie = ~(u64)0;
        state.pte_idx = (base - iommu->page_table);
        state.nc = 0;

        for_each_sg(sg, s, num_sg, i) {
                fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
                             s->offset, s->length);
        }

        return state.nc;
}
EXPORT_SYMBOL(ldc_map_sg);

int ldc_map_single(struct ldc_channel *lp,
                   void *buf, unsigned int len,
                   struct ldc_trans_cookie *cookies, int ncookies,
                   unsigned int map_perm)
{
        unsigned long npages, pa;
        struct ldc_mtable_entry *base;
        struct cookie_state state;
        struct ldc_iommu *iommu;

        if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
                return -EINVAL;

        pa = __pa(buf);
        if ((pa | len) & (8UL - 1))
                return -EFAULT;

        npages = pages_in_region(pa, len);

        iommu = &lp->iommu;

        base = alloc_npages(iommu, npages);

        if (!base)
                return -ENOMEM;

        state.page_table = iommu->page_table;
        state.cookies = cookies;
        state.mte_base = perm_to_mte(map_perm);
        state.prev_cookie = ~(u64)0;
        state.pte_idx = (base - iommu->page_table);
        state.nc = 0;
        fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
        BUG_ON(state.nc > ncookies);

        return state.nc;
}
EXPORT_SYMBOL(ldc_map_single);


static void free_npages(unsigned long id, struct ldc_iommu *iommu,
                        u64 cookie, u64 size)
{
        unsigned long npages, entry;

        npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;

        entry = ldc_cookie_to_index(cookie, iommu);
        ldc_demap(iommu, id, cookie, entry, npages);
        iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
}

void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
               int ncookies)
{
        struct ldc_iommu *iommu = &lp->iommu;
        int i;
        unsigned long flags;

        spin_lock_irqsave(&iommu->lock, flags);
        for (i = 0; i < ncookies; i++) {
                u64 addr = cookies[i].cookie_addr;
                u64 size = cookies[i].cookie_size;

                free_npages(lp->id, iommu, addr, size);
        }
        spin_unlock_irqrestore(&iommu->lock, flags);
}
EXPORT_SYMBOL(ldc_unmap);

int ldc_copy(struct ldc_channel *lp, int copy_dir,
             void *buf, unsigned int len, unsigned long offset,
             struct ldc_trans_cookie *cookies, int ncookies)
{
        unsigned int orig_len;
        unsigned long ra;
        int i;

        if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
                printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
                       lp->id, copy_dir);
                return -EINVAL;
        }

        ra = __pa(buf);
        if ((ra | len | offset) & (8UL - 1)) {
                printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
                       "ra[%lx] len[%x] offset[%lx]\n",
                       lp->id, ra, len, offset);
                return -EFAULT;
        }

        if (lp->hs_state != LDC_HS_COMPLETE ||
            (lp->flags & LDC_FLAG_RESET)) {
                printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
                       "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
                return -ECONNRESET;
        }

        orig_len = len;
        for (i = 0; i < ncookies; i++) {
                unsigned long cookie_raddr = cookies[i].cookie_addr;
                unsigned long this_len = cookies[i].cookie_size;
                unsigned long actual_len;

                if (unlikely(offset)) {
                        unsigned long this_off = offset;

                        if (this_off > this_len)
                                this_off = this_len;

                        offset -= this_off;
                        this_len -= this_off;
                        if (!this_len)
                                continue;
                        cookie_raddr += this_off;
                }

                if (this_len > len)
                        this_len = len;

                while (1) {
                        unsigned long hv_err;

                        hv_err = sun4v_ldc_copy(lp->id, copy_dir,
                                                cookie_raddr, ra,
                                                this_len, &actual_len);
                        if (unlikely(hv_err)) {
                                printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
                                       "HV error %lu\n",
                                       lp->id, hv_err);
                                if (lp->hs_state != LDC_HS_COMPLETE ||
                                    (lp->flags & LDC_FLAG_RESET))
                                        return -ECONNRESET;
                                else
                                        return -EFAULT;
                        }

                        cookie_raddr += actual_len;
                        ra += actual_len;
                        len -= actual_len;
                        if (actual_len == this_len)
                                break;

                        this_len -= actual_len;
                }

                if (!len)
                        break;
        }

        /* It is caller policy what to do about short copies.
         * For example, a networking driver can declare the
         * packet a runt and drop it.
         */

        return orig_len - len;
}
EXPORT_SYMBOL(ldc_copy);

void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
                          struct ldc_trans_cookie *cookies, int *ncookies,
                          unsigned int map_perm)
{
        void *buf;
        int err;

        if (len & (8UL - 1))
                return ERR_PTR(-EINVAL);

        buf = kzalloc(len, GFP_ATOMIC);
        if (!buf)
                return ERR_PTR(-ENOMEM);

        err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
        if (err < 0) {
                kfree(buf);
                return ERR_PTR(err);
        }
        *ncookies = err;

        return buf;
}
EXPORT_SYMBOL(ldc_alloc_exp_dring);

void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
                        struct ldc_trans_cookie *cookies, int ncookies)
{
        ldc_unmap(lp, cookies, ncookies);
        kfree(buf);
}
EXPORT_SYMBOL(ldc_free_exp_dring);

static int __init ldc_init(void)
{
        unsigned long major, minor;
        struct mdesc_handle *hp;
        const u64 *v;
        int err;
        u64 mp;

        hp = mdesc_grab();
        if (!hp)
                return -ENODEV;

        mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
        err = -ENODEV;
        if (mp == MDESC_NODE_NULL)
                goto out;

        v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
        if (!v)
                goto out;

        major = 1;
        minor = 0;
        if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
                printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
                goto out;
        }

        printk(KERN_INFO "%s", version);

        if (!*v) {
                printk(KERN_INFO PFX "Domaining disabled.\n");
                goto out;
        }
        ldom_domaining_enabled = 1;
        err = 0;

out:
        mdesc_release(hp);
        return err;
}

core_initcall(ldc_init);