/*-
 * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "icl_iser.h"

static MALLOC_DEFINE(M_ISER_INITIATOR, "iser_initiator", "iser initiator backend");

/* Register user buffer memory and initialize passive rdma
 *  dto descriptor. Data size is stored in
 *  task->data[ISER_DIR_IN].data_len, Protection size
 *  os stored in task->prot[ISER_DIR_IN].data_len
 */
static int
iser_prepare_read_cmd(struct icl_iser_pdu *iser_pdu)
{
        struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
        struct iser_data_buf *buf_in = &iser_pdu->data[ISER_DIR_IN];
        struct iser_mem_reg *mem_reg;
        int err;

        err = iser_dma_map_task_data(iser_pdu,
                                     buf_in,
                                     ISER_DIR_IN,
                                     DMA_FROM_DEVICE);
        if (err)
                return (err);

        err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_IN);
        if (err) {
                ISER_ERR("Failed to set up Data-IN RDMA");
                return (err);
        }

        mem_reg = &iser_pdu->rdma_reg[ISER_DIR_IN];

        hdr->flags    |= ISER_RSV;
        hdr->read_stag = cpu_to_be32(mem_reg->rkey);
        hdr->read_va   = cpu_to_be64(mem_reg->sge.addr);

        return (0);
}

/* Register user buffer memory and initialize passive rdma
 *  dto descriptor. Data size is stored in
 *  task->data[ISER_DIR_OUT].data_len, Protection size
 *  is stored at task->prot[ISER_DIR_OUT].data_len
 */
static int
iser_prepare_write_cmd(struct icl_iser_pdu *iser_pdu)
{
        struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
        struct iser_data_buf *buf_out = &iser_pdu->data[ISER_DIR_OUT];
        struct iser_mem_reg *mem_reg;
        int err;

        err = iser_dma_map_task_data(iser_pdu,
                                     buf_out,
                                     ISER_DIR_OUT,
                                     DMA_TO_DEVICE);
        if (err)
                return (err);

        err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_OUT);
        if (err) {
                ISER_ERR("Failed to set up Data-out RDMA");
                return (err);
        }

        mem_reg = &iser_pdu->rdma_reg[ISER_DIR_OUT];

        hdr->flags     |= ISER_WSV;
        hdr->write_stag = cpu_to_be32(mem_reg->rkey);
        hdr->write_va   = cpu_to_be64(mem_reg->sge.addr);

        return (0);
}

/* creates a new tx descriptor and adds header regd buffer */
void
iser_create_send_desc(struct iser_conn *iser_conn,
                      struct iser_tx_desc *tx_desc)
{
        struct iser_device *device = iser_conn->ib_conn.device;

        ib_dma_sync_single_for_cpu(device->ib_device,
                tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);

        memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
        tx_desc->iser_header.flags = ISER_VER;

        tx_desc->num_sge = 1;

        if (tx_desc->tx_sg[0].lkey != device->mr->lkey) {
                tx_desc->tx_sg[0].lkey = device->mr->lkey;
                ISER_DBG("sdesc %p lkey mismatch, fixing", tx_desc);
        }
}

void
iser_free_login_buf(struct iser_conn *iser_conn)
{
        struct iser_device *device = iser_conn->ib_conn.device;

        if (!iser_conn->login_buf)
                return;

        if (iser_conn->login_req_dma)
                ib_dma_unmap_single(device->ib_device,
                                    iser_conn->login_req_dma,
                                    ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);

        if (iser_conn->login_resp_dma)
                ib_dma_unmap_single(device->ib_device,
                                    iser_conn->login_resp_dma,
                                    ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);

        free(iser_conn->login_buf, M_ISER_INITIATOR);

        /* make sure we never redo any unmapping */
        iser_conn->login_req_dma = 0;
        iser_conn->login_resp_dma = 0;
        iser_conn->login_buf = NULL;
}

int
iser_alloc_login_buf(struct iser_conn *iser_conn)
{
        struct iser_device *device = iser_conn->ib_conn.device;
        int req_err, resp_err;

        BUG_ON(device == NULL);

        iser_conn->login_buf = malloc(ISCSI_DEF_MAX_RECV_SEG_LEN + ISER_RX_LOGIN_SIZE,
                                      M_ISER_INITIATOR, M_WAITOK | M_ZERO);

        if (!iser_conn->login_buf)
                goto out_err;

        iser_conn->login_req_buf  = iser_conn->login_buf;
        iser_conn->login_resp_buf = iser_conn->login_buf +
                                    ISCSI_DEF_MAX_RECV_SEG_LEN;

        iser_conn->login_req_dma = ib_dma_map_single(device->ib_device,
                                                     iser_conn->login_req_buf,
                                                     ISCSI_DEF_MAX_RECV_SEG_LEN,
                                                     DMA_TO_DEVICE);

        iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device,
                                                      iser_conn->login_resp_buf,
                                                      ISER_RX_LOGIN_SIZE,
                                                      DMA_FROM_DEVICE);

        req_err  = ib_dma_mapping_error(device->ib_device,
                                        iser_conn->login_req_dma);
        resp_err = ib_dma_mapping_error(device->ib_device,
                                        iser_conn->login_resp_dma);

        if (req_err || resp_err) {
                if (req_err)
                        iser_conn->login_req_dma = 0;
                if (resp_err)
                        iser_conn->login_resp_dma = 0;
                goto free_login_buf;
        }

        return (0);

free_login_buf:
        iser_free_login_buf(iser_conn);

out_err:
        ISER_DBG("unable to alloc or map login buf");
        return (ENOMEM);
}

int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, int cmds_max)
{
        int i, j;
        u64 dma_addr;
        struct iser_rx_desc *rx_desc;
        struct ib_sge       *rx_sg;
        struct ib_conn *ib_conn = &iser_conn->ib_conn;
        struct iser_device *device = ib_conn->device;

        iser_conn->qp_max_recv_dtos = cmds_max;
        iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;

        if (iser_create_fastreg_pool(ib_conn, cmds_max))
                goto create_rdma_reg_res_failed;


        iser_conn->num_rx_descs = cmds_max;
        iser_conn->rx_descs = malloc(iser_conn->num_rx_descs *
                                sizeof(struct iser_rx_desc), M_ISER_INITIATOR,
                                M_WAITOK | M_ZERO);
        if (!iser_conn->rx_descs)
                goto rx_desc_alloc_fail;

        rx_desc = iser_conn->rx_descs;

        for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)  {
                dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
                                        ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
                if (ib_dma_mapping_error(device->ib_device, dma_addr))
                        goto rx_desc_dma_map_failed;

                rx_desc->dma_addr = dma_addr;

                rx_sg = &rx_desc->rx_sg;
                rx_sg->addr   = rx_desc->dma_addr;
                rx_sg->length = ISER_RX_PAYLOAD_SIZE;
                rx_sg->lkey   = device->mr->lkey;
        }

        iser_conn->rx_desc_head = 0;

        return (0);

rx_desc_dma_map_failed:
        rx_desc = iser_conn->rx_descs;
        for (j = 0; j < i; j++, rx_desc++)
                ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
                                    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
        free(iser_conn->rx_descs, M_ISER_INITIATOR);
        iser_conn->rx_descs = NULL;
rx_desc_alloc_fail:
        iser_free_fastreg_pool(ib_conn);
create_rdma_reg_res_failed:
        ISER_ERR("failed allocating rx descriptors / data buffers");

        return (ENOMEM);
}

void
iser_free_rx_descriptors(struct iser_conn *iser_conn)
{
        int i;
        struct iser_rx_desc *rx_desc;
        struct ib_conn *ib_conn = &iser_conn->ib_conn;
        struct iser_device *device = ib_conn->device;

        iser_free_fastreg_pool(ib_conn);

        rx_desc = iser_conn->rx_descs;
        for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)
                ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
                                    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);

        free(iser_conn->rx_descs, M_ISER_INITIATOR);

        /* make sure we never redo any unmapping */
        iser_conn->rx_descs = NULL;
}

static void
iser_buf_to_sg(void *buf, struct iser_data_buf *data_buf)
{
        struct scatterlist *sg;
        int i;
        size_t len, tlen;
        int offset;

        tlen = data_buf->data_len;

        for (i = 0; 0 < tlen; i++, tlen -= len)  {
                sg = &data_buf->sgl[i];
                offset = ((uintptr_t)buf) & ~PAGE_MASK;
                len = min(PAGE_SIZE - offset, tlen);
                sg_set_buf(sg, buf, len);
                buf = (void *)((uintptr_t)buf + len);
        }

        data_buf->size = i;
        sg_mark_end(sg);
}


static void
iser_bio_to_sg(struct bio *bp, struct iser_data_buf *data_buf)
{
        struct scatterlist *sg;
        int i;
        size_t len, tlen;
        int offset;

        tlen = bp->bio_bcount;
        offset = bp->bio_ma_offset;

        for (i = 0; 0 < tlen; i++, tlen -= len) {
                sg = &data_buf->sgl[i];
                len = min(PAGE_SIZE - offset, tlen);
                sg_set_page(sg, bp->bio_ma[i], len, offset);
                offset = 0;
        }

        data_buf->size = i;
        sg_mark_end(sg);
}

static int
iser_csio_to_sg(struct ccb_scsiio *csio, struct iser_data_buf *data_buf)
{
        struct ccb_hdr *ccbh;
        int err = 0;

        ccbh = &csio->ccb_h;
        switch ((ccbh->flags & CAM_DATA_MASK)) {
                case CAM_DATA_BIO:
                        iser_bio_to_sg((struct bio *) csio->data_ptr, data_buf);
                        break;
                case CAM_DATA_VADDR:
                        /*
                         * Support KVA buffers for various scsi commands such as:
                         *  - REPORT_LUNS
                         *  - MODE_SENSE_6
                         *  - INQUIRY
                         *  - SERVICE_ACTION_IN.
                         * The data of these commands always mapped into KVA.
                         */
                        iser_buf_to_sg(csio->data_ptr, data_buf);
                        break;
                default:
                        ISER_ERR("flags 0x%X unimplemented", ccbh->flags);
                        err = EINVAL;
        }
        return (err);
}

static inline bool
iser_signal_comp(u8 sig_count)
{
        return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0);
}

int
iser_send_command(struct iser_conn *iser_conn,
                  struct icl_iser_pdu *iser_pdu)
{
        struct iser_data_buf *data_buf;
        struct iser_tx_desc *tx_desc = &iser_pdu->desc;
        struct iscsi_bhs_scsi_command *hdr = (struct iscsi_bhs_scsi_command *) &(iser_pdu->desc.iscsi_header);
        struct ccb_scsiio *csio = iser_pdu->csio;
        int err = 0;
        u8 sig_count = ++iser_conn->ib_conn.sig_count;

        /* build the tx desc regd header and add it to the tx desc dto */
        tx_desc->type = ISCSI_TX_SCSI_COMMAND;
        iser_create_send_desc(iser_conn, tx_desc);

        if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
                data_buf = &iser_pdu->data[ISER_DIR_IN];
        } else {
                data_buf = &iser_pdu->data[ISER_DIR_OUT];
        }

        data_buf->sg = csio->data_ptr;
        data_buf->data_len = csio->dxfer_len;

        if (likely(csio->dxfer_len)) {
                err = iser_csio_to_sg(csio, data_buf);
                if (unlikely(err))
                        goto send_command_error;
        }

        if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
                err = iser_prepare_read_cmd(iser_pdu);
                if (err)
                        goto send_command_error;
        } else if (hdr->bhssc_flags & BHSSC_FLAGS_W) {
                err = iser_prepare_write_cmd(iser_pdu);
                if (err)
                        goto send_command_error;
        }

        err = iser_post_send(&iser_conn->ib_conn, tx_desc,
                             iser_signal_comp(sig_count));
        if (!err)
                return (0);

send_command_error:
        ISER_ERR("iser_conn %p itt %u len %u err %d", iser_conn,
                        hdr->bhssc_initiator_task_tag,
                        hdr->bhssc_expected_data_transfer_length,
                        err);
        return (err);
}

int
iser_send_control(struct iser_conn *iser_conn,
                  struct icl_iser_pdu *iser_pdu)
{
        struct iser_tx_desc *mdesc;
        struct iser_device *device;
        size_t datalen = iser_pdu->icl_pdu.ip_data_len;
        int err;

        mdesc = &iser_pdu->desc;

        /* build the tx desc regd header and add it to the tx desc dto */
        mdesc->type = ISCSI_TX_CONTROL;
        iser_create_send_desc(iser_conn, mdesc);

        device = iser_conn->ib_conn.device;

        if (datalen > 0) {
                struct ib_sge *tx_dsg = &mdesc->tx_sg[1];
                ib_dma_sync_single_for_cpu(device->ib_device,
                                iser_conn->login_req_dma, datalen,
                                DMA_TO_DEVICE);

                ib_dma_sync_single_for_device(device->ib_device,
                        iser_conn->login_req_dma, datalen,
                        DMA_TO_DEVICE);

                tx_dsg->addr    = iser_conn->login_req_dma;
                tx_dsg->length  = datalen;
                tx_dsg->lkey    = device->mr->lkey;
                mdesc->num_sge = 2;
        }

        /* For login phase and discovery session we re-use the login buffer */
        if (!iser_conn->handoff_done) {
                err = iser_post_recvl(iser_conn);
                if (err)
                        goto send_control_error;
        }

        err = iser_post_send(&iser_conn->ib_conn, mdesc, true);
        if (!err)
                return (0);

send_control_error:
        ISER_ERR("conn %p failed err %d", iser_conn, err);

        return (err);

}

/**
 * iser_rcv_dto_completion - recv DTO completion
 */
void
iser_rcv_completion(struct iser_rx_desc *rx_desc,
                    unsigned long rx_xfer_len,
                    struct ib_conn *ib_conn)
{
        struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
                                                   ib_conn);
        struct icl_conn *ic = &iser_conn->icl_conn;
        struct icl_pdu *response;
        struct iscsi_bhs *hdr;
        u64 rx_dma;
        int rx_buflen;
        int outstanding, count, err;

        /* differentiate between login to all other PDUs */
        if ((char *)rx_desc == iser_conn->login_resp_buf) {
                rx_dma = iser_conn->login_resp_dma;
                rx_buflen = ISER_RX_LOGIN_SIZE;
        } else {
                rx_dma = rx_desc->dma_addr;
                rx_buflen = ISER_RX_PAYLOAD_SIZE;
        }

        ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
                                   rx_buflen, DMA_FROM_DEVICE);

        hdr = &rx_desc->iscsi_header;

        response = iser_new_pdu(ic, M_NOWAIT);
        response->ip_bhs = hdr;
        response->ip_data_len = rx_xfer_len - ISER_HEADERS_LEN;

        /*
         * In case we got data in the receive buffer, assign the ip_data_mbuf
         * to the rx_buffer - later we'll copy it to upper layer buffers
         */
        if (response->ip_data_len)
                response->ip_data_mbuf = (struct mbuf *)(rx_desc->data);

        ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
                                      rx_buflen, DMA_FROM_DEVICE);

        /* decrementing conn->post_recv_buf_count only --after-- freeing the   *
         * task eliminates the need to worry on tasks which are completed in   *
         * parallel to the execution of iser_conn_term. So the code that waits *
         * for the posted rx bufs refcount to become zero handles everything   */
        ib_conn->post_recv_buf_count--;

        if (rx_dma == iser_conn->login_resp_dma)
                goto receive;

        outstanding = ib_conn->post_recv_buf_count;
        if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) {
                count = min(iser_conn->qp_max_recv_dtos - outstanding,
                            iser_conn->min_posted_rx);
                err = iser_post_recvm(iser_conn, count);
                if (err)
                        ISER_ERR("posting %d rx bufs err %d", count, err);
        }

receive:
        (ic->ic_receive)(response);
}

void
iser_snd_completion(struct iser_tx_desc *tx_desc,
                    struct ib_conn *ib_conn)
{
        struct icl_iser_pdu *iser_pdu = container_of(tx_desc, struct icl_iser_pdu, desc);
        struct iser_conn *iser_conn = iser_pdu->iser_conn;

        if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL)
                iser_pdu_free(&iser_conn->icl_conn, &iser_pdu->icl_pdu);
}