/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>

#include <sys/ioat.h>

static int ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred);
static int ioat_close(dev_t devp, int flag, int otyp, cred_t *cred);
static int ioat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
static int ioat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
static int ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
    void **result);
static int ioat_quiesce(dev_info_t *dip);

static  struct cb_ops ioat_cb_ops = {
        ioat_open,              /* cb_open */
        ioat_close,             /* cb_close */
        nodev,                  /* cb_strategy */
        nodev,                  /* cb_print */
        nodev,                  /* cb_dump */
        nodev,                  /* cb_read */
        nodev,                  /* cb_write */
        ioat_ioctl,             /* cb_ioctl */
        nodev,                  /* cb_devmap */
        nodev,                  /* cb_mmap */
        nodev,                  /* cb_segmap */
        nochpoll,               /* cb_chpoll */
        ddi_prop_op,            /* cb_prop_op */
        NULL,                   /* cb_stream */
        D_NEW | D_MP | D_64BIT | D_DEVMAP,      /* cb_flag */
        CB_REV
};

static struct dev_ops ioat_dev_ops = {
        DEVO_REV,               /* devo_rev */
        0,                      /* devo_refcnt */
        ioat_getinfo,           /* devo_getinfo */
        nulldev,                /* devo_identify */
        nulldev,                /* devo_probe */
        ioat_attach,            /* devo_attach */
        ioat_detach,            /* devo_detach */
        nodev,                  /* devo_reset */
        &ioat_cb_ops,           /* devo_cb_ops */
        NULL,                   /* devo_bus_ops */
        NULL,                   /* devo_power */
        ioat_quiesce,           /* devo_quiesce */
};

static struct modldrv ioat_modldrv = {
        &mod_driverops,         /* Type of module.  This one is a driver */
        "ioat driver",          /* Name of the module. */
        &ioat_dev_ops,          /* driver ops */
};

static struct modlinkage ioat_modlinkage = {
        MODREV_1,
        (void *) &ioat_modldrv,
        NULL
};


void *ioat_statep;

static int ioat_chip_init(ioat_state_t *state);
static void ioat_chip_fini(ioat_state_t *state);
static int ioat_drv_init(ioat_state_t *state);
static void ioat_drv_fini(ioat_state_t *state);
static uint_t ioat_isr(caddr_t parm);
static void ioat_intr_enable(ioat_state_t *state);
static void ioat_intr_disable(ioat_state_t *state);
void ioat_detach_finish(ioat_state_t *state);


ddi_device_acc_attr_t ioat_acc_attr = {
        DDI_DEVICE_ATTR_V0,             /* devacc_attr_version */
        DDI_NEVERSWAP_ACC,              /* devacc_attr_endian_flags */
        DDI_STORECACHING_OK_ACC,        /* devacc_attr_dataorder */
        DDI_DEFAULT_ACC                 /* devacc_attr_access */
};

/* dcopy callback interface */
dcopy_device_cb_t ioat_cb = {
        DCOPY_DEVICECB_V0,
        0,              /* reserved */
        ioat_channel_alloc,
        ioat_channel_free,
        ioat_cmd_alloc,
        ioat_cmd_free,
        ioat_cmd_post,
        ioat_cmd_poll,
        ioat_unregister_complete
};

/*
 * _init()
 */
int
_init(void)
{
        int e;

        e = ddi_soft_state_init(&ioat_statep, sizeof (ioat_state_t), 1);
        if (e != 0) {
                return (e);
        }

        e = mod_install(&ioat_modlinkage);
        if (e != 0) {
                ddi_soft_state_fini(&ioat_statep);
                return (e);
        }

        return (0);
}

/*
 * _info()
 */
int
_info(struct modinfo *modinfop)
{
        return (mod_info(&ioat_modlinkage, modinfop));
}

/*
 * _fini()
 */
int
_fini(void)
{
        int e;

        e = mod_remove(&ioat_modlinkage);
        if (e != 0) {
                return (e);
        }

        ddi_soft_state_fini(&ioat_statep);

        return (0);
}

/*
 * ioat_attach()
 */
static int
ioat_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        ioat_state_t *state;
        int instance;
        int e;


        switch (cmd) {
        case DDI_ATTACH:
                break;

        case DDI_RESUME:
                instance = ddi_get_instance(dip);
                state = ddi_get_soft_state(ioat_statep, instance);
                if (state == NULL) {
                        return (DDI_FAILURE);
                }
                e = ioat_channel_resume(state);
                if (e != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }
                ioat_intr_enable(state);
                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);
        }

        instance = ddi_get_instance(dip);
        e = ddi_soft_state_zalloc(ioat_statep, instance);
        if (e != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }
        state = ddi_get_soft_state(ioat_statep, instance);
        if (state == NULL) {
                goto attachfail_get_soft_state;
        }

        state->is_dip = dip;
        state->is_instance = instance;

        /* setup the registers, save away some device info */
        e = ioat_chip_init(state);
        if (e != DDI_SUCCESS) {
                goto attachfail_chip_init;
        }

        /* initialize driver state, must be after chip init */
        e = ioat_drv_init(state);
        if (e != DDI_SUCCESS) {
                goto attachfail_drv_init;
        }

        /* create the minor node (for the ioctl) */
        e = ddi_create_minor_node(dip, "ioat", S_IFCHR, instance, DDI_PSEUDO,
            0);
        if (e != DDI_SUCCESS) {
                goto attachfail_minor_node;
        }

        /* Enable device interrupts */
        ioat_intr_enable(state);

        /* Report that driver was loaded */
        ddi_report_dev(dip);

        /* register with dcopy */
        e = dcopy_device_register(state, &state->is_deviceinfo,
            &state->is_device_handle);
        if (e != DCOPY_SUCCESS) {
                goto attachfail_register;
        }

        return (DDI_SUCCESS);

attachfail_register:
        ioat_intr_disable(state);
        ddi_remove_minor_node(dip, NULL);
attachfail_minor_node:
        ioat_drv_fini(state);
attachfail_drv_init:
        ioat_chip_fini(state);
attachfail_chip_init:
attachfail_get_soft_state:
        (void) ddi_soft_state_free(ioat_statep, instance);

        return (DDI_FAILURE);
}

/*
 * ioat_detach()
 */
static int
ioat_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        ioat_state_t *state;
        int instance;
        int e;


        instance = ddi_get_instance(dip);
        state = ddi_get_soft_state(ioat_statep, instance);
        if (state == NULL) {
                return (DDI_FAILURE);
        }

        switch (cmd) {
        case DDI_DETACH:
                break;

        case DDI_SUSPEND:
                ioat_channel_suspend(state);
                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);
        }

        /*
         * try to unregister from dcopy.  Since this driver doesn't follow the
         * traditional parent/child model, we may still be in use so we can't
         * detach yet.
         */
        e = dcopy_device_unregister(&state->is_device_handle);
        if (e != DCOPY_SUCCESS) {
                if (e == DCOPY_PENDING) {
                        cmn_err(CE_NOTE, "device busy, performing asynchronous"
                            " detach\n");
                }
                return (DDI_FAILURE);
        }

        ioat_detach_finish(state);

        return (DDI_SUCCESS);
}

/*
 * ioat_getinfo()
 */
/*ARGSUSED*/
static int
ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
        ioat_state_t *state;
        int instance;
        dev_t dev;
        int e;


        dev = (dev_t)arg;
        instance = getminor(dev);

        switch (cmd) {
        case DDI_INFO_DEVT2DEVINFO:
                state = ddi_get_soft_state(ioat_statep, instance);
                if (state == NULL) {
                        return (DDI_FAILURE);
                }
                *result = (void *)state->is_dip;
                e = DDI_SUCCESS;
                break;

        case DDI_INFO_DEVT2INSTANCE:
                *result = (void *)(uintptr_t)instance;
                e = DDI_SUCCESS;
                break;

        default:
                e = DDI_FAILURE;
                break;
        }

        return (e);
}


/*
 * ioat_open()
 */
/*ARGSUSED*/
static int
ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred)
{
        ioat_state_t *state;
        int instance;

        instance = getminor(*devp);
        state = ddi_get_soft_state(ioat_statep, instance);
        if (state == NULL) {
                return (ENXIO);
        }

        return (0);
}


/*
 * ioat_close()
 */
/*ARGSUSED*/
static int
ioat_close(dev_t devp, int flag, int otyp, cred_t *cred)
{
        return (0);
}


/*
 * ioat_chip_init()
 */
static int
ioat_chip_init(ioat_state_t *state)
{
        ddi_device_acc_attr_t attr;
        int e;


        attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
        attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
        attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;

        e =  ddi_regs_map_setup(state->is_dip, 1, (caddr_t *)&state->is_genregs,
            0, 0, &attr, &state->is_reg_handle);
        if (e != DDI_SUCCESS) {
                goto chipinitfail_regsmap;
        }

        /* save away ioat chip info */
        state->is_num_channels = (uint_t)ddi_get8(state->is_reg_handle,
            &state->is_genregs[IOAT_CHANCNT]);

        /*
         * If we get a bogus value, something is wrong with the H/W, fail to
         * attach.
         */
        if (state->is_num_channels == 0) {
                goto chipinitfail_numchan;
        }

        state->is_maxxfer = (uint_t)ddi_get8(state->is_reg_handle,
            &state->is_genregs[IOAT_XFERCAP]);
        state->is_chanoff = (uintptr_t)ddi_get16(state->is_reg_handle,
            (uint16_t *)&state->is_genregs[IOAT_PERPORT_OFF]);
        state->is_cbver = (uint_t)ddi_get8(state->is_reg_handle,
            &state->is_genregs[IOAT_CBVER]);
        state->is_intrdelay = (uint_t)ddi_get16(state->is_reg_handle,
            (uint16_t *)&state->is_genregs[IOAT_INTRDELAY]);
        state->is_status = (uint_t)ddi_get16(state->is_reg_handle,
            (uint16_t *)&state->is_genregs[IOAT_CSSTATUS]);
        state->is_capabilities = (uint_t)ddi_get32(state->is_reg_handle,
            (uint32_t *)&state->is_genregs[IOAT_DMACAPABILITY]);

        if (state->is_cbver & 0x10) {
                state->is_ver = IOAT_CBv1;
        } else if (state->is_cbver & 0x20) {
                state->is_ver = IOAT_CBv2;
        } else {
                goto chipinitfail_version;
        }

        return (DDI_SUCCESS);

chipinitfail_version:
chipinitfail_numchan:
        ddi_regs_map_free(&state->is_reg_handle);
chipinitfail_regsmap:
        return (DDI_FAILURE);
}


/*
 * ioat_chip_fini()
 */
static void
ioat_chip_fini(ioat_state_t *state)
{
        ddi_regs_map_free(&state->is_reg_handle);
}


/*
 * ioat_drv_init()
 */
static int
ioat_drv_init(ioat_state_t *state)
{
        ddi_acc_handle_t handle;
        int e;


        mutex_init(&state->is_mutex, NULL, MUTEX_DRIVER, NULL);

        state->is_deviceinfo.di_dip = state->is_dip;
        state->is_deviceinfo.di_num_dma = state->is_num_channels;
        state->is_deviceinfo.di_maxxfer = state->is_maxxfer;
        state->is_deviceinfo.di_capabilities = state->is_capabilities;
        state->is_deviceinfo.di_cb = &ioat_cb;

        e = pci_config_setup(state->is_dip, &handle);
        if (e != DDI_SUCCESS) {
                goto drvinitfail_config_setup;
        }

        /* read in Vendor ID */
        state->is_deviceinfo.di_id = (uint64_t)pci_config_get16(handle, 0);
        state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 16;

        /* read in Device ID */
        state->is_deviceinfo.di_id |= (uint64_t)pci_config_get16(handle, 2);
        state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 32;

        /* Add in chipset version */
        state->is_deviceinfo.di_id |= (uint64_t)state->is_cbver;
        pci_config_teardown(&handle);

        e = ddi_intr_hilevel(state->is_dip, 0);
        if (e != 0) {
                cmn_err(CE_WARN, "hilevel interrupt not supported\n");
                goto drvinitfail_hilevel;
        }

        /* we don't support MSIs for v2 yet */
        e = ddi_add_intr(state->is_dip, 0, NULL, NULL, ioat_isr,
            (caddr_t)state);
        if (e != DDI_SUCCESS) {
                goto drvinitfail_add_intr;
        }

        e = ddi_get_iblock_cookie(state->is_dip, 0, &state->is_iblock_cookie);
        if (e != DDI_SUCCESS) {
                goto drvinitfail_iblock_cookie;
        }

        e = ioat_channel_init(state);
        if (e != DDI_SUCCESS) {
                goto drvinitfail_channel_init;
        }

        return (DDI_SUCCESS);

drvinitfail_channel_init:
drvinitfail_iblock_cookie:
        ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
drvinitfail_add_intr:
drvinitfail_hilevel:
drvinitfail_config_setup:
        mutex_destroy(&state->is_mutex);

        return (DDI_FAILURE);
}


/*
 * ioat_drv_fini()
 */
static void
ioat_drv_fini(ioat_state_t *state)
{
        ioat_channel_fini(state);
        ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
        mutex_destroy(&state->is_mutex);
}


/*
 * ioat_unregister_complete()
 */
void
ioat_unregister_complete(void *device_private, int status)
{
        ioat_state_t *state;


        state = device_private;

        if (status != DCOPY_SUCCESS) {
                cmn_err(CE_WARN, "asynchronous detach aborted\n");
                return;
        }

        cmn_err(CE_CONT, "detach completing\n");
        ioat_detach_finish(state);
}


/*
 * ioat_detach_finish()
 */
void
ioat_detach_finish(ioat_state_t *state)
{
        ioat_intr_disable(state);
        ddi_remove_minor_node(state->is_dip, NULL);
        ioat_drv_fini(state);
        ioat_chip_fini(state);
        (void) ddi_soft_state_free(ioat_statep, state->is_instance);
}


/*
 * ioat_intr_enable()
 */
static void
ioat_intr_enable(ioat_state_t *state)
{
        uint32_t intr_status;


        /* Clear any pending interrupts */
        intr_status = ddi_get32(state->is_reg_handle,
            (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
        if (intr_status != 0) {
                ddi_put32(state->is_reg_handle,
                    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS],
                    intr_status);
        }

        /* Enable interrupts on the device */
        ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
            IOAT_INTRCTL_MASTER_EN);
}


/*
 * ioat_intr_disable()
 */
static void
ioat_intr_disable(ioat_state_t *state)
{
        /*
         * disable interrupts on the device. A read of the interrupt control
         * register clears the enable bit.
         */
        (void) ddi_get8(state->is_reg_handle,
            &state->is_genregs[IOAT_INTRCTL]);
}


/*
 * ioat_isr()
 */
static uint_t
ioat_isr(caddr_t parm)
{
        uint32_t intr_status;
        ioat_state_t *state;
        uint8_t intrctrl;
        uint32_t chan;
        uint_t r;
        int i;

        state = (ioat_state_t *)parm;

        intrctrl = ddi_get8(state->is_reg_handle,
            &state->is_genregs[IOAT_INTRCTL]);
        /* master interrupt enable should always be set */
        ASSERT(intrctrl & IOAT_INTRCTL_MASTER_EN);

        /* If the interrupt status bit isn't set, it's not ours */
        if (!(intrctrl & IOAT_INTRCTL_INTR_STAT)) {
                /* re-set master interrupt enable (since it clears on read) */
                ddi_put8(state->is_reg_handle,
                    &state->is_genregs[IOAT_INTRCTL], intrctrl);
                return (DDI_INTR_UNCLAIMED);
        }

        /* see which channels generated the interrupt */
        intr_status = ddi_get32(state->is_reg_handle,
            (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);

        /* call the intr handler for the channels */
        r = DDI_INTR_UNCLAIMED;
        chan = 1;
        for (i = 0; i < state->is_num_channels; i++) {
                if (intr_status & chan) {
                        ioat_channel_intr(&state->is_channel[i]);
                        r = DDI_INTR_CLAIMED;
                }
                chan = chan << 1;
        }

        /*
         * if interrupt status bit was set, there should have been an
         * attention status bit set too.
         */
        ASSERT(r == DDI_INTR_CLAIMED);

        /* re-set master interrupt enable (since it clears on read) */
        ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
            intrctrl);

        return (r);
}

static int
ioat_quiesce(dev_info_t *dip)
{
        ioat_state_t *state;
        int instance;

        instance = ddi_get_instance(dip);
        state = ddi_get_soft_state(ioat_statep, instance);
        if (state == NULL) {
                return (DDI_FAILURE);
        }

        ioat_intr_disable(state);
        ioat_channel_quiesce(state);

        return (DDI_SUCCESS);
}