/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * dcopy.c
 *    dcopy misc module
 */

#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/sysmacros.h>
#include <sys/atomic.h>


#include <sys/dcopy.h>
#include <sys/dcopy_device.h>


/* Number of entries per channel to allocate */
uint_t dcopy_channel_size = 1024;


typedef struct dcopy_list_s {
        list_t                  dl_list;
        kmutex_t                dl_mutex;
        uint_t                  dl_cnt; /* num entries on list */
} dcopy_list_t;

/* device state for register/unregister */
struct dcopy_device_s {
        /* DMA device drivers private pointer */
        void                    *dc_device_private;

        /* to track list of channels from this DMA device */
        dcopy_list_t            dc_devchan_list;
        list_node_t             dc_device_list_node;

        /*
         * dc_removing_cnt track how many channels still have to be freed up
         * before it's safe to allow the DMA device driver to detach.
         */
        uint_t                  dc_removing_cnt;
        dcopy_device_cb_t       *dc_cb;

        dcopy_device_info_t     dc_info;

};

typedef struct dcopy_stats_s {
        kstat_named_t   cs_bytes_xfer;
        kstat_named_t   cs_cmd_alloc;
        kstat_named_t   cs_cmd_post;
        kstat_named_t   cs_cmd_poll;
        kstat_named_t   cs_notify_poll;
        kstat_named_t   cs_notify_pending;
        kstat_named_t   cs_id;
        kstat_named_t   cs_capabilities;
} dcopy_stats_t;

/* DMA channel state */
struct dcopy_channel_s {
        /* DMA driver channel private pointer */
        void                    *ch_channel_private;

        /* shortcut to device callbacks */
        dcopy_device_cb_t       *ch_cb;

        /*
         * number of outstanding allocs for this channel. used to track when
         * it's safe to free up this channel so the DMA device driver can
         * detach.
         */
        uint64_t                ch_ref_cnt;

        /* state for if channel needs to be removed when ch_ref_cnt gets to 0 */
        boolean_t               ch_removing;

        list_node_t             ch_devchan_list_node;
        list_node_t             ch_globalchan_list_node;

        /*
         * per channel list of commands actively blocking waiting for
         * completion.
         */
        dcopy_list_t            ch_poll_list;

        /* pointer back to our device */
        struct dcopy_device_s   *ch_device;

        dcopy_query_channel_t   ch_info;

        kstat_t                 *ch_kstat;
        dcopy_stats_t           ch_stat;
};

/*
 * If grabbing both device_list mutex & globalchan_list mutex,
 * Always grab globalchan_list mutex before device_list mutex
 */
typedef struct dcopy_state_s {
        dcopy_list_t            d_device_list;
        dcopy_list_t            d_globalchan_list;
} dcopy_state_t;
dcopy_state_t *dcopy_statep;


/* Module Driver Info */
static struct modlmisc dcopy_modlmisc = {
        &mod_miscops,
        "dcopy kernel module"
};

/* Module Linkage */
static struct modlinkage dcopy_modlinkage = {
        MODREV_1,
        &dcopy_modlmisc,
        NULL
};

static int dcopy_init();
static void dcopy_fini();

static int dcopy_list_init(dcopy_list_t *list, size_t node_size,
    offset_t link_offset);
static void dcopy_list_fini(dcopy_list_t *list);
static void dcopy_list_push(dcopy_list_t *list, void *list_node);
static void *dcopy_list_pop(dcopy_list_t *list);

static void dcopy_device_cleanup(dcopy_device_handle_t device,
    boolean_t do_callback);

static int dcopy_stats_init(dcopy_handle_t channel);
static void dcopy_stats_fini(dcopy_handle_t channel);


/*
 * _init()
 */
int
_init()
{
        int e;

        e = dcopy_init();
        if (e != 0) {
                return (e);
        }

        return (mod_install(&dcopy_modlinkage));
}


/*
 * _info()
 */
int
_info(struct modinfo *modinfop)
{
        return (mod_info(&dcopy_modlinkage, modinfop));
}


/*
 * _fini()
 */
int
_fini()
{
        int e;

        e = mod_remove(&dcopy_modlinkage);
        if (e != 0) {
                return (e);
        }

        dcopy_fini();

        return (e);
}

/*
 * dcopy_init()
 */
static int
dcopy_init()
{
        int e;


        dcopy_statep = kmem_zalloc(sizeof (*dcopy_statep), KM_SLEEP);

        /* Initialize the list we use to track device register/unregister */
        e = dcopy_list_init(&dcopy_statep->d_device_list,
            sizeof (struct dcopy_device_s),
            offsetof(struct dcopy_device_s, dc_device_list_node));
        if (e != DCOPY_SUCCESS) {
                goto dcopyinitfail_device;
        }

        /* Initialize the list we use to track all DMA channels */
        e = dcopy_list_init(&dcopy_statep->d_globalchan_list,
            sizeof (struct dcopy_channel_s),
            offsetof(struct dcopy_channel_s, ch_globalchan_list_node));
        if (e != DCOPY_SUCCESS) {
                goto dcopyinitfail_global;
        }

        return (0);

dcopyinitfail_global:
        dcopy_list_fini(&dcopy_statep->d_device_list);
dcopyinitfail_device:
        kmem_free(dcopy_statep, sizeof (*dcopy_statep));

        return (-1);
}


/*
 * dcopy_fini()
 */
static void
dcopy_fini()
{
        /*
         * if mod_remove was successfull, we shouldn't have any
         * devices/channels to worry about.
         */
        ASSERT(list_head(&dcopy_statep->d_globalchan_list.dl_list) == NULL);
        ASSERT(list_head(&dcopy_statep->d_device_list.dl_list) == NULL);

        dcopy_list_fini(&dcopy_statep->d_globalchan_list);
        dcopy_list_fini(&dcopy_statep->d_device_list);
        kmem_free(dcopy_statep, sizeof (*dcopy_statep));
}


/* *** EXTERNAL INTERFACE *** */
/*
 * dcopy_query()
 */
void
dcopy_query(dcopy_query_t *query)
{
        query->dq_version = DCOPY_QUERY_V0;
        query->dq_num_channels = dcopy_statep->d_globalchan_list.dl_cnt;
}


/*
 * dcopy_alloc()
 */
/*ARGSUSED*/
int
dcopy_alloc(int flags, dcopy_handle_t *handle)
{
        dcopy_handle_t channel;
        dcopy_list_t *list;


        /*
         * we don't use the dcopy_list_* code here because we need to due
         * some non-standard stuff.
         */

        list = &dcopy_statep->d_globalchan_list;

        /*
         * if nothing is on the channel list, return DCOPY_NORESOURCES. This
         * can happen if there aren't any DMA device registered.
         */
        mutex_enter(&list->dl_mutex);
        channel = list_head(&list->dl_list);
        if (channel == NULL) {
                mutex_exit(&list->dl_mutex);
                return (DCOPY_NORESOURCES);
        }

        /*
         * increment the reference count, and pop the channel off the head and
         * push it on the tail. This ensures we rotate through the channels.
         * DMA channels are shared.
         */
        channel->ch_ref_cnt++;
        list_remove(&list->dl_list, channel);
        list_insert_tail(&list->dl_list, channel);
        mutex_exit(&list->dl_mutex);

        *handle = (dcopy_handle_t)channel;
        return (DCOPY_SUCCESS);
}


/*
 * dcopy_free()
 */
void
dcopy_free(dcopy_handle_t *channel)
{
        dcopy_device_handle_t device;
        dcopy_list_t *list;
        boolean_t cleanup = B_FALSE;


        ASSERT(*channel != NULL);

        /*
         * we don't need to add the channel back to the list since we never
         * removed it. decrement the reference count.
         */
        list = &dcopy_statep->d_globalchan_list;
        mutex_enter(&list->dl_mutex);
        (*channel)->ch_ref_cnt--;

        /*
         * if we need to remove this channel, and the reference count is down
         * to 0, decrement the number of channels which still need to be
         * removed on the device.
         */
        if ((*channel)->ch_removing && ((*channel)->ch_ref_cnt == 0)) {
                device = (*channel)->ch_device;
                mutex_enter(&device->dc_devchan_list.dl_mutex);
                device->dc_removing_cnt--;
                if (device->dc_removing_cnt == 0) {
                        cleanup = B_TRUE;
                }
                mutex_exit(&device->dc_devchan_list.dl_mutex);
        }
        mutex_exit(&list->dl_mutex);

        /*
         * if there are no channels which still need to be removed, cleanup the
         * device state and call back into the DMA device driver to tell them
         * the device is free.
         */
        if (cleanup) {
                dcopy_device_cleanup(device, B_TRUE);
        }

        *channel = NULL;
}


/*
 * dcopy_query_channel()
 */
void
dcopy_query_channel(dcopy_handle_t channel, dcopy_query_channel_t *query)
{
        *query = channel->ch_info;
}


/*
 * dcopy_cmd_alloc()
 */
int
dcopy_cmd_alloc(dcopy_handle_t handle, int flags, dcopy_cmd_t *cmd)
{
        dcopy_handle_t channel;
        dcopy_cmd_priv_t priv;
        int e;


        channel = handle;

        atomic_inc_64(&channel->ch_stat.cs_cmd_alloc.value.ui64);
        e = channel->ch_cb->cb_cmd_alloc(channel->ch_channel_private, flags,
            cmd);
        if (e == DCOPY_SUCCESS) {
                priv = (*cmd)->dp_private;
                priv->pr_channel = channel;
                /*
                 * we won't initialize the blocking state until we actually
                 * need to block.
                 */
                priv->pr_block_init = B_FALSE;
        }

        return (e);
}


/*
 * dcopy_cmd_free()
 */
void
dcopy_cmd_free(dcopy_cmd_t *cmd)
{
        dcopy_handle_t channel;
        dcopy_cmd_priv_t priv;


        ASSERT(*cmd != NULL);

        priv = (*cmd)->dp_private;
        channel = priv->pr_channel;

        /* if we initialized the blocking state, clean it up too */
        if (priv->pr_block_init) {
                cv_destroy(&priv->pr_cv);
                mutex_destroy(&priv->pr_mutex);
        }

        channel->ch_cb->cb_cmd_free(channel->ch_channel_private, cmd);
}


/*
 * dcopy_cmd_post()
 */
int
dcopy_cmd_post(dcopy_cmd_t cmd)
{
        dcopy_handle_t channel;
        int e;


        channel = cmd->dp_private->pr_channel;

        atomic_inc_64(&channel->ch_stat.cs_cmd_post.value.ui64);
        if (cmd->dp_cmd == DCOPY_CMD_COPY) {
                atomic_add_64(&channel->ch_stat.cs_bytes_xfer.value.ui64,
                    cmd->dp.copy.cc_size);
        }
        e = channel->ch_cb->cb_cmd_post(channel->ch_channel_private, cmd);
        if (e != DCOPY_SUCCESS) {
                return (e);
        }

        return (DCOPY_SUCCESS);
}


/*
 * dcopy_cmd_poll()
 */
int
dcopy_cmd_poll(dcopy_cmd_t cmd, int flags)
{
        dcopy_handle_t channel;
        dcopy_cmd_priv_t priv;
        int e;


        priv = cmd->dp_private;
        channel = priv->pr_channel;

        /*
         * if the caller is trying to block, they needed to post the
         * command with DCOPY_CMD_INTR set.
         */
        if ((flags & DCOPY_POLL_BLOCK) && !(cmd->dp_flags & DCOPY_CMD_INTR)) {
                return (DCOPY_FAILURE);
        }

        atomic_inc_64(&channel->ch_stat.cs_cmd_poll.value.ui64);

repoll:
        e = channel->ch_cb->cb_cmd_poll(channel->ch_channel_private, cmd);
        if (e == DCOPY_PENDING) {
                /*
                 * if the command is still active, and the blocking flag
                 * is set.
                 */
                if (flags & DCOPY_POLL_BLOCK) {

                        /*
                         * if we haven't initialized the state, do it now. A
                         * command can be re-used, so it's possible it's
                         * already been initialized.
                         */
                        if (!priv->pr_block_init) {
                                priv->pr_block_init = B_TRUE;
                                mutex_init(&priv->pr_mutex, NULL, MUTEX_DRIVER,
                                    NULL);
                                cv_init(&priv->pr_cv, NULL, CV_DRIVER, NULL);
                                priv->pr_cmd = cmd;
                        }

                        /* push it on the list for blocking commands */
                        priv->pr_wait = B_TRUE;
                        dcopy_list_push(&channel->ch_poll_list, priv);

                        mutex_enter(&priv->pr_mutex);
                        /*
                         * it's possible we already cleared pr_wait before we
                         * grabbed the mutex.
                         */
                        if (priv->pr_wait) {
                                cv_wait(&priv->pr_cv, &priv->pr_mutex);
                        }
                        mutex_exit(&priv->pr_mutex);

                        /*
                         * the command has completed, go back and poll so we
                         * get the status.
                         */
                        goto repoll;
                }
        }

        return (e);
}

/* *** END OF EXTERNAL INTERFACE *** */

/*
 * dcopy_list_init()
 */
static int
dcopy_list_init(dcopy_list_t *list, size_t node_size, offset_t link_offset)
{
        mutex_init(&list->dl_mutex, NULL, MUTEX_DRIVER, NULL);
        list_create(&list->dl_list, node_size, link_offset);
        list->dl_cnt = 0;

        return (DCOPY_SUCCESS);
}


/*
 * dcopy_list_fini()
 */
static void
dcopy_list_fini(dcopy_list_t *list)
{
        list_destroy(&list->dl_list);
        mutex_destroy(&list->dl_mutex);
}


/*
 * dcopy_list_push()
 */
static void
dcopy_list_push(dcopy_list_t *list, void *list_node)
{
        mutex_enter(&list->dl_mutex);
        list_insert_tail(&list->dl_list, list_node);
        list->dl_cnt++;
        mutex_exit(&list->dl_mutex);
}


/*
 * dcopy_list_pop()
 */
static void *
dcopy_list_pop(dcopy_list_t *list)
{
        list_node_t *list_node;

        mutex_enter(&list->dl_mutex);
        list_node = list_head(&list->dl_list);
        if (list_node == NULL) {
                mutex_exit(&list->dl_mutex);
                return (list_node);
        }
        list->dl_cnt--;
        list_remove(&list->dl_list, list_node);
        mutex_exit(&list->dl_mutex);

        return (list_node);
}


/* *** DEVICE INTERFACE *** */
/*
 * dcopy_device_register()
 */
int
dcopy_device_register(void *device_private, dcopy_device_info_t *info,
    dcopy_device_handle_t *handle)
{
        struct dcopy_channel_s *channel;
        struct dcopy_device_s *device;
        int e;
        int i;


        /* initialize the per device state */
        device = kmem_zalloc(sizeof (*device), KM_SLEEP);
        device->dc_device_private = device_private;
        device->dc_info = *info;
        device->dc_removing_cnt = 0;
        device->dc_cb = info->di_cb;

        /*
         * we have a per device channel list so we can remove a device in the
         * future.
         */
        e = dcopy_list_init(&device->dc_devchan_list,
            sizeof (struct dcopy_channel_s),
            offsetof(struct dcopy_channel_s, ch_devchan_list_node));
        if (e != DCOPY_SUCCESS) {
                goto registerfail_devchan;
        }

        /*
         * allocate state for each channel, allocate the channel,  and then add
         * the devices dma channels to the devices channel list.
         */
        for (i = 0; i < info->di_num_dma; i++) {
                channel = kmem_zalloc(sizeof (*channel), KM_SLEEP);
                channel->ch_device = device;
                channel->ch_removing = B_FALSE;
                channel->ch_ref_cnt = 0;
                channel->ch_cb = info->di_cb;

                e = info->di_cb->cb_channel_alloc(device_private, channel,
                    DCOPY_SLEEP, dcopy_channel_size, &channel->ch_info,
                    &channel->ch_channel_private);
                if (e != DCOPY_SUCCESS) {
                        kmem_free(channel, sizeof (*channel));
                        goto registerfail_alloc;
                }

                e = dcopy_stats_init(channel);
                if (e != DCOPY_SUCCESS) {
                        info->di_cb->cb_channel_free(
                            &channel->ch_channel_private);
                        kmem_free(channel, sizeof (*channel));
                        goto registerfail_alloc;
                }

                e = dcopy_list_init(&channel->ch_poll_list,
                    sizeof (struct dcopy_cmd_priv_s),
                    offsetof(struct dcopy_cmd_priv_s, pr_poll_list_node));
                if (e != DCOPY_SUCCESS) {
                        dcopy_stats_fini(channel);
                        info->di_cb->cb_channel_free(
                            &channel->ch_channel_private);
                        kmem_free(channel, sizeof (*channel));
                        goto registerfail_alloc;
                }

                dcopy_list_push(&device->dc_devchan_list, channel);
        }

        /* add the device to device list */
        dcopy_list_push(&dcopy_statep->d_device_list, device);

        /*
         * add the device's dma channels to the global channel list (where
         * dcopy_alloc's come from)
         */
        mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
        mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
        channel = list_head(&device->dc_devchan_list.dl_list);
        while (channel != NULL) {
                list_insert_tail(&dcopy_statep->d_globalchan_list.dl_list,
                    channel);
                dcopy_statep->d_globalchan_list.dl_cnt++;
                channel = list_next(&device->dc_devchan_list.dl_list, channel);
        }
        mutex_exit(&dcopy_statep->d_device_list.dl_mutex);
        mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);

        *handle = device;

        /* last call-back into kernel for dcopy KAPI enabled */
        uioa_dcopy_enable();

        return (DCOPY_SUCCESS);

registerfail_alloc:
        channel = list_head(&device->dc_devchan_list.dl_list);
        while (channel != NULL) {
                /* remove from the list */
                channel = dcopy_list_pop(&device->dc_devchan_list);
                ASSERT(channel != NULL);

                dcopy_list_fini(&channel->ch_poll_list);
                dcopy_stats_fini(channel);
                info->di_cb->cb_channel_free(&channel->ch_channel_private);
                kmem_free(channel, sizeof (*channel));
        }

        dcopy_list_fini(&device->dc_devchan_list);
registerfail_devchan:
        kmem_free(device, sizeof (*device));

        return (DCOPY_FAILURE);
}


/*
 * dcopy_device_unregister()
 */
/*ARGSUSED*/
int
dcopy_device_unregister(dcopy_device_handle_t *handle)
{
        struct dcopy_channel_s *channel;
        dcopy_device_handle_t device;
        boolean_t device_busy;

        /* first call-back into kernel for dcopy KAPI disable */
        uioa_dcopy_disable();

        device = *handle;
        device_busy = B_FALSE;

        /*
         * remove the devices dma channels from the global channel list (where
         * dcopy_alloc's come from)
         */
        mutex_enter(&dcopy_statep->d_globalchan_list.dl_mutex);
        mutex_enter(&device->dc_devchan_list.dl_mutex);
        channel = list_head(&device->dc_devchan_list.dl_list);
        while (channel != NULL) {
                /*
                 * if the channel has outstanding allocs, mark it as having
                 * to be removed and increment the number of channels which
                 * need to be removed in the device state too.
                 */
                if (channel->ch_ref_cnt != 0) {
                        channel->ch_removing = B_TRUE;
                        device_busy = B_TRUE;
                        device->dc_removing_cnt++;
                }
                dcopy_statep->d_globalchan_list.dl_cnt--;
                list_remove(&dcopy_statep->d_globalchan_list.dl_list, channel);
                channel = list_next(&device->dc_devchan_list.dl_list, channel);
        }
        mutex_exit(&device->dc_devchan_list.dl_mutex);
        mutex_exit(&dcopy_statep->d_globalchan_list.dl_mutex);

        /*
         * if there are channels which still need to be removed, we will clean
         * up the device state after they are freed up.
         */
        if (device_busy) {
                return (DCOPY_PENDING);
        }

        dcopy_device_cleanup(device, B_FALSE);

        *handle = NULL;
        return (DCOPY_SUCCESS);
}


/*
 * dcopy_device_cleanup()
 */
static void
dcopy_device_cleanup(dcopy_device_handle_t device, boolean_t do_callback)
{
        struct dcopy_channel_s *channel;

        /*
         * remove all the channels in the device list, free them, and clean up
         * the state.
         */
        mutex_enter(&dcopy_statep->d_device_list.dl_mutex);
        channel = list_head(&device->dc_devchan_list.dl_list);
        while (channel != NULL) {
                device->dc_devchan_list.dl_cnt--;
                list_remove(&device->dc_devchan_list.dl_list, channel);
                dcopy_list_fini(&channel->ch_poll_list);
                dcopy_stats_fini(channel);
                channel->ch_cb->cb_channel_free(&channel->ch_channel_private);
                kmem_free(channel, sizeof (*channel));
                channel = list_head(&device->dc_devchan_list.dl_list);
        }

        /* remove it from the list of devices */
        list_remove(&dcopy_statep->d_device_list.dl_list, device);

        mutex_exit(&dcopy_statep->d_device_list.dl_mutex);

        /*
         * notify the DMA device driver that the device is free to be
         * detached.
         */
        if (do_callback) {
                device->dc_cb->cb_unregister_complete(
                    device->dc_device_private, DCOPY_SUCCESS);
        }

        dcopy_list_fini(&device->dc_devchan_list);
        kmem_free(device, sizeof (*device));
}


/*
 * dcopy_device_channel_notify()
 */
/*ARGSUSED*/
void
dcopy_device_channel_notify(dcopy_handle_t handle, int status)
{
        struct dcopy_channel_s *channel;
        dcopy_list_t *poll_list;
        dcopy_cmd_priv_t priv;
        int e;


        ASSERT(status == DCOPY_COMPLETION);
        channel = handle;

        poll_list = &channel->ch_poll_list;

        /*
         * when we get a completion notification from the device, go through
         * all of the commands blocking on this channel and see if they have
         * completed. Remove the command and wake up the block thread if they
         * have. Once we hit a command which is still pending, we are done
         * polling since commands in a channel complete in order.
         */
        mutex_enter(&poll_list->dl_mutex);
        if (poll_list->dl_cnt != 0) {
                priv = list_head(&poll_list->dl_list);
                while (priv != NULL) {
                        atomic_inc_64(&channel->
                            ch_stat.cs_notify_poll.value.ui64);
                        e = channel->ch_cb->cb_cmd_poll(
                            channel->ch_channel_private,
                            priv->pr_cmd);
                        if (e == DCOPY_PENDING) {
                                atomic_inc_64(&channel->
                                    ch_stat.cs_notify_pending.value.ui64);
                                break;
                        }

                        poll_list->dl_cnt--;
                        list_remove(&poll_list->dl_list, priv);

                        mutex_enter(&priv->pr_mutex);
                        priv->pr_wait = B_FALSE;
                        cv_signal(&priv->pr_cv);
                        mutex_exit(&priv->pr_mutex);

                        priv = list_head(&poll_list->dl_list);
                }
        }

        mutex_exit(&poll_list->dl_mutex);
}


/*
 * dcopy_stats_init()
 */
static int
dcopy_stats_init(dcopy_handle_t channel)
{
#define CHANSTRSIZE     20
        char chanstr[CHANSTRSIZE];
        dcopy_stats_t *stats;
        int instance;
        char *name;


        stats = &channel->ch_stat;
        name = (char *)ddi_driver_name(channel->ch_device->dc_info.di_dip);
        instance = ddi_get_instance(channel->ch_device->dc_info.di_dip);

        (void) snprintf(chanstr, CHANSTRSIZE, "channel%d",
            (uint32_t)channel->ch_info.qc_chan_num);

        channel->ch_kstat = kstat_create(name, instance, chanstr, "misc",
            KSTAT_TYPE_NAMED, sizeof (dcopy_stats_t) / sizeof (kstat_named_t),
            KSTAT_FLAG_VIRTUAL);
        if (channel->ch_kstat == NULL) {
                return (DCOPY_FAILURE);
        }
        channel->ch_kstat->ks_data = stats;

        kstat_named_init(&stats->cs_bytes_xfer, "bytes_xfer",
            KSTAT_DATA_UINT64);
        kstat_named_init(&stats->cs_cmd_alloc, "cmd_alloc",
            KSTAT_DATA_UINT64);
        kstat_named_init(&stats->cs_cmd_post, "cmd_post",
            KSTAT_DATA_UINT64);
        kstat_named_init(&stats->cs_cmd_poll, "cmd_poll",
            KSTAT_DATA_UINT64);
        kstat_named_init(&stats->cs_notify_poll, "notify_poll",
            KSTAT_DATA_UINT64);
        kstat_named_init(&stats->cs_notify_pending, "notify_pending",
            KSTAT_DATA_UINT64);
        kstat_named_init(&stats->cs_id, "id",
            KSTAT_DATA_UINT64);
        kstat_named_init(&stats->cs_capabilities, "capabilities",
            KSTAT_DATA_UINT64);

        kstat_install(channel->ch_kstat);

        channel->ch_stat.cs_id.value.ui64 = channel->ch_info.qc_id;
        channel->ch_stat.cs_capabilities.value.ui64 =
            channel->ch_info.qc_capabilities;

        return (DCOPY_SUCCESS);
}


/*
 * dcopy_stats_fini()
 */
static void
dcopy_stats_fini(dcopy_handle_t channel)
{
        kstat_delete(channel->ch_kstat);
}
/* *** END OF DEVICE INTERFACE *** */